diff --git a/.gitignore b/.gitignore index e2ca956..beb446c 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,10 @@ build/ .DS_Store # Ignore Windows thumbnail cache -Thumbs.db \ No newline at end of file +Thumbs.db +gui/__pycache__ +__pycache__ + + +Testfiles/TestOutputs +Testfiles_ diff --git a/.vscode/settings.json b/.vscode/settings.json index 14d694d..33d89d2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,9 +8,6 @@ ".vscode": true, ".vs": true, ".lh": true, - "__pycache__": true, - "Deprecated-POC": true, - "BlenderDocumentation": true, - "PythonCheatsheats": true + "__pycache__": true } } \ No newline at end of file diff --git a/AUTOTEST_GUI_PLAN.md b/AUTOTEST_GUI_PLAN.md new file mode 100644 index 0000000..0296dd1 --- /dev/null +++ b/AUTOTEST_GUI_PLAN.md @@ -0,0 +1,112 @@ +# Plan for Autotest GUI Mode Implementation + +**I. Objective:** +Create an `autotest.py` script that can launch the Asset Processor GUI headlessly, load a predefined asset (`.zip`), select a predefined preset, verify the predicted rule structure against an expected JSON, trigger processing to a predefined output directory, check the output, and analyze logs for errors or specific messages. This serves as a sanity check for core GUI-driven workflows. + +**II. `TestFiles` Directory:** +A new directory named `TestFiles` will be created in the project root (`c:/Users/Theis/Assetprocessor/Asset-Frameworker/TestFiles/`). This directory will house: +* Sample asset `.zip` files for testing (e.g., `TestFiles/SampleAsset1.zip`). +* Expected rule structure JSON files (e.g., `TestFiles/SampleAsset1_PresetX_expected_rules.json`). +* A subdirectory for test outputs (e.g., `TestFiles/TestOutputs/`). + +**III. `autotest.py` Script:** + +1. **Location:** `c:/Users/Theis/Assetprocessor/Asset-Frameworker/autotest.py` (or `scripts/autotest.py`). +2. **Command-Line Arguments (with defaults pointing to `TestFiles/`):** + * `--zipfile`: Path to the test asset. Default: `TestFiles/default_test_asset.zip`. + * `--preset`: Name of the preset. Default: `DefaultTestPreset`. + * `--expectedrules`: Path to expected rules JSON. Default: `TestFiles/default_test_asset_rules.json`. + * `--outputdir`: Path for processing output. Default: `TestFiles/TestOutputs/DefaultTestOutput`. + * `--search` (optional): Log search term. Default: `None`. + * `--additional-lines` (optional): Context lines for log search. Default: `0`. +3. **Core Structure:** + * Imports necessary modules from the main application and PySide6. + * Adds project root to `sys.path` for imports. + * `AutoTester` class: + * **`__init__(self, app_instance: App)`:** + * Stores `app_instance` and `main_window`. + * Initializes `QEventLoop`. + * Connects `app_instance.all_tasks_finished` to `self._on_all_tasks_finished`. + * Loads expected rules from the `--expectedrules` file. + * **`run_test(self)`:** Orchestrates the test steps sequentially: + 1. Load ZIP (`main_window.add_input_paths()`). + 2. Select Preset (`main_window.preset_editor_widget.editor_preset_list.setCurrentItem()`). + 3. Await Prediction (using `QTimer` to poll `main_window._pending_predictions`, manage with `QEventLoop`). + 4. Retrieve & Compare Rulelist: + * Get actual rules: `main_window.unified_model.get_all_source_rules()`. + * Convert actual rules to comparable dict (`_convert_rules_to_comparable()`). + * Compare with loaded expected rules (`_compare_rules()`). If mismatch, log and fail. + 5. Start Processing (emit `main_window.start_backend_processing` with rules and output settings). + 6. Await Processing (use `QEventLoop` waiting for `_on_all_tasks_finished`). + 7. Check Output Path (verify existence of output dir, list contents, basic sanity checks like non-emptiness or presence of key asset folders). + 8. Retrieve & Analyze Logs (`main_window.log_console.log_console_output.toPlainText()`, filter by `--search`, check for tracebacks). + 9. Report result and call `cleanup_and_exit()`. + * **`_check_prediction_status(self)`:** Slot for prediction polling timer. + * **`_on_all_tasks_finished(self, processed_count, skipped_count, failed_count)`:** Slot for `App.all_tasks_finished` signal. + * **`_convert_rules_to_comparable(self, source_rules_list: List[SourceRule]) -> dict`:** Converts `SourceRule` objects to the JSON structure defined below. + * **`_compare_rules(self, actual_rules_data: dict, expected_rules_data: dict) -> bool`:** Implements Option 1 comparison logic: + * Errors if an expected field is missing or its value mismatches. + * Logs (but doesn't error on) fields present in actual but not in expected. + * **`_process_and_display_logs(self, logs_text: str)`:** Handles log filtering/display. + * **`cleanup_and_exit(self, success=True)`:** Quits `QCoreApplication` and `sys.exit()`. + * `main()` function: + * Parses CLI arguments. + * Initializes `QApplication`. + * Instantiates `main.App()` (does *not* show the GUI). + * Instantiates `AutoTester(app_instance)`. + * Uses `QTimer.singleShot(0, tester.run_test)` to start the test. + * Runs `q_app.exec()`. + +**IV. `expected_rules.json` Structure (Revised):** +Located in `TestFiles/`. Example: `TestFiles/SampleAsset1_PresetX_expected_rules.json`. +```json +{ + "source_rules": [ + { + "input_path": "SampleAsset1.zip", + "supplier_identifier": "ExpectedSupplier", + "preset_name": "PresetX", + "assets": [ + { + "asset_name": "AssetNameFromPrediction", + "asset_type": "Prop", + "files": [ + { + "file_path": "relative/path/to/file1.png", + "item_type": "MAP_COL", + "target_asset_name_override": null + } + ] + } + ] + } + ] +} +``` + +**V. Mermaid Diagram of Autotest Flow:** +```mermaid +graph TD + A[Start autotest.py with CLI Args (defaults to TestFiles/)] --> B{Setup Args & Logging}; + B --> C[Init QApplication & main.App (GUI Headless)]; + C --> D[Instantiate AutoTester(app_instance)]; + D --> E[QTimer.singleShot -> AutoTester.run_test()]; + + subgraph AutoTester.run_test() + E --> F[Load Expected Rules from --expectedrules JSON]; + F --> G[Load ZIP (--zipfile) via main_window.add_input_paths()]; + G --> H[Select Preset (--preset) via main_window.preset_editor_widget]; + H --> I[Await Prediction (Poll main_window._pending_predictions via QTimer & QEventLoop)]; + I -- Prediction Done --> J[Get Actual Rules from main_window.unified_model]; + J --> K[Convert Actual Rules to Comparable JSON Structure]; + K --> L{Compare Actual vs Expected Rules (Option 1 Logic)}; + L -- Match --> M[Start Processing (Emit main_window.start_backend_processing with --outputdir)]; + L -- Mismatch --> ZFAIL[Log Mismatch & Call cleanup_and_exit(False)]; + M --> N[Await Processing (QEventLoop for App.all_tasks_finished signal)]; + N -- Processing Done --> O[Check Output Dir (--outputdir): Exists? Not Empty? Key Asset Folders?]; + O --> P[Retrieve & Analyze Logs (Search, Tracebacks)]; + P --> Q[Log Test Success & Call cleanup_and_exit(True)]; + end + + ZFAIL --> ZEND[AutoTester.cleanup_and_exit() -> QCoreApplication.quit() & sys.exit()]; + Q --> ZEND; \ No newline at end of file diff --git a/Deprecated/POC/Blender-MaterialsFromNodegroups.py b/Deprecated/POC/Blender-MaterialsFromNodegroups.py deleted file mode 100644 index 15511b5..0000000 --- a/Deprecated/POC/Blender-MaterialsFromNodegroups.py +++ /dev/null @@ -1,291 +0,0 @@ -import bpy -from pathlib import Path -import time -import os -import math - -# Try importing NumPy -try: - import numpy as np - numpy_available = True - # print("NumPy module found.") # Less verbose -except ImportError: - print("Warning: NumPy module not found. Median calc disabled, mean uses loop.") - numpy_available = False - -# --- Configuration --- -ASSET_LIBRARY_NAME = "Nodes-Linked" # <<< Name of Asset Library in Prefs -TEMPLATE_MATERIAL_NAME = "Template_PBRMaterial" # <<< Name of template Material in current file -PLACEHOLDER_NODE_LABEL = "PBRSET_PLACEHOLDER" # <<< Label of placeholder node in template mat -ASSET_NAME_PREFIX = "PBRSET_" # <<< Prefix of Node Group assets to process -MATERIAL_NAME_PREFIX = "Mat_" # <<< Prefix for created Materials -THUMBNAIL_PROPERTY_NAME = "thumbnail_filepath" # <<< Custom property name on Node Groups -VALID_EXTENSIONS = {".jpg", ".jpeg", ".png", ".tif", ".tiff"} -DERIVED_MAP_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.tif', '.tiff'] -VIEWPORT_GAMMA = 0.4 -SCALED_SIZE = (32, 32) # Downscale target size for calculations - -# --- >>> SET MATERIAL CREATION LIMIT HERE <<< --- -# Max number of *new* materials created per run (0 = no limit) -MATERIAL_CREATION_LIMIT = 900 -# ------------------------------------------------ - -# --- Helper Functions --- -def find_node_by_label(node_tree, label, node_type=None): - # Finds first node by label and optional type (using node.type) - if not node_tree: return None - for node in node_tree.nodes: - if node.label and node.label == label: - if node_type is None or node.type == node_type: return node - return None - -def calculate_value_from_image(image, target_size=(64, 64), mode='color', method='median'): - # Calculates median/mean from downscaled image copy, cleans up temp image - temp_img = None; #... (Full implementation from previous step) ... - if not image: return None - try: - if not image.has_data: - try: _ = len(image.pixels); image.update() - except Exception: pass - if not image.has_data: return None # Cannot proceed - temp_img = image.copy() - if not temp_img: return None - temp_img.scale(target_size[0], target_size[1]) - try: _ = len(temp_img.pixels); temp_img.update() - except Exception: pass # Ignore access error, check has_data - if not temp_img.has_data: return None - width=temp_img.size[0]; height=temp_img.size[1]; channels=temp_img.channels - if width == 0 or height == 0 or channels == 0: return None - pixels = temp_img.pixels[:]; result_value = None; - if numpy_available: # Use NumPy - np_pixels = np.array(pixels); num_elements = len(np_pixels); num_pixels_actual = num_elements // channels; - if num_pixels_actual == 0: return None - np_pixels = np_pixels[:num_pixels_actual * channels]; pixels_reshaped = np_pixels.reshape((num_pixels_actual, channels)) - if mode == 'color': # Color Median/Mean (NumPy) - if channels < 3: return None - calc_linear = np.median(pixels_reshaped[:, :3], axis=0) if method == 'median' else np.mean(pixels_reshaped[:, :3], axis=0) - inv_gamma = 1.0 / VIEWPORT_GAMMA; calc_linear_clamped = np.clip(calc_linear, 0.0, None) - calc_srgb_np = np.power(calc_linear_clamped, inv_gamma); calc_srgb_clamped = np.clip(calc_srgb_np, 0.0, 1.0) - result_value = (calc_srgb_clamped[0], calc_srgb_clamped[1], calc_srgb_clamped[2], 1.0) - elif mode == 'grayscale': # Grayscale Median/Mean (NumPy) - calc_val = np.median(pixels_reshaped[:, 0]) if method == 'median' else np.mean(pixels_reshaped[:, 0]) - result_value = min(max(0.0, calc_val), 1.0) - elif method == 'mean': # Fallback Mean Loop - # print(" Calculating mean using standard loop...") # Verbose - actual_len = len(pixels); #... (Mean loop logic) ... - if actual_len == 0: return None; num_pixels_in_buffer=actual_len//channels; max_elements=num_pixels_in_buffer*channels - if num_pixels_in_buffer == 0: return None - if mode == 'color': - sum_r,sum_g,sum_b = 0.0,0.0,0.0; step=channels - for i in range(0, max_elements, step): - if i+2 >= actual_len: break; sum_r+=pixels[i]; sum_g+=pixels[i+1]; sum_b+=pixels[i+2] - avg_r_lin,avg_g_lin,avg_b_lin = sum_r/num_pixels_in_buffer, sum_g/num_pixels_in_buffer, sum_b/num_pixels_in_buffer - inv_gamma = 1.0/VIEWPORT_GAMMA - avg_r_srgb,avg_g_srgb,avg_b_srgb = min(max(0.0,pow(max(0.0,avg_r_lin),inv_gamma)),1.0), min(max(0.0,pow(max(0.0,avg_g_lin),inv_gamma)),1.0), min(max(0.0,pow(max(0.0,avg_b_lin),inv_gamma)),1.0) - result_value = (avg_r_srgb, avg_g_srgb, avg_b_srgb, 1.0) - elif mode == 'grayscale': - sum_val=0.0; step=channels - for i in range(0, max_elements, step): sum_val+=pixels[i] - result_value = min(max(0.0, sum_val/num_pixels_in_buffer), 1.0) - else: print(" Error: NumPy required for median calculation."); return None - return result_value - except Exception as e: print(f" Error during value calculation for '{image.name}': {e}"); return None - finally: # Cleanup - if temp_img: - try: bpy.data.images.remove(temp_img, do_unlink=True) - except Exception: pass # Ignore cleanup errors - - -# --- Main Function --- -def create_materials_for_library_assets(library_name): - start_time = time.time(); print(f"--- Starting Material Creation for Library '{library_name}' ---") - print(f"Material Creation Limit per run: {'Unlimited' if MATERIAL_CREATION_LIMIT <= 0 else MATERIAL_CREATION_LIMIT}") - # (Prerequisite checks...) - template_mat=bpy.data.materials.get(TEMPLATE_MATERIAL_NAME); #... etc ... - if not template_mat or not template_mat.use_nodes or not find_node_by_label(template_mat.node_tree, PLACEHOLDER_NODE_LABEL, 'GROUP'): print("Template Prereq Failed."); return - library=bpy.context.preferences.filepaths.asset_libraries.get(library_name); #... etc ... - if not library or not Path(bpy.path.abspath(library.path)).exists(): print("Library Prereq Failed."); return - print(f"Found template material and library path...") - - # (File scanning...) - materials_created=0; materials_skipped=0; nodegroups_processed=0; link_errors=0; files_to_process=[]; library_path_obj=Path(bpy.path.abspath(library.path)) - #... (populate files_to_process) ... - if library_path_obj.is_dir(): - for item in library_path_obj.iterdir(): - if item.is_file() and item.suffix.lower() == '.blend': files_to_process.append(str(item)) - if not files_to_process: print(f"Warning: No .blend files found in dir: {library_path_obj}") - elif library_path_obj.is_file() and library_path_obj.suffix.lower() == '.blend': - files_to_process.append(str(library_path_obj)) - else: print(f"Error: Library path not dir or .blend: {library_path_obj}"); return - print(f"Found {len(files_to_process)} .blend file(s) to inspect.") - - # Initialize counters and flag for limit - created_in_this_run = 0 - limit_reached_flag = False - - for blend_file_path in files_to_process: # ... (inspect loop) ... - print(f"\nInspecting library file: {os.path.basename(blend_file_path)}...") - potential_nodegroups = []; # ... (inspection logic) ... - try: - with bpy.data.libraries.load(blend_file_path, link=False) as (data_from, data_to): potential_nodegroups = list(data_from.node_groups) - except Exception as e_load_inspect: print(f" Error inspecting file '{blend_file_path}': {e_load_inspect}"); continue - print(f" Found {len(potential_nodegroups)} NGs. Checking for '{ASSET_NAME_PREFIX}'...") - - - for asset_nodegroup_name in potential_nodegroups: # ... (NG loop) ... - if not asset_nodegroup_name.startswith(ASSET_NAME_PREFIX): continue - nodegroups_processed += 1 - base_name = asset_nodegroup_name.removeprefix(ASSET_NAME_PREFIX) - material_name = f"{MATERIAL_NAME_PREFIX}{base_name}" - if bpy.data.materials.get(material_name): materials_skipped += 1; continue - - linked_nodegroup = None; preview_path = None - try: # --- Start Main Processing Block for NG --- - # (Linking logic...) - existing_group = bpy.data.node_groups.get(asset_nodegroup_name); #... etc linking ... - is_correctly_linked = (existing_group and existing_group.library and bpy.path.abspath(existing_group.library.filepath) == blend_file_path) - if is_correctly_linked: linked_nodegroup = existing_group - else: # Link it - with bpy.data.libraries.load(blend_file_path, link=True, relative=False) as (data_from, data_to): - if asset_nodegroup_name in data_from.node_groups: data_to.node_groups = [asset_nodegroup_name] - else: print(f" Error: NG '{asset_nodegroup_name}' not found during link."); continue # Skip NG - linked_nodegroup = bpy.data.node_groups.get(asset_nodegroup_name) - if not linked_nodegroup or not linked_nodegroup.library: print(f" Error: NG '{asset_nodegroup_name}' link failed."); linked_nodegroup = None; link_errors += 1 - if not linked_nodegroup: print(f" Failed link NG '{asset_nodegroup_name}'. Skip."); continue # Skip NG - - preview_path = linked_nodegroup.get(THUMBNAIL_PROPERTY_NAME) # Path to COL-1 1K - - # (Duplicate, Rename, Replace Placeholder...) - new_material = template_mat.copy(); #... checks ... - if not new_material: print(f" Error: Failed copy template mat. Skip."); continue - new_material.name = material_name - if not new_material.use_nodes or not new_material.node_tree: print(f" Error: New mat '{material_name}' no nodes."); continue - placeholder_node = find_node_by_label(new_material.node_tree, PLACEHOLDER_NODE_LABEL, 'GROUP'); #... checks ... - if not placeholder_node: print(f" Error: Placeholder '{PLACEHOLDER_NODE_LABEL}' not found."); continue - placeholder_node.node_tree = linked_nodegroup - print(f" Created material '{material_name}' and linked NG '{linked_nodegroup.name}'.") - - # --- Load base COL-1 image once --- - thumbnail_image = None - if preview_path and Path(preview_path).is_file(): - try: thumbnail_image = bpy.data.images.load(preview_path, check_existing=True) - except Exception as e_load_base: print(f" Error loading base thumbnail '{preview_path}': {e_load_base}") - - # --- Set Viewport Color (Median) --- - median_color = None - if thumbnail_image: median_color = calculate_value_from_image(thumbnail_image, target_size=SCALED_SIZE, mode='color', method='median') - if median_color: new_material.diffuse_color = median_color; print(f" Set viewport color: {median_color[:3]}") - else: print(f" Warn: Could not set viewport color.") - - - # --- Determine Paths and Metal Map Existence --- - roughness_path = None; metallic_path = None; metal_map_found = False; #... etc ... - if preview_path and "_COL-1" in preview_path: - try: # ... path derivation logic ... - base_path_obj=Path(preview_path); directory=base_path_obj.parent; base_stem=base_path_obj.stem - if "_COL-1" in base_stem: - rough_stem=base_stem.replace("_COL-1", "_ROUGH") - for ext in DERIVED_MAP_EXTENSIONS: - potential_path=directory/f"{rough_stem}{ext}"; - if potential_path.is_file(): roughness_path=str(potential_path); break - metal_stem=base_stem.replace("_COL-1", "_METAL") - for ext in DERIVED_MAP_EXTENSIONS: - potential_path=directory/f"{metal_stem}{ext}"; - if potential_path.is_file(): metallic_path=str(potential_path); metal_map_found=True; break - except Exception as e_derive: print(f" Error deriving paths: {e_derive}") - if not metal_map_found: print(f" Info: No METAL map found. Assuming Spec/Gloss.") - - # --- Set Viewport Roughness (Median, Conditional Inversion) --- - median_roughness = None; # ... etc ... - if roughness_path: - try: rough_img = bpy.data.images.load(roughness_path, check_existing=True) - except Exception as e_load_rough: print(f" Error loading rough image: {e_load_rough}") - if rough_img: median_roughness = calculate_value_from_image(rough_img, target_size=SCALED_SIZE, mode='grayscale', method='median') - else: print(f" Error: load None for rough path.") - if median_roughness is not None: - final_roughness_value = median_roughness - if not metal_map_found: final_roughness_value = 1.0 - median_roughness; print(f" Inverting ROUGH->Gloss: {median_roughness:.3f} -> {final_roughness_value:.3f}") - new_material.roughness = min(max(0.0, final_roughness_value), 1.0); print(f" Set viewport roughness: {new_material.roughness:.3f}") - else: print(f" Warn: Could not set viewport roughness.") - - - # --- Set Viewport Metallic (Median) --- - median_metallic = None; # ... etc ... - if metal_map_found: - try: metal_img = bpy.data.images.load(metallic_path, check_existing=True) - except Exception as e_load_metal: print(f" Error loading metal image: {e_load_metal}") - if metal_img: median_metallic = calculate_value_from_image(metal_img, target_size=SCALED_SIZE, mode='grayscale', method='median') - else: print(f" Error: load None for metal path.") - if median_metallic is not None: new_material.metallic = median_metallic; print(f" Set viewport metallic: {median_metallic:.3f}") - else: new_material.metallic = 0.0; # Default - if metal_map_found: print(f" Warn: Could not calc viewport metallic. Set 0.0.") - else: print(f" Set viewport metallic to default: 0.0") - - - # --- Mark Material as Asset --- - mat_asset_data = None; # ... (logic remains same) ... - try: # ... asset marking ... - if not new_material.asset_data: new_material.asset_mark(); print(f" Marked material as asset.") - mat_asset_data = new_material.asset_data - except Exception as e_asset: print(f" Error marking mat asset: {e_asset}") - - # --- Copy Asset Tags --- - if mat_asset_data and linked_nodegroup.asset_data: # ... (logic remains same) ... - try: # ... tag copying ... - source_tags=linked_nodegroup.asset_data.tags; target_tags=mat_asset_data.tags - tags_copied_count=0; existing_target_tag_names={t.name for t in target_tags} - for src_tag in source_tags: - if src_tag.name not in existing_target_tag_names: target_tags.new(name=src_tag.name); tags_copied_count += 1 - if tags_copied_count > 0: print(f" Copied {tags_copied_count} asset tags.") - except Exception as e_tags: print(f" Error copying tags: {e_tags}") - - # --- Set Custom Preview for Material --- - if preview_path and Path(preview_path).is_file(): # ... (logic remains same) ... - try: # ... preview setting ... - with bpy.context.temp_override(id=new_material): bpy.ops.ed.lib_id_load_custom_preview(filepath=preview_path) - except RuntimeError as e_op: print(f" Error running preview op for mat '{new_material.name}': {e_op}") - except Exception as e_prev: print(f" Unexpected preview error for mat: {e_prev}") - elif preview_path: print(f" Warn: Thumb path not found for preview step: '{preview_path}'") - - - # --- Increment Counters & Check Limit --- - materials_created += 1 # Overall counter for summary - created_in_this_run += 1 # Counter for this run's limit - - # Check limit AFTER successful creation - if MATERIAL_CREATION_LIMIT > 0 and created_in_this_run >= MATERIAL_CREATION_LIMIT: - print(f"\n--- Material Creation Limit ({MATERIAL_CREATION_LIMIT}) Reached ---") - limit_reached_flag = True - break # Exit inner loop - - except Exception as e: # Catch errors for the whole NG processing block - print(f" An unexpected error occurred processing NG '{asset_nodegroup_name}': {e}") - # --- End Main Processing Block for NG --- - - # Check flag to stop outer loop - if limit_reached_flag: - print("Stopping library file iteration due to limit.") - break # Exit outer loop - - # (Completion summary...) - end_time = time.time(); duration = end_time - start_time; print("\n--- Material Creation Finished ---"); # ... etc ... - print(f"Duration: {duration:.2f} seconds") - print(f"Summary: Processed {nodegroups_processed} NGs. Created {materials_created} Mats this run. Skipped {materials_skipped}. Link Errors {link_errors}.") - if limit_reached_flag: print(f"NOTE: Script stopped early due to creation limit ({MATERIAL_CREATION_LIMIT}). Run again to process more.") - -# --- How to Run --- -# 1. Rerun Script 1 to add "thumbnail_filepath" property. -# 2. Setup Asset Library in Prefs. Set ASSET_LIBRARY_NAME below. -# 3. In current file, create "Template_PBRMaterial" with "PBRSET_PLACEHOLDER" node. -# 4. Set MATERIAL_CREATION_LIMIT in Config section above (0 for unlimited). -# 5. Paste script & Run (Alt+P). - -if __name__ == "__main__": - # Only need ASSET_LIBRARY_NAME configuration here now - if ASSET_LIBRARY_NAME == "My Asset Library": # Default check - print("\nERROR: Please update the 'ASSET_LIBRARY_NAME' variable in the script's Configuration section.") - print(" Set it to the name of your asset library in Blender Preferences before running.\n") - elif not bpy.data.materials.get(TEMPLATE_MATERIAL_NAME): - print(f"\nERROR: Template material '{TEMPLATE_MATERIAL_NAME}' not found in current file.\n") - else: - create_materials_for_library_assets(ASSET_LIBRARY_NAME) \ No newline at end of file diff --git a/Deprecated/POC/Blender-NodegroupsFromPBRSETS.py b/Deprecated/POC/Blender-NodegroupsFromPBRSETS.py deleted file mode 100644 index 66e5917..0000000 --- a/Deprecated/POC/Blender-NodegroupsFromPBRSETS.py +++ /dev/null @@ -1,988 +0,0 @@ -# Full script - PBR Texture Importer V4 (Manifest, Auto-Save/Reload, Aspect Ratio, Asset Tags) - -import bpy -import os # For auto-save rename/remove -from pathlib import Path -import time -import base64 -import numpy as np # For stats calculation -import json # For manifest handling -import re # For parsing scaling string - -# --- USER CONFIGURATION --- -# File Paths & Templates -texture_root_directory = r"G:\02 Content\10-19 Content\13 Textures Power of Two\13.00" # <<< CHANGE THIS PATH! -PARENT_TEMPLATE_NAME = "Template_PBRSET" # Name of the parent node group template -CHILD_TEMPLATE_NAME = "Template_PBRTYPE" # Name of the child node group template - -# Processing Limits & Intervals -MAX_NEW_GROUPS_PER_RUN = 1000 # Max NEW parent groups created per run before stopping -SAVE_INTERVAL = 25 # Auto-save interval during NEW group creation (every N groups) - -# Features & Behavior -AUTO_SAVE_ENABLED = True # Enable periodic auto-saving (main file + manifest) during processing? -AUTO_RELOAD_ON_FINISH = True # Save and reload the blend file upon successful script completion? - -# Naming & Structure Conventions -VALID_EXTENSIONS = {".jpg", ".jpeg", ".png", ".tif", ".tiff"} # Allowed texture file types -RESOLUTION_LABELS = ["1k", "2k", "4k", "8k"] # Expected resolution labels (LOWEST FIRST for aspect/tag calc) -SG_VALUE_NODE_LABEL = "SpecularGlossy" # Label for the Specular/Glossy value node in parent template -HISTOGRAM_NODE_PREFIX = "Histogram-" # Prefix for Combine XYZ nodes storing stats (e.g., "Histogram-ROUGH") -ASPECT_RATIO_NODE_LABEL = "AspectRatioCorrection" # Label for the Value node storing the aspect ratio correction factor - -# Texture Map Properties -PBR_COLOR_SPACE_MAP = { # Map PBR type (from filename) to Blender color space - "AO": "sRGB", "COL-1": "sRGB", "COL-2": "sRGB", "COL-3": "sRGB", - "DISP": "Non-Color", "NRM": "Non-Color", "REFL": "Non-Color", "ROUGH": "Non-Color", - "METAL": "Non-Color", "FUZZ": "Non-Color", "MASK": "Non-Color", "SSS": "sRGB", -} -DEFAULT_COLOR_SPACE = "sRGB" # Fallback color space if PBR type not in map - -# --- END USER CONFIGURATION --- - - -# --- Helper Functions --- - -def parse_texture_filename(filename_stem): - """Parses texture filename stem based on expected convention.""" - parts = filename_stem.split('_'); - # Expecting Tag_Groupname_Resolution_Scaling_PBRType - if len(parts) == 5: - return {"Tag": parts[0], "Groupname": parts[1], "Resolution": parts[2], "Scaling": parts[3], "PBRType": parts[4]} - else: - print(f" Warn: Skip '{filename_stem}' - Expected 5 parts, found {len(parts)}."); - return None - -def find_nodes_by_label(node_tree, label, node_type=None): - """Finds ALL nodes in a node tree matching the label and optionally type.""" - if not node_tree: - return [] - matching_nodes = [] - for node in node_tree.nodes: - if node.label and node.label == label: - if node_type is None or node.type == node_type: - matching_nodes.append(node) - return matching_nodes - -def encode_name_b64(name_str): - """Encodes a string using URL-safe Base64 for node group names.""" - try: - return base64.urlsafe_b64encode(name_str.encode('utf-8')).decode('ascii') - except Exception as e: - print(f" Error base64 encoding '{name_str}': {e}"); - return name_str # Fallback to original name on error - -def calculate_image_stats(image): - """Calculates Min, Max, Median of the first channel of a Blender image.""" - if not image: return None - pixels_arr, value_channel_arr, result = None, None, None - try: - width = image.size[0]; height = image.size[1]; channels = image.channels - if width == 0 or height == 0 or channels == 0: - print(f" Warn: Invalid dims for '{image.name}'. Skip stats."); return None - actual_len = len(image.pixels); expected_len = width * height * channels - if expected_len != actual_len: - print(f" Warn: Pixel buffer mismatch for '{image.name}'. Skip stats."); return None - if actual_len == 0: return None - - pixels_arr = np.fromiter(image.pixels, dtype=np.float32, count=actual_len) - - if channels == 1: value_channel_arr = pixels_arr - elif channels >= 2: value_channel_arr = pixels_arr[0::channels] - else: return None - - if value_channel_arr is None or value_channel_arr.size == 0: - print(f" Warn: No value channel for '{image.name}'. Skip stats."); return None - - min_val = float(np.min(value_channel_arr)) - max_val = float(np.max(value_channel_arr)) - median_val = float(np.median(value_channel_arr)) - result = (min_val, max_val, median_val) - - except MemoryError: - print(f" Error: Not enough memory for stats calc on '{image.name}'.") - except Exception as e: - print(f" Error during stats calc for '{image.name}': {e}"); - import traceback; traceback.print_exc() - finally: - # Explicitly delete potentially large numpy arrays - if 'value_channel_arr' in locals() and value_channel_arr is not None: - try: - del value_channel_arr - except NameError: - pass # Ignore if already gone - if 'pixels_arr' in locals() and pixels_arr is not None: - try: - del pixels_arr - except NameError: - pass # Ignore if already gone - return result - -def calculate_aspect_ratio_factor(image_width, image_height, scaling_string): - """Calculates the X-axis UV scaling factor based on image dims and scaling string.""" - if image_height <= 0: - print(" Warn: Image height is zero, cannot calculate aspect ratio. Returning 1.0.") - return 1.0 # Return 1.0 if height is invalid - - # Calculate the actual aspect ratio of the image file - current_aspect_ratio = image_width / image_height - - # Check the scaling string - if scaling_string.upper() == "EVEN": - # 'EVEN' means uniform scaling was applied (or none needed). - # The correction factor is the image's own aspect ratio. - return current_aspect_ratio - else: - # Handle non-uniform scaling cases ("Xnnn", "Ynnn") - match = re.match(r"([XY])(\d+)", scaling_string, re.IGNORECASE) - if not match: - print(f" Warn: Invalid Scaling string format '{scaling_string}'. Returning current ratio {current_aspect_ratio:.4f} as fallback.") - # Fallback to the image's own ratio if scaling string is invalid - return current_aspect_ratio - - axis = match.group(1).upper() - try: - amount = int(match.group(2)) - if amount <= 0: - print(f" Warn: Zero or negative Amount in Scaling string '{scaling_string}'. Returning current ratio {current_aspect_ratio:.4f}.") - return current_aspect_ratio - except ValueError: - print(f" Warn: Invalid Amount in Scaling string '{scaling_string}'. Returning current ratio {current_aspect_ratio:.4f}.") - return current_aspect_ratio - - # Apply the non-uniform correction formula - factor = current_aspect_ratio # Default to current ratio in case of issues below - scaling_factor_percent = amount / 100.0 - try: - if axis == 'X': - if scaling_factor_percent == 0: raise ZeroDivisionError - factor = current_aspect_ratio / scaling_factor_percent - elif axis == 'Y': - factor = current_aspect_ratio * scaling_factor_percent - # No 'else' needed due to regex structure - - except ZeroDivisionError: - print(f" Warn: Division by zero during factor calculation. Returning current ratio {current_aspect_ratio:.4f}.") - return current_aspect_ratio - - return factor - -# --- Manifest Helper Functions --- -def get_manifest_path(context_filepath): - """Gets the expected path for the manifest JSON file based on blend filepath.""" - if not context_filepath: - return None - blend_path = Path(context_filepath) - manifest_filename = f"{blend_path.stem}_manifest.json" - return blend_path.parent / manifest_filename - -def load_manifest(manifest_path): - """Loads the manifest data from the JSON file.""" - if not manifest_path or not manifest_path.exists(): - return {} - try: - with open(manifest_path, 'r', encoding='utf-8') as f: - data = json.load(f) - print(f" Loaded manifest from: {manifest_path.name}") - return data - except json.JSONDecodeError: - print(f"!!! ERROR: Manifest file '{manifest_path.name}' is corrupted. Starting fresh. !!!") - return {} - except Exception as e: - print(f"!!! ERROR: Could not load manifest file '{manifest_path.name}': {e} !!!") - return {} - -def save_manifest(manifest_path, data): - """Saves the manifest data to the JSON file.""" - if not manifest_path: - return False - try: - with open(manifest_path, 'w', encoding='utf-8') as f: - json.dump(data, f, indent=2) - return True - except Exception as e: - print(f"!!!!!!!!!!!!!!!!!!!\n!!! Manifest save FAILED: {e} !!!\n!!!!!!!!!!!!!!!!!!!") - return False - -# --- Auto-Save Helper Function --- -def perform_safe_autosave(manifest_path, manifest_data): - """Performs a safe auto-save of the main blend file and manifest.""" - blend_filepath = bpy.data.filepath - if not blend_filepath or not manifest_path: - print(" Skipping auto-save: Blend file is not saved.") - return False - - print(f"\n--- Attempting Auto-Save ({time.strftime('%H:%M:%S')}) ---") - blend_path = Path(blend_filepath) - manifest_path_obj = Path(manifest_path) # Ensure it's a Path object - - blend_bak_path = blend_path.with_suffix('.blend.bak') - manifest_bak_path = manifest_path_obj.with_suffix('.json.bak') - - # 1. Delete old backups if they exist - try: - if blend_bak_path.exists(): - blend_bak_path.unlink() - if manifest_bak_path.exists(): - manifest_bak_path.unlink() - except OSError as e: - print(f" Warn: Could not delete old backup file: {e}") - # Continue anyway, renaming might still work - - # 2. Rename current files to backup - renamed_blend = False - renamed_manifest = False - try: - if blend_path.exists(): - os.rename(blend_path, blend_bak_path) - renamed_blend = True - # print(f" Renamed '{blend_path.name}' to '{blend_bak_path.name}'") # Optional verbose log - if manifest_path_obj.exists(): - os.rename(manifest_path_obj, manifest_bak_path) - renamed_manifest = True - # print(f" Renamed '{manifest_path_obj.name}' to '{manifest_bak_path.name}'") # Optional verbose log - except OSError as e: - print(f"!!! ERROR: Failed to rename files for backup: {e} !!!") - # Attempt to roll back renames if only one succeeded - if renamed_blend and not renamed_manifest and blend_bak_path.exists(): - print(f" Attempting rollback: Renaming {blend_bak_path.name} back...") - try: - os.rename(blend_bak_path, blend_path) - except OSError as rb_e: - print(f" Rollback rename of blend file FAILED: {rb_e}") - if renamed_manifest and not renamed_blend and manifest_bak_path.exists(): - print(f" Attempting rollback: Renaming {manifest_bak_path.name} back...") - try: - os.rename(manifest_bak_path, manifest_path_obj) - except OSError as rb_e: - print(f" Rollback rename of manifest file FAILED: {rb_e}") - print("--- Auto-Save ABORTED ---") - return False - - # 3. Save new main blend file - save_blend_success = False - try: - bpy.ops.wm.save_mainfile() - print(f" Saved main blend file: {blend_path.name}") - save_blend_success = True - except Exception as e: - print(f"!!!!!!!!!!!!!!!!!!!!!!!!!\n!!! Auto-Save FAILED (Blend File Save): {e} !!!\n!!!!!!!!!!!!!!!!!!!!!!!!!") - # Attempt to restore from backup - print(" Attempting to restore from backup...") - try: - if blend_bak_path.exists(): - os.rename(blend_bak_path, blend_path) - if manifest_bak_path.exists(): - os.rename(manifest_bak_path, manifest_path_obj) - print(" Restored from backup.") - except OSError as re: - print(f"!!! CRITICAL: Failed to restore from backup after save failure: {re} !!!") - print(f"!!! Please check for '.bak' files manually in: {blend_path.parent} !!!") - print("--- Auto-Save ABORTED ---") - return False - - # 4. Save new manifest file (only if blend save succeeded) - if save_blend_success: - if save_manifest(manifest_path, manifest_data): - print(f" Saved manifest file: {manifest_path_obj.name}") - print("--- Auto-Save Successful ---") - return True - else: - # Manifest save failed, but blend file is okay. Warn user. - print("!!! WARNING: Auto-save completed for blend file, but manifest save FAILED. Manifest may be out of sync. !!!") - return True # Still counts as 'completed' in terms of blend file safety - - return False # Should not be reached - -# --- Asset Tagging Helper Functions --- - -def add_tag_if_new(asset_data, tag_name): - """Adds a tag to the asset data if it's not None/empty and doesn't already exist.""" - if not asset_data or not tag_name or not isinstance(tag_name, str) or tag_name.strip() == "": - return False # Invalid input - cleaned_tag_name = tag_name.strip() # Remove leading/trailing whitespace - if not cleaned_tag_name: - return False # Don't add empty tags - - # Check if tag already exists - if cleaned_tag_name not in [t.name for t in asset_data.tags]: - try: - asset_data.tags.new(cleaned_tag_name) - print(f" + Added Asset Tag: '{cleaned_tag_name}'") - return True - except Exception as e: - print(f" Error adding tag '{cleaned_tag_name}': {e}") - return False - else: - # print(f" Tag '{cleaned_tag_name}' already exists.") # Optional info - return False # Not added because it existed - -def get_supplier_tag_from_path(file_path_str, groupname): - """ - Determines supplier tag based on directory structure. - Assumes structure is .../Supplier/Groupname/file.ext or .../Supplier/file.ext - """ - try: - file_path = Path(file_path_str).resolve() - groupname_lower = groupname.lower() - - if not file_path.is_file(): - print(f" Warn (get_supplier_tag): Input path is not a file: {file_path_str}") - return None - - current_dir = file_path.parent # Directory containing the file - if not current_dir: - print(f" Warn (get_supplier_tag): Cannot get parent directory for {file_path_str}") - return None # Cannot determine without parent - - parent_dir = current_dir.parent # Directory potentially containing the 'supplier' name - # Check if we are at the root or have no parent - if not parent_dir or parent_dir == current_dir: - # If the file is in the root scan directory or similar shallow path, - # maybe the directory it's in IS the supplier tag? Or return None? - # Returning current_dir.name might be unexpected, let's return None for safety. - print(f" Warn (get_supplier_tag): File path too shallow to determine supplier reliably: {file_path_str}") - return None - - # Compare the file's directory name with the groupname - if current_dir.name.lower() == groupname_lower: - # Structure is likely .../Supplier/Groupname/file.ext - # Return the name of the directory ABOVE the groupname directory - return parent_dir.name - else: - # Structure is likely .../Supplier/file.ext - # Return the name of the directory CONTAINING the file - return current_dir.name - - except Exception as e: - print(f" Error getting supplier tag for {groupname} from path {file_path_str}: {e}") - return None - -def apply_asset_tags(parent_group, groupname, group_info): - """Applies various asset tags to the parent node group.""" - if not parent_group: - return - - # 1. Ensure group is marked as an asset - try: - if not parent_group.asset_data: - print(f" Marking '{parent_group.name}' as asset for tagging.") - parent_group.asset_mark() - - # Ensure asset_data is available after marking - if not parent_group.asset_data: - print(f" Error: Could not access asset_data for '{parent_group.name}' after marking.") - return - - asset_data = parent_group.asset_data - except Exception as e_mark: - print(f" Error marking group '{parent_group.name}' as asset: {e_mark}") - return # Cannot proceed without asset_data - - # 2. Apply Supplier Tag (Current Requirement) - try: - # Find lowest resolution path (reuse logic from aspect ratio) - lowest_res_path = None; found_res = False - pbr_types_dict = group_info.get("pbr_types", {}) - # Check RESOLUTION_LABELS in order (assuming lowest is first) - for res_label in RESOLUTION_LABELS: - for res_data in pbr_types_dict.values(): # Check all PBR types for this res - if res_label in res_data: - lowest_res_path = res_data[res_label] - found_res = True - break # Found path for this resolution label - if found_res: - break # Found lowest available resolution path - - if lowest_res_path: - supplier_tag = get_supplier_tag_from_path(lowest_res_path, groupname) - if supplier_tag: - add_tag_if_new(asset_data, supplier_tag) # Use helper to add if new - else: - print(f" Warn (apply_asset_tags): No image path found for group '{groupname}' to determine supplier tag.") - - except Exception as e_supp: - print(f" Error during supplier tag processing for '{groupname}': {e_supp}") - - # 3. --- Future Tagging Logic Placeholder --- - # Example: Tag based on PBR Types present - # try: - # present_pbr_types = list(group_info.get("pbr_types", {}).keys()) - # for pbr_tag in present_pbr_types: - # # Maybe add prefix or modify tag name - # add_tag_if_new(asset_data, f"PBR_{pbr_tag}") - # except Exception as e_pbr: - # print(f" Error during PBR type tagging for '{groupname}': {e_pbr}") - - # Example: Tag based on filename Tag (if not default like 'T-MR') - # filename_tag = group_info.get("tag") # Need to store 'Tag' in group_info during scan - # if filename_tag and filename_tag not in ["T-MR", "T-SG"]: - # add_tag_if_new(asset_data, f"Tag_{filename_tag}") - - # --- End Future Tagging Logic --- - - -# --- Main Processing Function --- -def process_textures_to_groups(root_directory): - """Scans textures, creates/updates node groups based on templates and manifest.""" - start_time = time.time() - print(f"--- Starting Texture Processing ---") - print(f"Scanning directory: {root_directory}") - root_path = Path(root_directory) - if not root_path.is_dir(): - print(f"Error: Directory not found: {root_directory}") - return False # Indicate failure - - # --- Manifest Setup --- - current_blend_filepath = bpy.data.filepath - manifest_path = get_manifest_path(current_blend_filepath) - manifest_data = {} - manifest_enabled = False - if manifest_path: - manifest_data = load_manifest(manifest_path) - manifest_enabled = True - # Flag will be True if any change requires saving the manifest - manifest_needs_saving = False - # --- End Manifest Setup --- - - # --- Load Templates --- - template_parent = bpy.data.node_groups.get(PARENT_TEMPLATE_NAME) - template_child = bpy.data.node_groups.get(CHILD_TEMPLATE_NAME) - if not template_parent: - print(f"Error: Parent template '{PARENT_TEMPLATE_NAME}' not found.") - return False - if not template_child: - print(f"Error: Child template '{CHILD_TEMPLATE_NAME}' not found.") - return False - print(f"Found templates: '{PARENT_TEMPLATE_NAME}', '{CHILD_TEMPLATE_NAME}'") - # --- End Load Templates --- - - # --- Initialize Data Structures --- - # Stores {"GroupName": {"pbr_types": {...}, "scaling": "...", "sg": False, "thumb": "..."}} - texture_data = {} - file_count, processed_files = 0, 0 - groups_created, groups_updated, child_groups_created, child_groups_updated = 0, 0, 0, 0 - nodes_updated, links_created = 0, 0 - # Cache for image datablocks loaded in THIS RUN only - loaded_images_this_run = {} - # --- End Initialize Data Structures --- - - print("Scanning files...") - # --- File Scanning --- - for dirpath, _, filenames in os.walk(root_directory): - for filename in filenames: - file_path = Path(dirpath) / filename - # Check extension - if file_path.suffix.lower() not in VALID_EXTENSIONS: - continue - - file_count += 1 - filename_stem = file_path.stem - parsed = parse_texture_filename(filename_stem) - if not parsed: - continue # Skip if filename doesn't match format - - # Extract parts - groupname = parsed["Groupname"] - pbr_type = parsed["PBRType"] - resolution_label = parsed["Resolution"].lower() - scaling_str = parsed["Scaling"] - tag_str = parsed["Tag"].upper() - file_path_str = str(file_path) - - # Validate resolution label - if resolution_label not in RESOLUTION_LABELS: - print(f"Warn: Skip '{filename}' - Invalid Res '{resolution_label}'. Expected one of {RESOLUTION_LABELS}") - continue - - # Ensure base structure for group exists in texture_data - group_entry = texture_data.setdefault(groupname, { - "pbr_types": {}, "scaling": None, "sg": False, "thumb": None - }) - - # Store texture path under the specific PBR type and resolution - group_entry["pbr_types"].setdefault(pbr_type, {})[resolution_label] = file_path_str - - # Store scaling string ONCE per groupname (first encountered wins) - if group_entry["scaling"] is None: - group_entry["scaling"] = scaling_str - elif group_entry["scaling"] != scaling_str: - # Warn only once per group if inconsistency found - if not group_entry.get("scaling_warning_printed", False): - print(f" Warn: Inconsistent 'Scaling' string found for group '{groupname}'. " - f"Using first encountered: '{group_entry['scaling']}'.") - group_entry["scaling_warning_printed"] = True - - # Track SG status and thumbnail path - if tag_str == "T-SG": - group_entry["sg"] = True - # Use 1k COL-1 as the potential thumbnail source - if resolution_label == "1k" and pbr_type == "COL-1": - group_entry["thumb"] = file_path_str - - processed_files += 1 - # --- End File Scanning --- - - print(f"\nFile Scan Complete. Found {file_count} files, parsed {processed_files} valid textures.") - total_groups_found = len(texture_data) - print(f"Total unique Groupnames found: {total_groups_found}") - if not texture_data: - print("No valid textures found. Exiting.") - return True # No work needed is considered success - - print("\n--- Processing Node Groups ---") - - all_groupnames = sorted(list(texture_data.keys())) - processing_stopped_early = False - - # --- Main Processing Loop --- - for groupname in all_groupnames: - group_info = texture_data[groupname] # Get pre-scanned info - pbr_types_data = group_info.get("pbr_types", {}) - scaling_string_for_group = group_info.get("scaling") - sg_status_for_group = group_info.get("sg", False) - thumbnail_path_for_group = group_info.get("thumb") - - target_parent_name = f"PBRSET_{groupname}" - print(f"\nProcessing Group: '{target_parent_name}'") - - parent_group = bpy.data.node_groups.get(target_parent_name) - is_new_parent = False - - # --- Find or Create Parent Group --- - if parent_group is None: - # Check batch limit BEFORE creating - if groups_created >= MAX_NEW_GROUPS_PER_RUN: - print(f"\n--- Reached NEW parent group limit ({MAX_NEW_GROUPS_PER_RUN}). Stopping. ---") - processing_stopped_early = True - break # Exit the main groupname loop - - print(f" Creating new parent group: '{target_parent_name}'") - parent_group = template_parent.copy() - if not parent_group: - print(f" Error: Failed copy parent template. Skip group '{groupname}'.") - continue # Skip to next groupname - parent_group.name = target_parent_name - groups_created += 1 - is_new_parent = True - - # --- Auto-Save Trigger --- - # Trigger AFTER creating the group and incrementing counter - if AUTO_SAVE_ENABLED and groups_created > 0 and groups_created % SAVE_INTERVAL == 0: - if perform_safe_autosave(manifest_path, manifest_data): - # If auto-save succeeded, manifest is up-to-date on disk - manifest_needs_saving = False - else: - # Auto-save failed, continue but warn - print("!!! WARNING: Auto-save failed. Continuing processing... !!!") - # --- End Auto-Save Trigger --- - - else: # Update Existing Parent Group - print(f" Updating existing parent group: '{target_parent_name}'") - groups_updated += 1 - # --- End Find or Create Parent Group --- - - # --- Process Parent Group Internals --- - # This block processes both newly created and existing parent groups - try: - # --- Calculate and Store Aspect Ratio Correction (Once per group) --- - # Find the designated Value node in the parent template - aspect_node_list = find_nodes_by_label(parent_group, ASPECT_RATIO_NODE_LABEL, 'VALUE') - if aspect_node_list: - aspect_node = aspect_node_list[0] # Assume first found is correct - if scaling_string_for_group: - # Find the path to the lowest resolution image available - lowest_res_path = None; found_res = False - # Check resolution labels in configured order (e.g., "1k", "2k"...) - for res_label in RESOLUTION_LABELS: - # Check all PBR types for this resolution - for res_data in pbr_types_data.values(): - if res_label in res_data: - lowest_res_path = res_data[res_label] - found_res = True - break # Found path for this resolution label - if found_res: - break # Found lowest available resolution path - - if lowest_res_path: - # Load the image (use cache if possible) - img = None; img_load_error = False - if lowest_res_path in loaded_images_this_run: - img = loaded_images_this_run[lowest_res_path] - img_load_error = (img is None) # Check if cached result was failure - else: - # Attempt to load if not cached - try: - img_path_obj = Path(lowest_res_path) - if img_path_obj.is_file(): - img = bpy.data.images.load(lowest_res_path, check_existing=True) - else: - img_load_error = True - print(f" Error: Aspect source image not found: {lowest_res_path}") - if img is None and not img_load_error: # Check if load function returned None - img_load_error = True - print(f" Error: Failed loading aspect source image: {lowest_res_path}") - except Exception as e_load_aspect: - print(f" Error loading aspect source image: {e_load_aspect}") - img_load_error = True - # Cache the result (image object or None) - loaded_images_this_run[lowest_res_path] = img if not img_load_error else None - - if not img_load_error and img: - # Get dimensions and calculate factor - img_width, img_height = img.size[0], img.size[1] - factor = calculate_aspect_ratio_factor(img_width, img_height, scaling_string_for_group) - print(f" Calculated Aspect Ratio Factor: {factor:.4f} (from {img_width}x{img_height}, Scaling='{scaling_string_for_group}')") - - # Store factor in node if value changed significantly - if abs(aspect_node.outputs[0].default_value - factor) > 0.0001: - aspect_node.outputs[0].default_value = factor - print(f" Set '{ASPECT_RATIO_NODE_LABEL}' node value to {factor:.4f}") - else: - print(f" Warn: Could not load image '{lowest_res_path}' for aspect ratio calc.") - else: - print(f" Warn: No suitable image found (e.g., 1k) to calculate aspect ratio for '{groupname}'.") - else: - print(f" Warn: No Scaling string found for group '{groupname}'. Cannot calculate aspect ratio.") - # else: # Optional Warning if node is missing from template - # print(f" Warn: Value node '{ASPECT_RATIO_NODE_LABEL}' not found in parent group '{parent_group.name}'. Cannot store aspect ratio.") - # --- End Aspect Ratio Correction --- - - # Set SG Value - sg_nodes = find_nodes_by_label(parent_group, SG_VALUE_NODE_LABEL, 'VALUE') - if sg_nodes: - sg_node = sg_nodes[0] - target_val = 1.0 if sg_status_for_group else 0.0 - if abs(sg_node.outputs[0].default_value - target_val) > 0.001: - sg_node.outputs[0].default_value = target_val - print(f" Set '{SG_VALUE_NODE_LABEL}' to: {target_val}") - - # Set Asset Info (Thumbnail Path Prop, Initial Preview & Tagging) - # This block runs for both new and existing groups - try: - # 1. Set/Update Thumbnail Path Property & Mark Asset - if not parent_group.asset_data: - parent_group.asset_mark() - print(f" Marked '{parent_group.name}' as asset.") - # Update thumbnail property logic - if thumbnail_path_for_group: - thumb_path_obj = Path(thumbnail_path_for_group) - if thumb_path_obj.is_file(): - if parent_group.get("thumbnail_filepath") != thumbnail_path_for_group: - parent_group["thumbnail_filepath"] = thumbnail_path_for_group - if not is_new_parent: print(f" Updated thumbnail path property.") # Log update only if not new - elif "thumbnail_filepath" in parent_group: - del parent_group["thumbnail_filepath"] - if not is_new_parent: print(f" Removed thumbnail path property (file not found).") - elif "thumbnail_filepath" in parent_group: - del parent_group["thumbnail_filepath"] - if not is_new_parent: print(f" Removed old thumbnail path property.") - - # 2. Set Initial Preview (Only if NEW parent) - if is_new_parent and thumbnail_path_for_group and Path(thumbnail_path_for_group).is_file(): - print(f" Attempting initial preview from '{Path(thumbnail_path_for_group).name}'...") - try: - with bpy.context.temp_override(id=parent_group): - bpy.ops.ed.lib_id_load_custom_preview(filepath=thumbnail_path_for_group) - print(f" Set initial custom preview.") - except Exception as e_prev: - print(f" Preview Error: {e_prev}") - - # 3. Apply Asset Tags (Supplier, etc.) - apply_asset_tags(parent_group, groupname, group_info) - - except Exception as e_asset_info: - print(f" Error setting asset info/tags: {e_asset_info}") - # --- End Asset Info --- - - - # --- Process Child Groups (PBR Types) --- - for pbr_type, resolutions_data in pbr_types_data.items(): - # print(f" Processing PBR Type: {pbr_type}") # Can be verbose - - # Find placeholder node in parent - holder_nodes = find_nodes_by_label(parent_group, pbr_type, 'GROUP') - if not holder_nodes: - print(f" Warn: No placeholder node labeled '{pbr_type}' in parent group '{parent_group.name}'. Skipping PBR Type.") - continue - holder_node = holder_nodes[0] # Assume first is correct - - # Determine child group name (Base64 encoded) - logical_child_name = f"{groupname}_{pbr_type}" - target_child_name_b64 = encode_name_b64(logical_child_name) - - # Find or Create Child Group - child_group = bpy.data.node_groups.get(target_child_name_b64) - if child_group is None: - # print(f" Creating new child group for '{pbr_type}'") # Verbose - child_group = template_child.copy() - if not child_group: - print(f" Error: Failed copy child template. Skip PBR Type.") - continue - child_group.name = target_child_name_b64 - child_groups_created += 1 - else: - # print(f" Updating existing child group for '{pbr_type}'") # Verbose - child_groups_updated += 1 - - # Assign child group to placeholder if needed - if holder_node.node_tree != child_group: - holder_node.node_tree = child_group - print(f" Assigned child group '{child_group.name}' to placeholder '{holder_node.label}'.") - - # Connect placeholder output to parent output socket if needed - try: - source_socket = holder_node.outputs[0] if holder_node.outputs else None - group_output_node = next((n for n in parent_group.nodes if n.type == 'GROUP_OUTPUT'), None) - target_socket = None - if group_output_node: - target_socket = group_output_node.inputs.get(pbr_type) # Get socket by name/label - - if source_socket and target_socket: - # Check if link already exists - link_exists = any(link.from_socket == source_socket and link.to_socket == target_socket for link in parent_group.links) - if not link_exists: - parent_group.links.new(source_socket, target_socket) - links_created += 1 - print(f" Connected '{holder_node.label}' output to parent output socket '{pbr_type}'.") - # else: # Optional warning if sockets aren't found - # if not source_socket: print(f" Warn: No output socket found on placeholder '{holder_node.label}'.") - # if not target_socket: print(f" Warn: No input socket '{pbr_type}' found on parent output node.") - - except Exception as e_link: - print(f" Error linking sockets for '{pbr_type}': {e_link}") - - # Ensure parent output socket type is Color - try: - item = parent_group.interface.items_tree.get(pbr_type) - if item and item.in_out == 'OUTPUT' and item.socket_type != 'NodeSocketColor': - item.socket_type = 'NodeSocketColor' - # print(f" Set parent output socket '{pbr_type}' type to Color.") # Optional info - except Exception as e_sock: - print(f" Error updating socket type for '{pbr_type}': {e_sock}") - - - # --- Process Resolutions within Child Group --- - for resolution_label, image_path_str in resolutions_data.items(): - - # Find image texture nodes within the CHILD group - image_nodes = find_nodes_by_label(child_group, resolution_label, 'TEX_IMAGE') - if not image_nodes: - # print(f" Warn: No node labeled '{resolution_label}' found in child group for '{pbr_type}'.") # Optional - continue - - # --- >>> Manifest Check <<< --- - is_processed = False - if manifest_enabled: # Only check if manifest is enabled - # Check if this specific group/pbr/res combo is done - processed_resolutions = manifest_data.get(groupname, {}).get(pbr_type, []) - if resolution_label in processed_resolutions: - is_processed = True - # print(f" Skipping {groupname}/{pbr_type}/{resolution_label} (Manifest)") # Verbose skip log - - if is_processed: - continue # Skip to the next resolution - # --- >>> End Manifest Check <<< --- - - # --- Load Image & Assign (if not skipped) --- - # print(f" Processing Resolution: {resolution_label} for {pbr_type}") # Verbose - img = None - image_load_failed = False - - # Check intra-run cache first - if image_path_str in loaded_images_this_run: - img = loaded_images_this_run[image_path_str] - image_load_failed = (img is None) # Respect cached failure - else: - # Not cached in this run, attempt to load - try: - image_path = Path(image_path_str) - if not image_path.is_file(): - print(f" Error: Image file not found: {image_path_str}") - image_load_failed = True - else: - # Use check_existing=True to potentially reuse existing datablocks - img = bpy.data.images.load(str(image_path), check_existing=True) - if not img: - print(f" Error: Failed loading image via bpy.data.images.load: {image_path_str}") - image_load_failed = True - # else: # Success block is handled below - # pass - except RuntimeError as e_runtime_load: - print(f" Runtime Error loading image '{image_path_str}': {e_runtime_load}") - image_load_failed = True - except Exception as e_gen_load: - print(f" Unexpected error loading image '{image_path_str}': {e_gen_load}") - image_load_failed = True - # Cache result (image object or None for failure) - loaded_images_this_run[image_path_str] = img if not image_load_failed else None - - # --- Process image if loaded/cached successfully --- - if not image_load_failed and img: - try: - # Set Color Space - correct_color_space = PBR_COLOR_SPACE_MAP.get(pbr_type, DEFAULT_COLOR_SPACE) - if img.colorspace_settings.name != correct_color_space: - print(f" Setting '{Path(img.filepath).name}' color space -> {correct_color_space}") - img.colorspace_settings.name = correct_color_space - - # Histogram Stats Calculation - if resolution_label == "1k" and pbr_type in ["ROUGH", "DISP"]: - target_node_label = f"{HISTOGRAM_NODE_PREFIX}{pbr_type}" - target_nodes = find_nodes_by_label(parent_group, target_node_label, 'COMBXYZ') - if target_nodes: - target_node = target_nodes[0] - try: - socket_x = target_node.inputs.get("X") - socket_y = target_node.inputs.get("Y") - socket_z = target_node.inputs.get("Z") - if socket_x and socket_y and socket_z: - print(f" Calculating histogram stats for {pbr_type} 1K...") - stats = calculate_image_stats(img) - if stats: - min_val, max_val, median_val = stats - print(f" Stats: Min={min_val:.4f}, Max={max_val:.4f}, Median={median_val:.4f}") - # Store stats in the Combine XYZ node - socket_x.default_value = min_val - socket_y.default_value = max_val - socket_z.default_value = median_val - print(f" Stored stats in '{target_node_label}'.") - else: - print(f" Warn: Failed calc stats for '{Path(img.filepath).name}'.") - # else: print(f" Warn: Node '{target_node_label}' missing X/Y/Z sockets.") - except Exception as e_combxyz_store: - print(f" Error processing stats in '{target_node_label}': {e_combxyz_store}") - # else: print(f" Warn: No stats node '{target_node_label}' found.") - - # Assign Image to nodes in child group - nodes_updated_this_res = 0 - for image_node in image_nodes: - if image_node.image != img: - image_node.image = img - nodes_updated_this_res += 1 - nodes_updated += nodes_updated_this_res - if nodes_updated_this_res > 0: - print(f" Assigned image '{Path(img.filepath).name}' to {nodes_updated_this_res} node(s).") - - # --- >>> Update Manifest <<< --- - if manifest_enabled: - # Ensure nested structure exists - manifest_data.setdefault(groupname, {}).setdefault(pbr_type, []) - # Add resolution if not already present - if resolution_label not in manifest_data[groupname][pbr_type]: - manifest_data[groupname][pbr_type].append(resolution_label) - # Keep the list sorted for consistency in the JSON file - manifest_data[groupname][pbr_type].sort() - manifest_needs_saving = True # Mark that we need to save later - # print(f" Marked {groupname}/{pbr_type}/{resolution_label} processed in manifest.") # Verbose - # --- >>> End Update Manifest <<< --- - - except Exception as e_proc_img: - print(f" Error during post-load processing for image '{image_path_str}': {e_proc_img}") - # Continue to next resolution even if post-load fails - # --- End Process image --- - # --- End Resolution Loop --- - # --- End PBR Type Loop --- - except Exception as e_group: - print(f" !!! ERROR processing group '{groupname}': {e_group} !!!") - import traceback; traceback.print_exc() - continue # Continue to next groupname - - # --- End Main Processing Loop --- - - # --- Final Manifest Save --- - # Save if manifest is enabled AND changes were made since the last save/start. - # This happens even if the script stopped early due to MAX_NEW_GROUPS_PER_RUN. - if manifest_enabled and manifest_needs_saving: - print("\n--- Attempting Final Manifest Save (End of Run) ---") - if save_manifest(manifest_path, manifest_data): - print(" Manifest saved successfully.") - # Error message handled within save_manifest - # --- End Final Manifest Save --- - - # --- Final Summary --- - end_time = time.time(); duration = end_time - start_time - print("\n--- Script Run Finished ---") - if processing_stopped_early: - print(f"--- NOTE: Reached NEW parent group processing limit ({MAX_NEW_GROUPS_PER_RUN}). ---") - print(f"--- You may need to SAVE manually, REVERT/RELOAD file, and RUN SCRIPT AGAIN. ---") - print(f"Duration: {duration:.2f} seconds this run.") - print(f"Summary: New Parents={groups_created}, Updated Parents={groups_updated}, New Children={child_groups_created}, Updated Children={child_groups_updated}.") - print(f" Images assigned={nodes_updated} times. Links created={links_created}.") - # Add other stats if needed, e.g., number of tags added - # --- End Final Summary --- - - return True # Indicate successful completion (or reaching limit) - - -# --- How to Run --- -# 1. Ensure 'numpy' is available in Blender's Python environment. -# 2. Create Node Group "Template_PBRSET": Configure placeholders, Value nodes (SG, Aspect Ratio), Stats nodes, outputs. -# 3. Create Node Group "Template_PBRTYPE": Configure Image Texture nodes labeled by resolution. -# 4. !! SAVE YOUR BLEND FILE AT LEAST ONCE !! for manifest, auto-saving, and auto-reloading to work. -# 5. Adjust variables in the '--- USER CONFIGURATION ---' section at the top as needed. -# 6. Paste into Blender's Text Editor and run (Alt+P or Run Script button). Check Window -> Toggle System Console. -# 7. If script stops due to limit: SAVE manually, REVERT/REOPEN file, RUN SCRIPT AGAIN. Manifest prevents reprocessing. - -if __name__ == "__main__": - print(f"Script execution started at: {time.strftime('%Y-%m-%d %H:%M:%S')}") - - # Pre-run Checks using variables from CONFIG section - valid_run_setup = True - try: - tex_dir_path = Path(texture_root_directory) - # Basic check if path looks like a placeholder or doesn't exist - if texture_root_directory == r"C:\path\to\your\texture\library" or not tex_dir_path.is_dir() : - print(f"\nERROR: 'texture_root_directory' is invalid or a placeholder.") - print(f" Current value: '{texture_root_directory}'") - valid_run_setup = False - except Exception as e_path: - print(f"\nERROR checking texture_root_directory: {e_path}") - valid_run_setup = False - - # Check templates - if not bpy.data.node_groups.get(PARENT_TEMPLATE_NAME): - print(f"\nERROR: Parent template node group '{PARENT_TEMPLATE_NAME}' not found.") - valid_run_setup = False - if not bpy.data.node_groups.get(CHILD_TEMPLATE_NAME): - print(f"\nERROR: Child template node group '{CHILD_TEMPLATE_NAME}' not found.") - valid_run_setup = False - - # Check numpy (needed for stats) - try: - import numpy - except ImportError: - print("\nCRITICAL ERROR: Python library 'numpy' not found (required for image stats).") - print(" Please install numpy into Blender's Python environment.") - valid_run_setup = False - - # Execute main function if setup checks pass - script_completed_successfully = False - if valid_run_setup: - # Check if file is saved before running features that depend on it - if not bpy.data.filepath: - print("\nWARNING: Blend file not saved. Manifest, Auto-Save, and Auto-Reload features disabled.") - script_completed_successfully = process_textures_to_groups(texture_root_directory) - else: - print("\nScript aborted due to configuration errors.") - - # --- Final Save & Reload --- - # Use config variables directly as they are in module scope - if script_completed_successfully and AUTO_RELOAD_ON_FINISH: - if bpy.data.filepath: # Only if file is saved - print("\n--- Auto-saving and reloading blend file ---") - try: - bpy.ops.wm.save_mainfile() - print(" Blend file saved.") - print(" Reloading...") - # Ensure script execution stops cleanly before reload starts - bpy.ops.wm.open_mainfile(filepath=bpy.data.filepath) - # Script execution effectively stops here upon reload - except Exception as e: - print(f"!!! ERROR during final save/reload: {e} !!!") - else: - print("\nSkipping final save & reload because the blend file is not saved.") - # --- End Final Save & Reload --- - - # This print might not be reached if reload occurs - print(f"Script execution finished processing at: {time.strftime('%Y-%m-%d %H:%M:%S')}") \ No newline at end of file diff --git a/Deprecated/POC/Standalonebatcher-Main.py b/Deprecated/POC/Standalonebatcher-Main.py deleted file mode 100644 index a152645..0000000 --- a/Deprecated/POC/Standalonebatcher-Main.py +++ /dev/null @@ -1,1639 +0,0 @@ -import os -import json -import time -import shutil -import zipfile -from zipfile import ZipFile -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler -from pathlib import Path # Import pathlib -import cv2 -import re - - -# Define a class for watching directories and handling file events. -class DirectoryWatcher: - - # Constructor to initialize the class with required parameters. - def __init__( - self, - watched_directories, - trickle_down_callbacks, - json_path, - callback_json_file_update, - misc_passtrough - - ): - self.directories_to_watch = [ - Path(parent["name"]) / Path(directory) - for parent in extracted_json_settings["download_sources"] - for directory in parent["process_directories_names"] - ] - - self.watched_directories = watched_directories - self.callbacks = trickle_down_callbacks - self.json_file_path = json_path - self.callback_json = callback_json_file_update - self.misc_passtrough = misc_passtrough - self.event_handler = None - - # Method to start watching the directories and handle file events. - def watch(self): - - # Create a dictionary to store the initial state of files in the watched directories. - file_dict = {} - for parent in extracted_json_settings["download_sources"]: - parent_name = Path(parent["name"]) - for directory in parent["process_directories_names"]: - dir_path = parent_name / Path(directory) - # Store the initial state of files in the directory. - file_dict[dir_path] = set(os.listdir(dir_path)) - self.file_dict = file_dict - - # Get the modification time of the JSON file to monitor changes. - json_file_mtime = os.path.getmtime(self.json_file_path) - self.event_handler = CustomFileSystemEventHandler(self.callbacks, self.watched_directories, self.misc_passtrough) - observer = Observer() - for path in self.directories_to_watch: - # Add the event handler to each directory being watched. - observer.schedule(self.event_handler, path=path, recursive=False) - observer.start() - - try: - while True: - # Sleep for a second to avoid excessive checks. - time.sleep(1) - # Check if the JSON file has been modified since the last check. - if os.path.getmtime(self.json_file_path) > json_file_mtime: - json_file_mtime = os.path.getmtime(self.json_file_path) - # Call the JSON update callback function. - self.callback_json() - except KeyboardInterrupt: - # Stop the observer in case of a keyboard interrupt (e.g., Ctrl+C). - observer.stop() - # Wait for the observer to finish and join the main thread. - observer.join() - -# Define a custom FileSystemEventHandler to process file events. -class CustomFileSystemEventHandler(FileSystemEventHandler): - def __init__(self, callbacks, watched_directories, misc_passtrough): - super().__init__() - self.callbacks = callbacks - self.watched_directories = watched_directories - self.misc_passtrough = misc_passtrough - self.process_new_files = True - - - # Method to handle the "on_created" event when a file is created. - def on_created(self, event): - - file_path = event.src_path - - # Use split() method to extract the download_source directory of the path - download_source = file_path.split("\\", 1)[0] - - # Find the matching entry for the current download_source directory in the JSON data - matching_entries = [entry for entry in extracted_json_settings["download_sources"] if entry["name"] in download_source] - - if matching_entries: - # Extract the "queue_directory_path" from all matching entries - queue_directory_path = [] - for entry in matching_entries: - queue_directory_path.extend(entry["queue_directory_path"]) - - if matching_entries: - # Extract the "queue_directory_path" from all matching entries - process_directories_names = [] - for entry in matching_entries: - process_directories_names.extend(entry["process_directories_names"]) - - download_folder = f"{os.path.join(download_source, process_directories_names[0])}" - - print(f"File created: {file_path}") - - print(f"Watched_directories: {self.watched_directories}") - - break_loop = False - - # Trigger the corresponding callback for non-zip files based on the directory path - for list in self.watched_directories: - print(f"List: {list}") - - for index, directory in enumerate(list): - print(f"Directory: {directory}") - print(f"File path: {file_path}") - print(f"Index: {index}") - - # If new file is in layer 7 of trickle-down structure, then we set stop_processing_new_files false so we begin handling files again - if f'\\{directory}\\' in file_path and index == 7: - - print(f"Calling callback {self.callbacks[index].__name__} for file: {file_path}") - - self.callbacks[index](file_path) - - self.process_new_files = True - - print(f"'process_new_files' is set to: {self.process_new_files}") - - files_in_queue = get_files_in_directory(queue_directory_path[0]) - - print(f"Current files in queue: {files_in_queue}") - - if files_in_queue: - print(f"Getting next file from queue: {files_in_queue[0]}") - - # Sort the list of files by file size - files_in_queue.sort(key=lambda x: os.path.getsize(x)) - - file_mover(files_in_queue[0], os.path.join(download_folder, os.path.basename(files_in_queue[0]))) - - - break_loop = True - - - # If new file is in layer 4 of trickle-down structure we also pass the list of files to remove from directories - elif f'\\{directory}\\' in file_path and index == 4: - - print(f"Calling callback {self.callbacks[index].__name__} for file: {file_path}") - - self.callbacks[index](file_path) - - if self.process_new_files: - - files_in_queue = get_files_in_directory(queue_directory_path[0]) - - print(f"Current files in queue: {files_in_queue}") - - if files_in_queue: - print(f"Getting next file from queue: {files_in_queue[0]}") - - # Sort the list of files by file size - files_in_queue.sort(key=lambda x: os.path.getsize(x)) - - file_mover(files_in_queue[0], os.path.join(download_folder, os.path.basename(files_in_queue[0]))) - - break_loop = True - - - # If new file is in layer 3 of trickle-down structure we also pass the list of files to remove from directories - elif f'\\{directory}\\' in file_path and index == 3: - - print(f"Calling callback {self.callbacks[index].__name__} for file: {file_path}") - - self.callbacks[index](file_path, self.misc_passtrough[1]) - - break_loop = True - - # If new file is in layer 1 of trickle-down structure we also pass the list of archive directories - elif f'\\{directory}\\' in file_path and index == 1: - - print(f"Calling callback {self.callbacks[index].__name__} for file: {file_path}") - - self.callbacks[index](file_path, self.misc_passtrough[0]) - - break_loop = True - - - # If new file is in layer 0 of trickle-down structure and stop_processing_new_files is false, then we set stop_processing_new_files true so we do not handle any more files before stop_processing_new_files is set false again - elif f'\\{directory}\\' in file_path and index == 0 and self.process_new_files: - - if ".zip" in file_path: - self.process_new_files = False - - print(f"'process_new_files' is set to: {self.process_new_files}") - - - print(f"Calling callback {self.callbacks[index].__name__} for file: {file_path}") - - self.callbacks[index](file_path) - - - break_loop = True - - - # Otherwise we just pass the file path - elif f'\\{directory}\\' in file_path: - - print(f"'process_new_files' is set to: {self.process_new_files}") - - print(f"Calling callback {self.callbacks[index].__name__} for file: {file_path}") - if index == 0 and ".zip" in file_path: - print(f"Moving file: {file_path}\nTo queue at path: {str(os.path.join(queue_directory_path[0], os.path.basename(file_path)))}") - - file_mover(file_path, os.path.join(queue_directory_path[0], os.path.basename(file_path))) - - else: - self.callbacks[index](file_path) - - break_loop = True - - if break_loop == True: - break - if break_loop == True: - break - -def zip_files(files_to_zip, zip_filename): - with ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: - current_directory = os.getcwd() # Store the current working directory - for file_path in files_to_zip: - if os.path.exists(file_path): - base_name = os.path.basename(file_path) - os.chdir(os.path.dirname(file_path)) # Change CWD to the file's directory - zipf.write(base_name) # Write the file to the zip (without the directory tree) - os.chdir(current_directory) # Change back to the original CWD - else: - print(f"File not found: {file_path}") - - -def file_extractor(from_path, extract_to_path): - """Function for handling unzipping of new files""" - - # Making variables accessible in the scope of the function - global __settings__ - - # Add a short delay before further operations to allow the watchdog library to release its lock on the file. - time.sleep(__settings__[3]) - - # Convert extract_to_path to a string - extract_to_path_str = str(extract_to_path) - - # Creation of directory name for extracted directory - new_file_name = extract_to_path_str + from_path[from_path.rindex("\\") : -4] - - if __settings__[1]: # Debugging - print(f"Attempting extraction to: {new_file_name}") - else: - pass - - # Attempt extracting all files from .zip file - try: - if os.path.isdir(extract_to_path_str): - with ZipFile(from_path, "r") as zipped_object: - zipped_object.extractall(new_file_name) - if __settings__[1]: # Debugging - print(f"Successfully extracted new file to {new_file_name}") - else: - pass - else: - if __settings__[2]: - if __settings__[1]: # Debugging - print(f"Attempting to create directory: {extract_to_path_str}") - else: - pass - os.mkdir(extract_to_path_str) - if __settings__[1]: # Debugging - print(f"Successfully created directory: {extract_to_path_str}") - else: - pass - with ZipFile(from_path, "r") as zipped_object: - zipped_object.extractall(new_file_name) - if __settings__[1]: # Debugging - print(f"Successfully extracted new file to {new_file_name}") - else: - pass - - except Exception as e: - if __settings__[1]: # Debugging - print(f"Failed with error: {e}") - else: - pass - - - -def file_mover(from_path, to_path): - """Function for moving files from path to path""" - - # Making variables accessible in scope of function - global __settings__ - - if __settings__[1]: # Debugging - print(f"Attempting to move file: {from_path}\n to: {to_path}") - else: - pass - - # Add a short delay before further operations to allow the watchdog library to release its lock on the file. - time.sleep(__settings__[3]) - - # Attempt moving the file - try: - if os.path.exists(to_path): - if __settings__[1]: # Debugging - print(f"Duplicate found. Attempting to remove existing file or directory.") - else: - pass - - try: - if os.path.isfile(to_path): - os.remove(to_path) - else: - shutil.rmtree(to_path) - - if __settings__[1]: # Debugging - print(f"Successfully removed existing file or directory.") - else: - pass - - except Exception as e: - if __settings__[1]: # Debugging - print(f"Failed removing existing file or directory with error: {e}") - else: - pass - - # Create the destination directory if it doesn't exist - os.makedirs(os.path.dirname(to_path), exist_ok=True) - - shutil.move(from_path, to_path) - - current_time = time.time() - - os.utime(to_path, (current_time, current_time)) - - if __settings__[1]: # Debugging - print(f"Successfully moved file: {from_path}\n to: {to_path}") - else: - pass - - # In case of a failed attempt, handle error - except Exception as e: - if __settings__[1]: # Debugging - print(f"Failed moving file: {from_path}\n to: {to_path} \n with error: {e}") - - -def file_renamer(current_name, new_name): - """Function for renaming files""" - - # Making variables accessible in scope of function - global __settings__ - - if __settings__[1]: # Debugging - print(f"Attempting to rename file: {current_name}\n to: {new_name}") - else: - pass - - # Add a short delay before further operations to allow the watchdog library to release its lock on the file. - time.sleep(__settings__[3]) - - # Attempt moving the file - try: - if os.path.exists(new_name): - if __settings__[1]: # Debugging - print(f"Duplicate found. Attempting to remove existing file or directory.") - else: - pass - try: - if os.path.isfile(new_name): - os.remove(new_name) - else: - shutil.rmtree(new_name) - - if __settings__[1]: # Debugging - print(f"Successfully removed existing file or directory.") - else: - pass - - except Exception as e: - if __settings__[1]: # Debugging - print(f"Failed to remove existing file or directory with error: {e}") - else: - pass - - os.rename(current_name, new_name) - - if __settings__[1]: # Debugging - print(f"Successfully renamed file: {current_name}\n to: {new_name}") - else: - pass - - # In case of failed attempt, handle error - except Exception as e: - if __settings__[1]: # Debugging - print(f"Failed renamed file: {current_name}\n to: {new_name}\nwith error: {e}") - else: - pass - - -def get_files_in_directory(directory): - """Function for returning all filenames of a specified directory""" - - file_list = [] # List to store the file names - - # Iterate over all files in the directory - for root, dirs, files in os.walk(directory): - for file in files: - file_path = os.path.join(root, file) # Get the full path of the file - file_list.append(file_path) # Add the file name to the list - return file_list - - -def archetype_determiner(directory_path, json_object): - """Function for determining the type of data downloaded and adding the appropriate prefix to the naming scheme""" - - #### Minus keywords is keywords which does not fit in a naming scheme. - #### And keywords is keywords which by themselves, when no minus keyword is present, can determine an archetype. - #### Plus keywords is keywords which is welcome in the namingscheme. - - # Making variables accessible in the scope of the function - global __settings__ - if __settings__[1]: # Debugging - print(f"Attempting to determine archetype for file: {directory_path}") - else: - pass - - # Get the file names in the directory - files = get_files_in_directory(directory_path) - - # Use split() method to extract the download_source directory of the path - download_source = directory_path.split("\\", 1)[0] - - # Find the matching entry for the current download_source directory in the JSON data - matching_entries = [entry for entry in json_object["download_sources"] if entry["name"] in download_source] - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - scheme = [] - for entry in matching_entries: - scheme.extend(entry["scheme"]) - - # Assuming there's only one dictionary in the "scheme" list - scheme_folder = list(scheme[0].keys()) - print(scheme_folder) - - - ########################################## - ###### Finding Keywords in filename ###### - ########################################## - - # Lists for keeping the found keywords. Initiated as empty. - found_minus = [] - found_and = [] - found_plus = [] - - # Iterate through the json file subfolders of the scheme folder - for subfolder in scheme_folder: - if __settings__[1]: # Debugging - print(f"Searching for keywords of scheme: {subfolder}") - else: - pass - - # Iterate through all files in the directory - for file in files: - if __settings__[1]: # Debugging - print(f"Searching for keywords in file: {file}") - else: - pass - - # Iterate through all keywords listed in the minus subsubfolder of the current subfolder of the scheme folder - for keyword in (scheme[0][subfolder])[0]["-"]: - - # We check if the current keyword is in the filename without regarding capital case letters. - # If True, then we append it to the corresponding list initialized earlier. - if keyword.lower() in file.lower(): - found_minus.append(keyword) - - # Iterate through all keywords listed in the and subsubfolder of the current subfolder of the scheme folder - for keyword in (scheme[0][subfolder])[0]["&"]: - - # We check if the current keyword is in the filename without regarding capital case letters. - # If True, then we append it to the corresponding list initialized earlier. - if keyword.lower() in file.lower(): - found_and.append(keyword) - - # Iterate through all keywords listed in the plus subsubfolder of the current subfolder of the scheme folder - for keyword in (scheme[0][subfolder])[0]["+"]: - - # We check if the current keyword is in the filename without regarding capital case letters. - # If True, then we append it to the corresponding list initialized earlier. - if keyword.lower() in file.lower(): - found_plus.append(keyword) - - if __settings__[0]: # Debugging - print( - f"Found following keywords: \n Minus : {found_minus} \n And : {found_and} \n Plus : {found_plus}" - ) - else: - pass - - ######################################## - ###### Determining file archetype ###### - ######################################## - - # Making all found keywords lowercase, while also sorting alphabetically and remove duplicates - sorted_found_minus = sorted(set([item.lower() for item in found_minus])) - sorted_found_and = sorted(set([item.lower() for item in found_and])) - sorted_found_plus = sorted(set([item.lower() for item in found_plus])) - if __settings__[0]: # Debugging - print( - f"Sorting keywords alphabetically, removing duplicates and making keywords lowercase: \n Minus : {sorted_found_minus} \n And : {sorted_found_and} \n Plus : {sorted_found_plus}" - ) - else: - pass - try: - # Iterate through the json file subfolders of the scheme folder - for subfolder in scheme_folder: - if __settings__[0]: # Debugging - print( - f"Attempting to find file archetype by comparing found keywords to the naming scheme: {subfolder}" - ) - else: - pass - - # Making all keywords of the subfolder lowercase, while also sorting alphabetically and remove duplicates - sorted_minus = sorted( - set([item.lower() for item in (scheme[0][subfolder])[0]["-"]]) - ) - sorted_and = sorted( - set([item.lower() for item in (scheme[0][subfolder])[0]["&"]]) - ) - sorted_plus = sorted( - set([item.lower() for item in (scheme[0][subfolder])[0]["+"]]) - ) - - # Proceed if none of the minus keywords are found - if not any(keyword.lower() in sorted_found_minus for keyword in sorted_minus): - - # If any and keywords found match those of the subfolder then save the subfolder as archetype and return it to function call - if any(keyword.lower() in sorted_found_and for keyword in sorted_and): - archetype = subfolder - - if __settings__[0]: # Debugging - print(f"Found archetype: {archetype}") - else: - pass - return archetype - - # If all plus keywords found match those of the subfolder then save the subfolder as archetype and return it to functioncall - # We check equality without regarding order of appearance using set() - if set(sorted_plus) == set(sorted_found_plus): - archetype = subfolder - if __settings__[0]: # Debugging - print(f"Found archetype: {archetype}") - else: - pass - return archetype - - except Exception as e: - if __settings__[0]: # Debugging - print(f"Failed to find file archetype with error: {e}") - else: - pass - archetype = None - return archetype - - -def normalize_aspect_ratio_change(original_width, original_height, resized_width, resized_height, decimals=2): - """Function for calculating decimal change in width and height from a original image to a resized image normalised as values between 0 and 2, with 1 being no change.""" - - width_change_percentage = ((resized_width - original_width) / original_width) * 100 - height_change_percentage = ((resized_height - original_height) / original_height) * 100 - - # Normalize width and height changes to be between -100 and 100 - normalized_width_change = width_change_percentage / 100 - normalized_height_change = height_change_percentage / 100 - - # Scale the normalized values to be between 0 and 2 with 1 being unchanged - normalized_width_change = min(max(normalized_width_change + 1, 0), 2) - normalized_height_change = min(max(normalized_height_change + 1, 0), 2) - - closest_value_to_one = min(abs(normalized_width_change), abs(normalized_height_change)) - scale_factor = 1 / closest_value_to_one if closest_value_to_one else 1 - - scaled_normalized_width_change = scale_factor * normalized_width_change - scaled_normalized_height_change = scale_factor * normalized_height_change - - # Round the normalized values to the specified number of decimals - output_width = round(scaled_normalized_width_change, decimals) - output_height = round(scaled_normalized_height_change, decimals) - - if str(output_width) == "1.0": - output_width = int(1) - - if str(output_height) == "1.0": - output_height = int(1) - - if original_width == original_height or output_width == output_height: - output = "EVEN" - - return output, normalized_width_change, normalized_height_change - - elif output_width != 1 and output_height == 1: - output = f"X{str(output_width).replace('.', '')}" - - return output, normalized_width_change, normalized_height_change - - elif output_height != 1 and output_width == 1: - output = f"Y{str(output_height).replace('.', '')}" - - return output, normalized_width_change, normalized_height_change - - else: - output = f"X{str(output_width).replace('.', '')}Y{str(output_height).replace('.', '')}" - - return output, normalized_width_change, normalized_height_change - -def resize_images(path, new_name_seperator, number_of_decimals): - download_source = path.split("\\", 1)[0] - - matching_entries = [entry for entry in extracted_json_settings["download_sources"] if entry["name"] in download_source] - - if matching_entries: - lookup_table_greatest_axis = [] - for entry in matching_entries: - lookup_table_greatest_axis.extend(entry["lookup_table_greatest_axis"]) - - for filename in os.listdir(path): - file_name_parts = split_filename(os.path.basename(filename), new_name_seperator) - file_type_part = split_filename(filename, '.') - - if filename.endswith('.jpg') or filename.endswith('.png') or filename.endswith('.tif'): - image_path = os.path.join(path, filename) - img = cv2.imread(image_path) - original_height, original_width, _ = img.shape - - if original_width > int((lookup_table_greatest_axis[0])[0]) and original_height > int((lookup_table_greatest_axis[0])[0]): - img = cv2.resize(img, (int((lookup_table_greatest_axis[0])[0]), int((lookup_table_greatest_axis[0])[0]))) - - original_resized_width = 2 ** (original_width - 1).bit_length() - original_resized_height = 2 ** (original_height - 1).bit_length() - - try: - if __settings__[0]: # Debugging - print(f"Attempting to resize image to 'width x height': {original_resized_width} x {original_resized_height}") - else: - pass - - img = cv2.resize(img, (original_resized_width, original_resized_height)) - - if __settings__[0]: # Debugging - print(f"Resized image to 'width x height': {original_resized_width} x {original_resized_height}") - else: - pass - - except Exception as e: - if __settings__[0]: # Debugging - print(f"Failed resizing image to 'width x height': {original_resized_width} x {original_resized_height}\nwith error: {e}") - else: - pass - - for element in lookup_table_greatest_axis: - if original_resized_width == element[0] or original_resized_height == element[0]: - resolution = element[1] - break - - scaling_value, width_change, height_change = normalize_aspect_ratio_change(original_width, original_height, original_resized_width, original_resized_height, number_of_decimals) - - try: - if __settings__[0]: # Debugging - print(f"Attempting to save resized image with 'width x height': {original_resized_width} x {original_resized_height}\nAnd aspect ratio: {width_change}x{height_change}\nFormatted as: {scaling_value}") - else: - pass - - temp_path = os.path.join(path, "temp.jpg") # Temporary file path - cv2.imwrite(temp_path, img, [cv2.IMWRITE_JPEG_QUALITY, 98]) # Save with temporary high quality - - img_temp = cv2.imread(temp_path) - cv2.imwrite( - os.path.join(path, f"{file_name_parts[0]}{new_name_seperator}{file_name_parts[1]}{new_name_seperator}{resolution}{new_name_seperator}{scaling_value}{new_name_seperator}{file_name_parts[2]}.{file_type_part[-1]}"), img_temp - ) # Save with desired quality - - os.unlink(temp_path) # Remove temporary file - - if __settings__[0]: # Debugging - print(f"Saved resized image with 'width x height': {original_resized_width} x {original_resized_height}\nAnd aspect ratio: {width_change}x{height_change}\nFormatted as: {scaling_value}") - else: - pass - - except Exception as e: - if __settings__[0]: # Debugging - print(f"Failed to save resized image with 'width x height': {original_resized_width} x {original_resized_height}\nAnd aspect ratio: {width_change}x{height_change}\nFormatted as: {scaling_value}\nWith error: {e}") - else: - pass - - downscale_resized_width = original_resized_width - downscale_resized_height = original_resized_height - - upscale_resized_width = original_resized_width - upscale_resized_height = original_resized_height - - if original_resized_width >= (lookup_table_greatest_axis[-1])[0] or original_resized_height >= (lookup_table_greatest_axis[-1])[0]: - for element in reversed(lookup_table_greatest_axis): - if original_resized_width == element[0] or original_resized_height == element[0]: - resolution = element[1] - break - - while resolution is not (lookup_table_greatest_axis[-1])[1]: - img_copy = img.copy() - - downscale_resized_width /= 2 - downscale_resized_height /= 2 - downscaled_img = cv2.resize( - img_copy, (int(downscale_resized_width), int(downscale_resized_height))) - - for element in lookup_table_greatest_axis: - if downscale_resized_width == element[0] or downscale_resized_height == element[0]: - resolution = element[1] - break - - temp_path = os.path.join(path, "temp.jpg") # Temporary file path - cv2.imwrite(temp_path, downscaled_img, [cv2.IMWRITE_JPEG_QUALITY, 98]) # Save with temporary high quality - - downscaled_img_temp = cv2.imread(temp_path) - cv2.imwrite( - os.path.join(path, f"{file_name_parts[0]}{new_name_seperator}{file_name_parts[1]}{new_name_seperator}{resolution}{new_name_seperator}{scaling_value}{new_name_seperator}{file_name_parts[2]}.{file_type_part[-1]}"), downscaled_img_temp - ) # Save with desired quality - - os.unlink(temp_path) # Remove temporary file - - if __settings__[0]: # Debugging - print(f"Saved downscaled image with resolution '{resolution}' and dimensions 'width x height': {downscale_resized_width} x {downscale_resized_height}") - else: - pass - - else: - for element in reversed(lookup_table_greatest_axis): - if original_resized_width == element[0] or original_resized_height == element[0]: - resolution = element[1] - break - - while resolution is not (lookup_table_greatest_axis[-1])[1]: - img_copy = img.copy() - - downscale_resized_width /= 2 - downscale_resized_height /= 2 - downscaled_img = cv2.resize( - img_copy, (int(downscale_resized_width), int(downscale_resized_height))) - - for element in lookup_table_greatest_axis: - if downscale_resized_width == element[0] or downscale_resized_height == element[0]: - resolution = element[1] - break - - temp_path = os.path.join(path, "temp.jpg") # Temporary file path - cv2.imwrite(temp_path, downscaled_img, [cv2.IMWRITE_JPEG_QUALITY, 98]) # Save with temporary high quality - - downscaled_img_temp = cv2.imread(temp_path) - cv2.imwrite( - os.path.join(path, f"{file_name_parts[0]}{new_name_seperator}{file_name_parts[1]}{new_name_seperator}{resolution}{new_name_seperator}{scaling_value}{new_name_seperator}{file_name_parts[2]}.{file_type_part[-1]}"), downscaled_img_temp - ) # Save with desired quality - - os.unlink(temp_path) # Remove temporary file - - if __settings__[0]: # Debugging - print(f"Saved downscaled image with resolution '{resolution}' and dimensions 'width x height': {downscale_resized_width} x {downscale_resized_height}") - else: - pass - - time.sleep(1) - - os.unlink(image_path) - - -def split_filename(filename, separator): - """Function used to get a list of all the parts that make up the file name""" - - parts = filename.split(separator) - - return parts - - -def move_to_trickle_down_layer(file_path, from_layer, to_layer, directory_list_to_search_in): - """Function used when moving files to the next layer of the trickle-down structure""" - - # Making variables accessible in scope of function - global __settings__ - - try: - - if __settings__[1]: # Debugging - print(f"Attempting to move file from trickle-down layer {str(from_layer)} to trickle-down layer {str(to_layer)}: {file_path}") - else: - pass - - # Use split() method to extract the parent directory of the path - parent = file_path.split("\\", 1)[0] - - if __settings__[1]: # Debugging - print(f"Detected that file is within parent directory: {parent}") - else: - pass - - - layer_string = f'{parent}\\{str(to_layer)}_' - - - if __settings__[1]: # Debugging - print(f"Searching for string '{layer_string}' in following directories: {directory_list_to_search_in}") - else: - pass - - # Variable for controlling breaking out of loop early - break_loop = False - - for folder, folders_list in zip(download_sources, directory_list_to_search_in): - for directory in folders_list: - full_path = os.path.join(folder["name"], directory) - - - if __settings__[1]: # Debugging - print(f"Checking if {layer_string} is in: {full_path}") - else: - pass - - if layer_string in full_path: - target_path = f"{full_path}\\{os.path.basename(file_path)}" - - file_mover(file_path, target_path) - break_loop = True - - if break_loop == True: - break - if break_loop == True: - break - - except Exception as e: - if __settings__[1]: # Debugging - print(f"Failed to move file from trickle-down layer {str(from_layer)} to trickle-down layer {str(to_layer)}: {file_path}\nWith error: {e}") - else: - pass - - -def extract_to_trickle_down_layer(file_path, from_layer, to_layer, directory_list_to_search_in): - """Function used when extracting files to the next layer of the trickle-down structure""" - - # Making variables accessible in scope of function - global __settings__ - - if '.zip' in file_path: - if __settings__[1]: # Debugging - print(f"Attempting to extract file from trickle-down layer {str(from_layer)} to trickle-down layer {str(to_layer)}: {file_path}") - else: - pass - - # Use split() method to extract the parent directory of the path - parent = file_path.split("\\", 1)[0] - - if __settings__[1]: # Debugging - print(f"Detected that file is within parent directory: {parent}") - else: - pass - - break_loop = False - - for folder, folders_list in zip(download_sources, directory_list_to_search_in): - for directory in folders_list: - full_path = os.path.join(folder["name"], directory) - - layer_string = f'{parent}\\{str(to_layer)}_' - - if __settings__[1]: # Debugging - print(f"Checking if {layer_string} is in: {full_path}") - else: - pass - - if layer_string in full_path: - target_directory = full_path - - file_extractor(file_path, target_directory) - break_loop = True - - if break_loop == True: - break - if break_loop == True: - break - else: - if __settings__[1]: # Debugging - print(f"Error extracting file from trickle-down layer {str(from_layer)} to trickle-down layer {str(to_layer)}. File is not a .zip file: {file_path}") - else: - pass - - -def file_added_callback(file_path): - """ - Function called by the watcher class whenever a new file is added to a watched directory. - This function moves all .zip files to the first layer of the trickle-down structure. - """ - - # Making variables accessible in scope of function - global __settings__ - global process_directory_list - global download_sources - - file_path = str(file_path) # Convert file_path to a string - - if __settings__[1]: # Debugging - print("New file detected in trickle-down layer 0:", file_path) - else: - pass - - # If the downloaded file has '.zip' in the name, then we move it to the first layer of the trickle-down structure. Else we pass. - if ".zip" in file_path: - - move_to_trickle_down_layer(file_path, from_layer=0, to_layer=1, directory_list_to_search_in=process_directory_list) - - else: - pass - - -def callback_trickleDownLayer_1(file_path, archive_directories): - """Function called when a new file is detected in the first layer of the trickle-down processing structure.""" - - # Making variables accessible in scope of function - global __settings__ - global process_directory_list - - if __settings__[1]: # Debugging - print(f"File arrived in trickle-down layer 1: {file_path}") - else: - pass - - # Use split() method to extract the download_source directory of the path - download_source = file_path.split("\\", 1)[0] - - # Find the matching entry for the current download_source directory in the JSON data - matching_entries = [entry for entry in extracted_json_settings["download_sources"] if entry["name"] in download_source] - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - zip_archive_directory_path = [] - for entry in matching_entries: - zip_archive_directory_path.extend(entry["zip_archive_directory_path"]) - - break_loop = False - - # Iterate trough the lists of directories we have specified to watch - for list in process_directory_list: - - # Iterate trough the directories in the list - for directory in list: - - # We check which watched directory the file is in - if directory in file_path: - - # We unzip the file to second layer of the trickle-down struckture - extract_to_trickle_down_layer(file_path, from_layer=1, to_layer=2, directory_list_to_search_in=process_directory_list) - - # We move the sourcefile to the archive directory - file_mover(file_path , os.path.join(zip_archive_directory_path[0], os.path.basename(file_path))) - - break_loop = True - - if break_loop: - break - if break_loop: - break - - -def callback_trickleDownLayer_2(file_path): - """Function called when a new file is detected in the second layer of the trickle-down processing structure""" - - if __settings__[1]: # Debugging - print(f"File arrived in trickle-down layer 2: {file_path}") - else: - pass - - files_in_directory = get_files_in_directory(file_path) - - if __settings__[1]: # Debugging - print(f"Checking if more than one set of files are present in file: {file_path}") - else: - pass - - # Use split() method to extract the download_source directory of the path - download_source = file_path.split("\\", 1)[0] - - # Find the matching entry for the current download_source directory in the JSON data - matching_entries = [entry for entry in extracted_json_settings["download_sources"] if entry["name"] in download_source] - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - filetype_indicating_a_set_of_files_list = [] - for entry in matching_entries: - filetype_indicating_a_set_of_files_list.extend(entry["filetype_indicating_a_set_of_files"]) - - # Variable holding any files with filetypes that are both listed in the json file and found in the directory - files_found = [] - for file in files_in_directory: - for filetype in filetype_indicating_a_set_of_files_list: - if filetype in file: - files_found.append(file) - - if len(files_found) > 1: - if __settings__[1]: # Debugging - print(f"There are {len(files_found)} sets of files in file: {file_path}") - else: - pass - - basenames = [] - for file in files_found: - basename = (os.path.basename(file))[:-int(len(filetype_indicating_a_set_of_files_list[0]))] - basenames.append(basename) - - if __settings__[1]: # Debugging - print(f"Found sets {basenames} in file: {file_path}") - else: - pass - - new_directories = [] - sets = [] - for name in basenames: - - current_file_directory_path = os.path.dirname(file_path) - new_file_directory_path = os.path.join(file_path, name) - new_directories.append(new_file_directory_path) - - files_in_set = [] - for file in files_in_directory: - if name in os.path.basename(file): - files_in_set.append(file) - - sets.append(files_in_set) - - for directory in new_directories: - os.mkdir(directory) - - for i in range(len(sets)): - for file in sets[i]: - shutil.move(file, os.path.join(os.path.join(os.path.dirname(file), basenames[i]), os.path.basename(file))) - - for directory in new_directories: - - # We move the file to second layer of the trickle-down struckture - move_to_trickle_down_layer(directory, from_layer=2, to_layer=3, directory_list_to_search_in=process_directory_list) - - shutil.rmtree(file_path) - - else: - move_to_trickle_down_layer(file_path, from_layer=2, to_layer=3, directory_list_to_search_in=process_directory_list) - - -def callback_trickleDownLayer_3(file_path, file_to_remove_list_of_lists): - """Function called when a new file is detected in the third layer of the trickle-down processing structure""" - - if __settings__[1]: # Debugging - print(f"File arrived in trickle-down layer 3: {file_path}") - else: - pass - - files_in_directory = get_files_in_directory(file_path) - - # Use split() method to extract the download_source directory of the path - download_source = file_path.split("\\", 1)[0] - - # Find the matching entry for the current download_source directory in the JSON data - matching_entries = [entry for entry in extracted_json_settings["download_sources"] if entry["name"] in download_source] - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - files_to_remove_from_directories_list = [] - for entry in matching_entries: - files_to_remove_from_directories_list.extend(entry["files_to_remove_from_directories"]) - - # Extract the "minimum_image_size" from all matching entries - minimum_image_size = [] - for entry in matching_entries: - minimum_image_size.extend(entry["minimum_image_size"]) - - try: - if __settings__[1]: # Debugging - print(f"Attempting to remove files: {file_to_remove_list_of_lists}\n from file: {file_path}") - else: - pass - - # Iterate through the files in the directory - for file in files_in_directory: - for file_to_remove in files_to_remove_from_directories_list: - if file_to_remove.lower() in file.lower(): - - if __settings__[1]: # Debugging - print(f"Removing file: {file}") - else: - pass - - # Remove the file - os.remove(file) - - except Exception as e: - if __settings__[1]: # Debugging - print(f"Failed removing files from {file_path} with error {e}") - else: - pass - - files_in_directory = get_files_in_directory(file_path) - - try: - if __settings__[1]: # Debugging - print(f"Attempting to remove files with dimensions less than: {minimum_image_size[0]}x{minimum_image_size[1]}\nFrom file: {file_path}") - else: - pass - - # Iterate through the files in the directory - for file in files_in_directory: - img = cv2.imread(file, 1) - - if img is not None: - height, width, channels = img.shape - - if height < minimum_image_size[0] and width < minimum_image_size[1]: - if __settings__[1]: # Debugging - print(f"Removing file: {file}") - else: - pass - - # Remove the file - os.remove(file) - - except Exception as e: - if __settings__[1]: # Debugging - print(f"Failed removing files from {file_path} with error {e}") - else: - pass - - else: - if __settings__[1]: # Debugging - print(f"ERROR: No matching download_source directory found in JSON data for '{file_path}'.") - else: - pass - - move_to_trickle_down_layer(file_path, from_layer=3, to_layer=4, directory_list_to_search_in=process_directory_list) - - -def callback_trickleDownLayer_4(file_path): - """Function called when a new file is detected in the fourth layer of the trickle-down processing structure""" - - global watcher - - if __settings__[1]: # Debugging - print(f"File arrived in trickle-down layer 4: {file_path}") - else: - pass - - # We determine what prefix the downloaded file shall get - file_archetype = archetype_determiner(file_path, extracted_json_settings) - - # Use split() method to extract the download_source directory of the path - download_source = file_path.split("\\", 1)[0] - - # Find the matching entry for the current download_source directory in the JSON data - matching_entries = [entry for entry in extracted_json_settings["download_sources"] if entry["name"] in download_source] - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - new_name_seperator = [] - for entry in matching_entries: - new_name_seperator.extend(entry["new_name_seperator"]) - - if matching_entries: - # Extract the "filetype_indicating_a_set_of_files" from all matching entries - filetype_indicating_a_set_of_files = [] - for entry in matching_entries: - filetype_indicating_a_set_of_files.extend(entry["filetype_indicating_a_set_of_files"]) - - if matching_entries: - # Extract the "model_output_path" from all matching entries - model_output_path = [] - for entry in matching_entries: - model_output_path.extend(entry["model_output_path"]) - - new_dir_name = os.path.join(os.path.dirname(file_path), f"{file_archetype}{new_name_seperator[0]}{os.path.basename(file_path)}") - - time.sleep(1) - - try: - if __settings__[1]: # Debugging - print(f"Attempting renaiming file: {file_path}") - print(f" to: {new_dir_name}") - else: pass - file_renamer(file_path, new_dir_name) - except Exception as e: - if __settings__[1]: # Debugging - print(f"Failed renaming file: {file_path}") - print(f" to: {new_dir_name}") - print(f" with error: {e}") - else: pass - - files = get_files_in_directory(new_dir_name) - - for file in files: - for file_type in filetype_indicating_a_set_of_files: - if file_type in file: - new_fbx_file_name = f"{file_archetype}{new_name_seperator[0]}{os.path.basename(file)}" - file_renamer(file, os.path.join(new_dir_name, new_fbx_file_name)) - file_mover(os.path.join(new_dir_name, new_fbx_file_name), os.path.join(model_output_path[0], new_fbx_file_name)) - - - if "None" in os.path.basename(new_dir_name): - if __settings__[1]: # Debugging - print(f"Failed finding file archetype. Archetype set to 'None'. Abandoning files.") - else: - pass - - - watcher.event_handler.process_new_files = True - if __settings__[1]: # Debugging - print(f"'process_new_files' is set to: {watcher.event_handler.process_new_files}") - else: - pass - - - else: - move_to_trickle_down_layer(new_dir_name, from_layer=4, to_layer=5, directory_list_to_search_in=process_directory_list) - - -def callback_trickleDownLayer_5(file_path): - """Function called when a new file is detected in the sixth layer of the trickle-down processing structure""" - - if __settings__[1]: # Debugging - print(f"File arrived in trickle-down layer 5: {file_path}") - else: - pass - - # Use split() method to extract the download_source directory of the path - download_source = file_path.split("\\", 1)[0] - - # Find the matching entry for the current download_source directory in the JSON data - matching_entries = [entry for entry in extracted_json_settings["download_sources"] if entry["name"] in download_source] - - if matching_entries: - # Extract the "scheme" from all matching entries - scheme = [] - for entry in matching_entries: - scheme.extend(entry["scheme"]) - - if matching_entries: - # Extract the "new_name_seperator" from all matching entries - new_name_seperator = [] - for entry in matching_entries: - new_name_seperator.extend(entry["new_name_seperator"]) - - if matching_entries: - # Extract the "original_name_seperator" from all matching entries - original_name_seperator = [] - for entry in matching_entries: - original_name_seperator.extend(entry["original_name_seperator"]) - - if matching_entries: - # Extract the "original_name_comes_before_seperator_number" from all matching entries - original_name_index = [] - for entry in matching_entries: - original_name_index.extend(entry["original_name_comes_before_seperator_number"]) - - if matching_entries: - # Extract the "maptype_comes_before_seperator_number" from all matching entries - maptype_index = [] - for entry in matching_entries: - maptype_index.extend(entry["maptype_comes_before_seperator_number"]) - - if matching_entries: - # Extract the "maptype_comes_before_seperator_number" from all matching entries - maptype_keywords_ending_with_standardized_maptype_list_of_lists = [] - for entry in matching_entries: - maptype_keywords_ending_with_standardized_maptype_list_of_lists.extend(entry["maptype_keywords_ending_with_standardized_maptype"]) - - # Assuming there's only one dictionary in the "scheme" list - scheme_folder = list(scheme[0].keys()) - - files = get_files_in_directory(file_path) - - file_archetype = "None" - - # Get archetype from directory name and apply it to the name of all files within the directory - for archetype in scheme_folder: - if f"{archetype}{new_name_seperator[0]}" in file_path: - file_archetype = archetype - - # Get all parts of file name and rename the file with the correct parts in the correct order - for file in files: - file_name_parts = split_filename(os.path.basename(file), original_name_seperator[0]) - file_type_part = split_filename(file, '.') - - if "col_var" in file.lower(): - match = re.search(r'col_var(\d+)', file.lower()) - - if match: - variant_value = int(match.group(1)) - - new_file_name = os.path.join(os.path.dirname(file), f"{file_archetype}{new_name_seperator[0]}{file_name_parts[original_name_index[0]]}{new_name_seperator[0]}COL-{variant_value}{new_name_seperator[0]}.{file_type_part[-1]}") - - else: - pass - - - elif "COL" in file or "col" in file: - new_file_name = os.path.join(os.path.dirname(file), f"{file_archetype}{new_name_seperator[0]}{file_name_parts[original_name_index[0]]}{new_name_seperator[0]}COL-1{new_name_seperator[0]}.{file_type_part[-1]}") - - else: - maptype = file_name_parts[maptype_index[0]] - - for a_list in maptype_keywords_ending_with_standardized_maptype_list_of_lists: - if maptype.lower() in list(map(str.lower, a_list)): - standardised_maptype = a_list[-1] - break - - else: - standardised_maptype = f"{maptype}-UNRECOGNIZED" - - new_file_name = os.path.join(os.path.dirname(file), f"{file_archetype}{new_name_seperator[0]}{file_name_parts[original_name_index[0]]}{new_name_seperator[0]}{standardised_maptype}{new_name_seperator[0]}.{file_type_part[-1]}") - - file_renamer(file, new_file_name) - - move_to_trickle_down_layer(file_path, from_layer=5, to_layer=6, directory_list_to_search_in=process_directory_list) - - - -def callback_trickleDownLayer_6(file_path): - """Function called when a new file is detected in the fifth layer of the trickle-down processing structure""" - - if __settings__[1]: # Debugging - print(f"File arrived in trickle-down layer 6: {file_path}") - else: - pass - - - # Use split() method to extract the download_source directory of the path - download_source = file_path.split("\\", 1)[0] - - # Find the matching entry for the current download_source directory in the JSON data - matching_entries = [entry for entry in extracted_json_settings["download_sources"] if entry["name"] in download_source] - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - new_name_seperator = [] - for entry in matching_entries: - new_name_seperator.extend(entry["new_name_seperator"]) - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - aspect_ratio_number_of_decimals = [] - for entry in matching_entries: - aspect_ratio_number_of_decimals.extend(entry["aspect_ratio_number_of_decimals"]) - - resize_images(file_path, new_name_seperator[0], int(aspect_ratio_number_of_decimals[0])) - - move_to_trickle_down_layer(file_path, from_layer=6, to_layer=7, directory_list_to_search_in=process_directory_list) - - - -def callback_trickleDownLayer_7(file_path): - """Function called when a new file is detected in the seventh layer of the trickle-down processing structure""" - - if __settings__[1]: # Debugging - print(f"File arrived in trickle-down layer 7: {file_path}") - else: - pass - - # Use split() method to extract the download_source directory of the path - download_source = file_path.split("\\", 1)[0] - - # Find the matching entry for the current download_source directory in the JSON data - matching_entries = [entry for entry in extracted_json_settings["download_sources"] if entry["name"] in download_source] - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - output_paths = [] - for entry in matching_entries: - output_paths.extend(entry["output_paths"]) - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - scheme = [] - for entry in matching_entries: - scheme.extend(entry["scheme"]) - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - new_name_seperator = [] - for entry in matching_entries: - new_name_seperator.extend(entry["new_name_seperator"]) - - if matching_entries: - # Extract the "files_to_remove_from_directories" from all matching entries - unrecognized_maptype = [] - for entry in matching_entries: - unrecognized_maptype.extend(entry["unrecognized_maptype_output_path"]) - - scheme_folder = list(scheme[0].keys()) - - files = get_files_in_directory(file_path) - - for file in files: - if "UNRECOGNIZED" in file: - file_mover(file, os.path.join(unrecognized_maptype[0], os.path.basename(file))) - - files = get_files_in_directory(file_path) - - for file in files: - for index, archetype in enumerate(scheme_folder): - if f"{archetype}{new_name_seperator[0]}" in file: - file_mover(file, os.path.join(output_paths[index], os.path.basename(file))) - if not os.listdir(file_path): - os.rmdir(file_path) - - -def load_settings_json(): - """Function for loading the settings.json file""" - - # Making variables accessible in scope of function - global extracted_json_settings - global __settings__ - - if __settings__[1]: # Debugging - print(f"Attempting to load {__settings__[0]}.") - else: - pass - - # We attempt to load settings.json. If the file is using wrong syntax, then this try except statement will notify the user without terminating the program - try: - # Open settings.json file in read mode and call it 'json_file' - with open(__settings__[0], "r") as json_file: - - # Read all lines disregarding any whitespace (spaces and newlines) - lines = [line.rstrip() for line in json_file] - - # Join the lines into a single string - json_string = "".join(lines) - - # Parse the JSON string into a Python object - extracted_json_settings = json.loads(json_string) - - except Exception as e: - if __settings__[1]: # Debugging - print(f"Failed to load {__settings__[0]} with error: {e}") - print("Retrying in 30 seconds") - time.sleep(30) - load_settings_json() - - -def create_directories_from_json_file(directory_list): - """Function for handling creation of directories listed in .json file""" - - # Making variables accessible in scope of function - global __settings__ - - # If the system shall create missing directories, then we check if any of the directories from settings.json does not exist, and if so we create it - if __settings__[2]: - for folder, folders_list in zip(download_sources, directory_list): - for directory in folders_list: - if ".\\" in directory: - full_path = directory - else: - full_path = os.path.join(folder["name"], directory) - if not os.path.isdir(full_path): - try: - if __settings__[1]: # Debugging - print(f"Attempting to create directory: {full_path}") - else: - pass - os.makedirs(full_path) - if __settings__[1]: # Debugging - print(f"Successfully created directory: {full_path}") - else: - pass - except Exception as e: - if __settings__[1]: # Debugging - print( - f"Failed to create directory {full_path} with error: {e}" - ) - else: - pass - - -def init(): - """Function for initializing the watcher. This is also the callback function for when settings.json is updated, so that the watcher is always up to date on its settings.""" - - global watcher - - # Making variables accessible in the scope of the function - global directory_list - global extracted_json_settings - global process_directory_list - global download_sources - - # Load the settings.json file - load_settings_json() - - # Extract all download folders from the .json file - download_sources = extracted_json_settings["download_sources"] - - # Extract the "process_directories_names" for each item in "download_sources" - process_directory_list = [ - folder["process_directories_names"] - for folder in extracted_json_settings["download_sources"] - ] - - # Extract the "zip_archive_directory_path" for each item in "download_sources" - archive_directory_list = [ - folder["zip_archive_directory_path"] - for folder in extracted_json_settings["download_sources"] - ] - - # Extract the "files_to_remove_from_directories" for each item in "download_sources" - files_to_remove_from_directories_list = [ - folder["files_to_remove_from_directories"] - for folder in extracted_json_settings["download_sources"] - ] - - - # Extract the "files_to_remove_from_directories" for each item in "download_sources" - unrecognized_maptype_output_path = [ - folder["unrecognized_maptype_output_path"] - for folder in extracted_json_settings["download_sources"] - ] - - - # Extract the "files_to_remove_from_directories" for each item in "download_sources" - fbx_output_path = [ - folder["model_output_path"] - for folder in extracted_json_settings["download_sources"] - ] - - - # Extract the "files_to_remove_from_directories" for each item in "download_sources" - output_paths = [ - folder["output_paths"] - for folder in extracted_json_settings["download_sources"] - ] - - # Extract the "files_to_remove_from_directories" for each item in "download_sources" - queue_directory_path = [ - folder["queue_directory_path"] - for folder in extracted_json_settings["download_sources"] - ] - - # Create directories listed in "process_directories_names" tags in the json file - create_directories_from_json_file(process_directory_list) - - # Create directories listed in "zip_archive_directory_path" tags in the json file - create_directories_from_json_file(archive_directory_list) - - # Create directories listed in "unrecognized_maptype_output_path" tags in the json file - create_directories_from_json_file(unrecognized_maptype_output_path) - - # Create directories listed in "fbx_output_path" tags in the json file - create_directories_from_json_file(fbx_output_path) - - # Create directories listed in "output_paths" tags in the json file - create_directories_from_json_file(output_paths) - - # Create directories listed in "queue_directory_path" tags in the json file - create_directories_from_json_file(queue_directory_path) - - process_directory_list_callback_functions = [ - file_added_callback, - callback_trickleDownLayer_1, - callback_trickleDownLayer_2, - callback_trickleDownLayer_3, - callback_trickleDownLayer_4, - callback_trickleDownLayer_5, - callback_trickleDownLayer_6, - callback_trickleDownLayer_7 - ] - - misc_passtrough = [archive_directory_list, files_to_remove_from_directories_list] - - # Create an instance of DirectoryWatcher - watcher = DirectoryWatcher( - process_directory_list, - process_directory_list_callback_functions, - __settings__[0], - init, - misc_passtrough - ) - - if __settings__[1]: # Debugging - print(f"Watching for new files in directories: {process_directory_list}") - else: - pass - - # Start watching the directories - watcher.watch() - - -if __name__ == "__main__": - #################################### - ########## SETTINGS ########## - #################################### - - ## Do touch ## - json_file_name = ( - "settings.json" # json file name contaning settings of the whatched directories - ) - debug_prints = True # Bool for activating debug print statements - create_missing_folders = True # Bool for determining whether or not the system shall create missing folders. - delay = 0.5 # Seconds - - - ## Do not touch ## - __settings__ = [json_file_name, debug_prints, create_missing_folders, delay] - - #################################### - ########## Main ########## - #################################### - - # Init variables - download_sources = [] - process_directory_list = [] - archive_directory_list = [] - extracted_json_settings = [] - watcher = None - - # Run initialization - init() \ No newline at end of file diff --git a/Deprecated/detailed_documentation_plan.md b/Deprecated/detailed_documentation_plan.md deleted file mode 100644 index 81765ed..0000000 --- a/Deprecated/detailed_documentation_plan.md +++ /dev/null @@ -1,90 +0,0 @@ -# Final Plan for Updating documentation.txt with Debugging Details - -This document outlines the final plan for enhancing `documentation.txt` with detailed internal information crucial for debugging the Asset Processor Tool. This plan incorporates analysis of `asset_processor.py`, `configuration.py`, `main.py`, and `gui/processing_handler.py`. - -## Task Objective - -Analyze relevant source code (`asset_processor.py`, `configuration.py`, `main.py`, `gui/processing_handler.py`) to gather specific details about internal logic, state management, error handling, data structures, concurrency, resource management, and limitations. Integrate this information into the existing `documentation.txt` to aid developers in debugging. - -## Analysis Steps Completed - -1. Read and analyzed `readme.md`. -2. Listed code definitions in the root directory (`.`). -3. Listed code definitions in the `gui/` directory. -4. Read and analyzed `asset_processor.py`. -5. Read and analyzed `configuration.py`. -6. Read and analyzed `main.py`. -7. Read and analyzed `gui/processing_handler.py`. - -## Final Integration Plan (Merged Structure) - -1. **Add New Top-Level Section:** - * Append the following section header to the end of `documentation.txt`: - ``` - ================================ - Internal Details for Debugging - ================================ - ``` - -2. **Create Subsections:** - * Under the new "Internal Details" section, create the following subsections (renumbering from 7 onwards): - * `7. Internal Logic & Algorithms` - * `8. State Management` - * `9. Error Handling & Propagation` - * `10. Key Data Structures` - * `11. Concurrency Models (CLI & GUI)` - * `12. Resource Management` - * `13. Known Limitations & Edge Cases` - -3. **Populate Subsections with Specific Details:** - - * **7. Internal Logic & Algorithms:** - * **Configuration Preparation (`Configuration` class):** Detail the `__init__` process: loading `config.py` and preset JSON, validating structure (`_validate_configs`), compiling regex (`_compile_regex_patterns` using `_fnmatch_to_regex`). Mention compiled patterns storage. - * **CLI Argument Parsing (`main.py:setup_arg_parser`):** Briefly describe `argparse` usage and key flags influencing execution. - * **Output Directory Resolution (`main.py:main`):** Explain how the final output path is determined and resolved. - * **Asset Processing (`AssetProcessor` class):** - * *Classification (`_inventory_and_classify_files`):* Describe multi-pass approach using compiled regex from `Configuration`. Detail variant sorting criteria. - * *Map Processing (`_process_maps`):* Detail image loading, Gloss->Rough inversion, resizing, format determination (using `Configuration` rules), bit depth conversion, stats calculation, aspect ratio logic, save fallback. - * *Merging (`_merge_maps`):* Detail common resolution finding, input loading, channel merging (using `Configuration` rules), output determination, save fallback. - * *Metadata (`_determine_base_metadata`, etc.):* Summarize base name extraction, category/archetype determination (using `Configuration` rules), `metadata.json` population. - * **Blender Integration (`main.py:run_blender_script`, `gui/processing_handler.py:_run_blender_script_subprocess`):** Explain subprocess execution, command construction (`-b`, `--python`, `--`), argument passing (`asset_root_dir`). - - * **8. State Management:** - * `Configuration` object: Holds loaded config state and compiled regex. Instantiated per worker. - * `AssetProcessor`: Primarily stateless between `process()` calls. Internal state within `process()` managed by local variables (e.g., `current_asset_metadata`). `self.classified_files` populated once and filtered per asset. - * `main.py`: Tracks overall CLI run counts (processed, skipped, failed). - * `gui/processing_handler.py`: Manages GUI processing run state via flags (`_is_running`, `_cancel_requested`) and stores `Future` objects (`_futures`). - - * **9. Error Handling & Propagation:** - * Custom Exceptions: `AssetProcessingError`, `ConfigurationError`. - * `Configuration`: Raises `ConfigurationError` on load/validation failure. Logs regex compilation warnings. - * `AssetProcessor`: Catches exceptions within per-asset loop, logs error, marks asset "failed", continues loop. Handles specific save fallbacks (EXR->PNG). Raises `AssetProcessingError` for critical setup failures. - * Worker Wrapper (`main.py:process_single_asset_wrapper`): Catches exceptions from `Configuration`/`AssetProcessor`, logs, returns "failed" status tuple. - * Process Pool (`main.py`, `gui/processing_handler.py`): `try...except` around executor block catches pool-level errors. - * GUI Communication (`ProcessingHandler`): Catches errors during future result retrieval, emits failure status via signals. - * Blender Scripts: Checks subprocess return code, logs stderr. Catches `FileNotFoundError`. - - * **10. Key Data Structures:** - * `Configuration` attributes: `compiled_map_keyword_regex`, `compiled_extra_regex`, etc. (compiled `re.Pattern` objects). - * `AssetProcessor` structures: `self.classified_files` (dict[str, list[dict]]), `processed_maps_details_asset` (dict[str, dict[str, dict]]), `file_to_base_name_map` (dict[Path, Optional[str]]). - * Return values: Status dictionary from `AssetProcessor.process()`, status tuple from `process_single_asset_wrapper`. - * `ProcessingHandler._futures`: dict[Future, str]. - - * **11. Concurrency Models (CLI & GUI):** - * **Common Core:** Both use `concurrent.futures.ProcessPoolExecutor` running `main.process_single_asset_wrapper`. `Configuration` and `AssetProcessor` instantiated within each worker process for isolation. - * **CLI Orchestration (`main.py:run_processing`):** Direct executor usage, `as_completed` gathers results synchronously. - * **GUI Orchestration (`gui/processing_handler.py`):** `ProcessingHandler` (QObject) runs executor logic in a `QThread`. Results processed in handler thread, communicated asynchronously to UI thread via Qt signals (`progress_updated`, `file_status_updated`, `processing_finished`). - * **Cancellation (`gui/processing_handler.py:request_cancel`):** Sets flag, attempts `executor.shutdown(wait=False)`, tries cancelling pending futures. Limitation: Does not stop already running workers. - - * **12. Resource Management:** - * `Configuration`: Uses `with open` for preset files. - * `AssetProcessor`: Manages temporary workspace (`tempfile.mkdtemp`, `shutil.rmtree` in `finally`). Uses `with open` for metadata JSON. - * `ProcessPoolExecutor`: Lifecycle managed via `with` statement in `main.py` and `gui/processing_handler.py`. - - * **13. Known Limitations & Edge Cases:** - * Configuration: Basic structural validation only; regex compilation errors are warnings; `_fnmatch_to_regex` helper is basic. - * AssetProcessor: Relies heavily on filename patterns; high memory potential for large images; limited intra-asset error recovery; simplified prediction logic. - * CLI: Basic preset file existence check only before starting workers; Blender executable finding logic order (config > PATH). - * GUI Concurrency: Cancellation doesn't stop currently executing worker processes. - -4. **Switch Mode:** Request switching to Code mode to apply these changes by appending to `documentation.txt`. \ No newline at end of file diff --git a/Deprecated/documentation.txt b/Deprecated/documentation.txt deleted file mode 100644 index 2f37c14..0000000 --- a/Deprecated/documentation.txt +++ /dev/null @@ -1,269 +0,0 @@ -================================ -Asset Processor Tool - Developer Documentation -================================ - -This document provides a concise overview of the Asset Processor Tool's codebase for developers joining the project. It focuses on the architecture, key components, and development workflow. - -**NOTE:** This documentation strictly excludes details on environment setup, dependency installation, building the project, or deployment procedures. It assumes familiarity with Python and the relevant libraries (OpenCV, NumPy, PySide6). - --------------------------------- -1. Project Overview --------------------------------- - -* **Purpose:** To process 3D asset source files (texture sets, models, etc., typically from ZIP archives or folders) into a standardized library format. -* **Core Functionality:** Uses configurable JSON presets to interpret different asset sources, automating tasks like file classification, image resizing, channel merging, and metadata generation. -* **High-Level Architecture:** Consists of a core processing engine (`AssetProcessor`), a configuration system handling presets (`Configuration`), multiple interfaces (GUI, CLI, Directory Monitor), and optional integration with Blender for automated material/nodegroup creation. - --------------------------------- -2. Codebase Structure --------------------------------- - -Key files and directories: - -* `asset_processor.py`: Contains the `AssetProcessor` class, the core logic for processing a single asset through the pipeline. Includes methods for classification, map processing, merging, metadata generation, and output organization. Also provides methods for predicting output structure used by the GUI. -* `configuration.py`: Defines the `Configuration` class. Responsible for loading core settings from `config.py` and merging them with a specified preset JSON file (`Presets/*.json`). Pre-compiles regex patterns from presets for efficiency. -* `config.py`: Stores global default settings, constants, and core rules (e.g., standard map types, default resolutions, merge rules, output format rules, Blender paths). -* `main.py`: Entry point for the Command-Line Interface (CLI). Handles argument parsing, logging setup, parallel processing orchestration (using `concurrent.futures.ProcessPoolExecutor`), calls `AssetProcessor` via a wrapper function, and optionally triggers Blender scripts. -* `monitor.py`: Implements the automated directory monitoring feature using the `watchdog` library. Contains the `ZipHandler` class to detect new ZIP files and trigger processing via `main.run_processing`. -* `gui/`: Directory containing all code related to the Graphical User Interface (GUI), built with PySide6. - * `main_window.py`: Defines the `MainWindow` class, the main application window structure, UI layout (preset editor, processing panel, drag-and-drop, preview table, controls), event handling (button clicks, drag/drop), and menu setup. Manages GUI-specific logging (`QtLogHandler`). - * `processing_handler.py`: Defines the `ProcessingHandler` class (runs on a `QThread`). Manages the execution of the main asset processing pipeline (using `ProcessPoolExecutor`) and Blender script execution in the background to keep the GUI responsive. Communicates progress and results back to the `MainWindow` via signals. - * `prediction_handler.py`: Defines the `PredictionHandler` class (runs on a `QThread`). Manages background file analysis/preview generation by calling `AssetProcessor.get_detailed_file_predictions()`. Sends results back to the `MainWindow` via signals to update the preview table. - * `preview_table_model.py`: Defines `PreviewTableModel` (inherits `QAbstractTableModel`) and `PreviewSortFilterProxyModel` for managing and displaying data in the GUI's preview table, including custom sorting logic. -* `blenderscripts/`: Contains Python scripts (`create_nodegroups.py`, `create_materials.py`) designed to be executed *within* Blender, typically triggered by the main tool after processing to automate PBR nodegroup and material setup in `.blend` files. -* `Presets/`: Contains supplier-specific configuration files in JSON format (e.g., `Poliigon.json`). These define rules for interpreting asset filenames, classifying maps, handling variants, etc. `_template.json` serves as a base for new presets. -* `Testfiles/`: Contains example input assets for testing purposes. -* `Tickets/`: Directory for issue and feature tracking using Markdown files. - --------------------------------- -3. Key Components/Modules --------------------------------- - -* **`AssetProcessor` (`asset_processor.py`):** The heart of the tool. Orchestrates the entire processing pipeline for a single input asset (ZIP or folder). Responsibilities include workspace management, file classification, metadata extraction, map processing (resizing, format conversion), channel merging, `metadata.json` generation, and organizing final output files. -* **`Configuration` (`configuration.py`):** Manages the loading and merging of configuration settings. Takes a preset name, loads defaults from `config.py`, loads the specified `Presets/*.json`, merges them, validates settings, and pre-compiles regex patterns defined in the preset for efficient use by `AssetProcessor`. -* **`MainWindow` (`gui/main_window.py`):** The main class for the GUI application. Sets up the UI layout, connects user actions (button clicks, drag/drop) to slots, manages the preset editor, interacts with background handlers (`ProcessingHandler`, `PredictionHandler`) via signals/slots, and displays feedback (logs, progress, status). -* **`ProcessingHandler` (`gui/processing_handler.py`):** Handles the execution of the core asset processing logic and Blender scripts in a background thread for the GUI. Manages the `ProcessPoolExecutor` for parallel asset processing and communicates progress/results back to the `MainWindow`. -* **`PredictionHandler` (`gui/prediction_handler.py`):** Handles the generation of file classification previews in a background thread for the GUI. Calls `AssetProcessor`'s prediction methods and sends results back to the `MainWindow` to populate the preview table without blocking the UI. -* **`ZipHandler` (`monitor.py`):** A `watchdog` event handler used by `monitor.py`. Detects newly created ZIP files in the monitored input directory, validates the filename format (for preset extraction), and triggers the main processing logic via `main.run_processing`. - --------------------------------- -4. Core Concepts & Data Flow --------------------------------- - -* **Preset-Driven Configuration:** - * Global defaults are set in `config.py`. - * Supplier-specific rules (filename patterns, map keywords, variant handling, etc.) are defined using regex in `Presets/*.json` files. - * The `Configuration` class loads `config.py` and merges it with the selected preset JSON, providing a unified configuration object to the `AssetProcessor`. Regex patterns are pre-compiled during `Configuration` initialization for performance. - -* **Asset Processing Pipeline (Simplified Flow):** - 1. **Workspace Setup:** Create a temporary directory. - 2. **Extract/Copy:** Extract ZIP or copy folder contents to the workspace. - 3. **Classify Files:** Scan workspace, use compiled regex from `Configuration` to classify files (Map, Model, Extra, Ignored, Unrecognized). Handle 16-bit variants and assign suffixes based on rules. - 4. **Determine Metadata:** Extract asset name, category, archetype based on preset rules. - 5. **Skip Check:** If overwrite is false, check if output already exists; if so, skip this asset. - 6. **Process Maps:** Load images, resize (no upscale), convert format/bit depth based on complex rules (`config.py` and preset), handle Gloss->Roughness inversion, calculate stats, determine aspect ratio change. Save processed maps. - 7. **Merge Maps:** Combine channels from different processed maps based on `MAP_MERGE_RULES` in `config.py`. Save merged maps. - 8. **Generate `metadata.json`:** Collect all relevant information (map details, stats, aspect ratio, category, etc.) and write to `metadata.json` in the workspace. - 9. **Organize Output:** Create the final output directory structure (`///`) and move processed maps, merged maps, models, `metadata.json`, Extra files, and Ignored files into it. - 10. **Cleanup Workspace:** Delete the temporary directory. - 11. **(Optional) Blender Scripts:** If triggered via CLI/GUI, execute `blenderscripts/*.py` using the configured Blender executable via a subprocess. - -* **Parallel Processing:** - * Multiple input assets are processed concurrently using `concurrent.futures.ProcessPoolExecutor`. - * This pool is managed by `main.py` (CLI) or `gui/processing_handler.py` (GUI). - * Each asset runs in an isolated worker process, ensuring separate `Configuration` and `AssetProcessor` instances. - -* **GUI Interaction & Threading:** - * The GUI (`PySide6`) uses `QThread` to run `ProcessingHandler` (asset processing) and `PredictionHandler` (file preview generation) in the background, preventing the UI from freezing. - * Communication between the main UI thread (`MainWindow`) and background threads relies on Qt's signals and slots mechanism for thread safety (e.g., updating progress, status messages, preview table data). - * `PredictionHandler` calls `AssetProcessor` methods to get file classification details, which are then sent back to `MainWindow` to populate the `PreviewTableModel`. - -* **Output (`metadata.json`):** - * A key output file generated for each processed asset. - * Contains structured data about the asset: map filenames, resolutions, formats, bit depths, merged map details, calculated image statistics, aspect ratio change info, asset category/archetype, source preset used, list of ignored source files, etc. This file is intended for use by downstream tools or scripts (like the Blender integration scripts). - --------------------------------- -5. Development Workflow --------------------------------- - -* **Modifying Core Processing Logic:** Changes to how assets are classified, maps are processed/resized/converted, channels are merged, or metadata is generated typically involve editing the `AssetProcessor` class in `asset_processor.py`. -* **Changing Global Settings/Rules:** Adjustments to default output paths, standard resolutions, default format rules, map merge definitions, or Blender paths should be made in `config.py`. -* **Adding/Modifying Supplier Rules:** To add support for a new asset source or change how an existing one is interpreted, create or edit the corresponding JSON file in the `Presets/` directory. Refer to `_template.json` and existing presets for structure. Focus on defining accurate regex patterns in `map_type_mapping`, `bit_depth_variants`, `model_patterns`, `source_naming_convention`, etc. -* **Adjusting CLI Behavior:** Changes to command-line arguments, argument parsing, or the overall CLI workflow are handled in `main.py`. -* **Modifying the GUI:** UI layout changes, adding new controls, altering event handling, or modifying background task management for the GUI involves working within the `gui/` directory, primarily `main_window.py`, `processing_handler.py`, and `prediction_handler.py`. UI elements are built using PySide6 widgets. -* **Enhancing Blender Integration:** Improvements or changes to how nodegroups or materials are created in Blender require editing the Python scripts within the `blenderscripts/` directory. Consider how these scripts are invoked and what data they expect (primarily from `metadata.json` and command-line arguments passed via subprocess calls in `main.py` or `gui/processing_handler.py`). - --------------------------------- -6. Coding Conventions --------------------------------- - -* **Object-Oriented:** The codebase heavily utilizes classes (e.g., `AssetProcessor`, `Configuration`, `MainWindow`, various Handlers). -* **Type Hinting:** Python type hints are used throughout the code for clarity and static analysis. -* **Logging:** Standard Python `logging` module is used for logging messages at different levels (DEBUG, INFO, WARNING, ERROR). The GUI uses a custom `QtLogHandler` to display logs in the UI console. -* **Error Handling:** Uses standard `try...except` blocks and defines some custom exceptions (e.g., `ConfigurationError`, `AssetProcessingError`). -* **Parallelism:** Uses `concurrent.futures.ProcessPoolExecutor` for CPU-bound tasks (asset processing). -* **GUI:** Uses `PySide6` (Qt for Python) with signals and slots for communication between UI elements and background threads (`QThread`). -* **Configuration:** Relies on Python modules (`config.py`) for core settings and JSON files (`Presets/`) for specific rule sets. -* **File Paths:** Uses `pathlib.Path` for handling file system paths. -================================ -Internal Details for Debugging -================================ - -This section provides deeper technical details about the internal workings, intended to aid in debugging unexpected behavior. - --------------------------------- -7. Internal Logic & Algorithms --------------------------------- - -* **Configuration Preparation (`Configuration` class in `configuration.py`):** - * Instantiated per preset (`__init__`). - * Loads core settings from `config.py` using `importlib.util`. - * Loads specified preset from `presets/{preset_name}.json`. - * Validates basic structure of loaded settings (`_validate_configs`), checking for required keys and basic types (e.g., `map_type_mapping` is a list of dicts). - * Compiles regex patterns (`_compile_regex_patterns`) from preset rules (extra, model, bit depth, map keywords) using `re.compile` (mostly case-insensitive) and stores them on the instance (e.g., `self.compiled_map_keyword_regex`). Uses `_fnmatch_to_regex` helper for basic wildcard conversion. - -* **CLI Argument Parsing (`main.py:setup_arg_parser`):** - * Uses `argparse` to define and parse command-line arguments. - * Key arguments influencing flow: `--preset` (required), `--output-dir` (optional override), `--workers` (concurrency), `--overwrite` (force reprocessing), `--verbose` (logging level), `--nodegroup-blend`, `--materials-blend`. - * Calculates a default worker count based on `os.cpu_count()`. - -* **Output Directory Resolution (`main.py:main`):** - * Determines the base output directory by checking `--output-dir` argument first, then falling back to `OUTPUT_BASE_DIR` from `config.py`. - * Resolves the path to an absolute path and ensures the directory exists (`Path.resolve()`, `Path.mkdir(parents=True, exist_ok=True)`). - -* **Asset Processing (`AssetProcessor` class in `asset_processor.py`):** - * **Classification (`_inventory_and_classify_files`):** - * Multi-pass approach: Explicit Extra (regex) -> Models (regex) -> Potential Maps (keyword regex) -> Standalone 16-bit check (regex) -> Prioritize 16-bit variants -> Final Maps -> Remaining as Unrecognised (Extra). - * Uses compiled regex patterns provided by the `Configuration` object passed during initialization. - * Sorts potential map variants based on: 1. Preset rule index, 2. Keyword index within rule, 3. Alphabetical path. Suffixes (`-1`, `-2`) are assigned later per-asset based on this sort order and `RESPECT_VARIANT_MAP_TYPES`. - * **Map Processing (`_process_maps`):** - * Loads images using `cv2.imread` (flags: `IMREAD_UNCHANGED` or `IMREAD_GRAYSCALE`). Converts loaded 3-channel images from BGR to RGB for internal consistency (stats, merging). - * **Saving Channel Order:** Before saving with `cv2.imwrite`, 3-channel images are conditionally converted back from RGB to BGR *only* if the target output format is *not* EXR (e.g., for PNG, JPG, TIF). This ensures correct channel order for standard formats while preserving RGB for EXR. (Fix for ISSUE-010). - * Handles Gloss->Roughness inversion: Loads gloss, inverts using float math (`1.0 - img/norm`), stores as float32 with original dtype. Prioritizes gloss source if both gloss and native rough exist. - * Resizes using `cv2.resize` (interpolation: `INTER_LANCZOS4` for downscale, `INTER_CUBIC` for potential same-size/upscale - though upscaling is generally avoided by checks). - * Determines output format based on hierarchy: `FORCE_LOSSLESS_MAP_TYPES` > `RESOLUTION_THRESHOLD_FOR_JPG` > Input format priority (TIF/EXR often lead to lossless) > Configured defaults (`OUTPUT_FORMAT_16BIT_PRIMARY`, `OUTPUT_FORMAT_8BIT`). - * Determines output bit depth based on `MAP_BIT_DEPTH_RULES` ('respect' vs 'force_8bit'). - * Converts dtype before saving (e.g., float to uint8/uint16 using scaling factors 255.0/65535.0). - * Calculates stats (`_calculate_image_stats`) on normalized float64 data (in RGB space) for a specific resolution (`CALCULATE_STATS_RESOLUTION`). - * Calculates aspect ratio string (`_normalize_aspect_ratio_change`) based on relative dimension changes. - * Handles save fallback: If primary 16-bit format (e.g., EXR) fails, attempts fallback (e.g., PNG). - * **Merging (`_merge_maps_from_source`):** - * Identifies the required *source* files for merge inputs based on classified files. - * Determines common resolutions based on available processed maps (as a proxy for size compatibility). - * Loads required source maps for each common resolution using the `_load_and_transform_source` helper (utilizing the cache). - * Converts loaded inputs to float32 (normalized 0-1). - * Injects default values (from rule `defaults`) for missing channels. - * Merges channels using `cv2.merge`. - * Determines output bit depth based on rule (`force_16bit`, `respect_inputs`). - * Determines output format based on complex rules (`config.py` and preset), considering the highest format among *source* inputs if not forced lossless or over JPG threshold. Handles JPG 16-bit conflict by forcing 8-bit. - * Saves the merged image using the `_save_image` helper, including final data type/color space conversions and fallback logic (e.g., EXR->PNG). - * **Metadata (`_determine_base_metadata`, `_determine_single_asset_metadata`, `_generate_metadata_file`):** - * Base name determined using `source_naming` separator/index from `Configuration`, with fallback to common prefix or input name. Handles multiple assets within one input. - * Category determined by model presence or `decal_keywords` from `Configuration`. - * Archetype determined by matching keywords in `archetype_rules` (from `Configuration`) against file stems/base name. - * Final `metadata.json` populated by accumulating results (map details, stats, features, etc.) during the per-asset processing loop. - -* **Blender Integration (`main.py:run_blender_script`, `gui/processing_handler.py:_run_blender_script_subprocess`):** - * Uses `subprocess.run` to execute Blender. - * Command includes `-b` (background), the target `.blend` file, `--python` followed by the script path (`blenderscripts/*.py`), and `--` separator. - * Arguments after `--` (currently just the `asset_root_dir`, and optionally the nodegroup blend path for the materials script) are passed to the Python script via `sys.argv`. - * Uses `--factory-startup` in GUI handler. Checks return code and logs stdout/stderr. - --------------------------------- -8. State Management --------------------------------- - -* **`Configuration` Object:** Holds the loaded and merged configuration state (core + preset) and compiled regex patterns. Designed to be immutable after initialization. Instantiated once per worker process. -* **`AssetProcessor` Instance:** Primarily stateless between calls to `process()`. State *within* a `process()` call is managed through local variables scoped to the overall call or the per-asset loop (e.g., `current_asset_metadata`, `processed_maps_details_asset`). `self.classified_files` is populated once by `_inventory_and_classify_files` early in `process()` and then used read-only (filtered copies) within the per-asset loop. -* **`main.py` (CLI):** Tracks overall run progress (processed, skipped, failed counts) based on results returned from worker processes. -* **`gui/processing_handler.py`:** Manages the state of a GUI processing run using internal flags (`_is_running`, `_cancel_requested`) and stores `Future` objects in `self._futures` dictionary while the pool is active. - --------------------------------- -9. Error Handling & Propagation --------------------------------- - -* **Custom Exceptions:** `ConfigurationError` (raised by `Configuration` on load/validation failure), `AssetProcessingError` (raised by `AssetProcessor` for various processing failures). -* **Configuration:** `ConfigurationError` halts initialization. Regex compilation errors are logged as warnings but do not stop initialization. -* **AssetProcessor:** Uses `try...except Exception` within key pipeline steps (`_process_maps`, `_merge_maps`, etc.) and within the per-asset loop in `process()`. Errors specific to one asset are logged (`log.error(exc_info=True)`), the asset is marked "failed" in the returned status dictionary, and the loop continues to the next asset. Critical setup errors (e.g., workspace creation) raise `AssetProcessingError`, halting the entire `process()` call. Includes specific save fallback logic (EXR->PNG) on `cv2.imwrite` failure for 16-bit formats. -* **Worker Wrapper (`main.py:process_single_asset_wrapper`):** Catches `ConfigurationError`, `AssetProcessingError`, and general `Exception` during worker execution. Logs the error and returns a ("failed", error_message) status tuple to the main process. -* **Process Pool (`main.py`, `gui/processing_handler.py`):** The `with ProcessPoolExecutor(...)` block handles pool setup/teardown. A `try...except` around `as_completed` or `future.result()` catches critical worker failures (e.g., process crash). -* **GUI Communication (`ProcessingHandler`):** Catches exceptions during `future.result()` retrieval. Emits `file_status_updated` signal with "failed" status and error message. Emits `processing_finished` with final counts. -* **Blender Scripts:** Checks `subprocess.run` return code. Logs stderr as ERROR if return code is non-zero, otherwise as WARNING. Catches `FileNotFoundError` if the Blender executable path is invalid. - --------------------------------- -10. Key Data Structures --------------------------------- - -* **`Configuration` Instance Attributes:** - * `compiled_map_keyword_regex`: `dict[str, list[tuple[re.Pattern, str, int]]]` (Base type -> list of compiled regex tuples) - * `compiled_extra_regex`, `compiled_model_regex`: `list[re.Pattern]` - * `compiled_bit_depth_regex_map`: `dict[str, re.Pattern]` (Base type -> compiled regex) -* **`AssetProcessor` Internal Structures (within `process()`):** - * `self.classified_files`: `dict[str, list[dict]]` (Category -> list of file info dicts like `{'source_path': Path, 'map_type': str, ...}`) - * `processed_maps_details_asset`, `merged_maps_details_asset`: `dict[str, dict[str, dict]]` (Map Type -> Resolution Key -> Details Dict `{'path': Path, 'width': int, ...}`) - * `file_to_base_name_map`: `dict[Path, Optional[str]]` (Source relative path -> Determined asset base name or None) - * `current_asset_metadata`: `dict` (Accumulates name, category, archetype, stats, map details per asset) -* **Return Values:** - * `AssetProcessor.process()`: `Dict[str, List[str]]` (e.g., `{"processed": [...], "skipped": [...], "failed": [...]}`) - * `main.process_single_asset_wrapper()`: `Tuple[str, str, Optional[str]]` (input_path, status_string, error_message) -* **`ProcessingHandler._futures`:** `dict[Future, str]` (Maps `concurrent.futures.Future` object to the input path string) -* **Image Data:** `numpy.ndarray` (Handled by OpenCV). - --------------------------------- -11. Concurrency Models (CLI & GUI) --------------------------------- - -* **Common Core:** Both CLI and GUI utilize `concurrent.futures.ProcessPoolExecutor` for parallel processing. The target function executed by workers is `main.process_single_asset_wrapper`. -* **Isolation:** Crucially, `Configuration` and `AssetProcessor` objects are instantiated *within* the `process_single_asset_wrapper` function, meaning each worker process gets its own independent configuration and processor instance based on the arguments passed. This prevents state conflicts between concurrent asset processing tasks. Data is passed between the main process and workers via pickling of arguments and return values. -* **CLI Orchestration (`main.py:run_processing`):** - * Creates the `ProcessPoolExecutor`. - * Submits all `process_single_asset_wrapper` tasks. - * Uses `concurrent.futures.as_completed` to iterate over finished futures as they complete, blocking until the next one is done. - * Gathers results synchronously within the main script's execution flow. -* **GUI Orchestration (`gui/processing_handler.py`):** - * The `ProcessingHandler` object (a `QObject`) contains the `run_processing` method. - * This method is intended to be run in a separate `QThread` (managed by `MainWindow`) to avoid blocking the main UI thread. - * Inside `run_processing`, it creates and manages the `ProcessPoolExecutor`. - * It uses `as_completed` similarly to the CLI to iterate over finished futures. - * **Communication:** Instead of blocking the thread gathering results, it emits Qt signals (`progress_updated`, `file_status_updated`, `processing_finished`) from within the `as_completed` loop. These signals are connected to slots in `MainWindow` (running on the main UI thread), allowing for thread-safe updates to the GUI (progress bar, table status, status bar messages). -* **Cancellation (GUI - `gui/processing_handler.py:request_cancel`):** - * Sets an internal `_cancel_requested` flag. - * Attempts `executor.shutdown(wait=False)` which prevents new tasks from starting and may cancel pending ones (depending on Python version). - * Manually iterates through stored `_futures` and calls `future.cancel()` on those not yet running or done. - * **Limitation:** This does *not* forcefully terminate worker processes that are already executing the `process_single_asset_wrapper` function. Cancellation primarily affects pending tasks and the processing of results from already running tasks (they will be marked as failed/cancelled when their future completes). - --------------------------------- -12. Resource Management --------------------------------- - -* **Configuration:** Preset JSON files are opened and closed using `with open(...)`. -* **AssetProcessor:** - * Temporary workspace directory created using `tempfile.mkdtemp()`. - * Cleanup (`_cleanup_workspace`) uses `shutil.rmtree()` and is called within a `finally` block in the main `process()` method, ensuring cleanup attempt even if errors occur. - * Metadata JSON file written using `with open(...)`. - * Image data is loaded into memory using OpenCV/NumPy; memory usage depends on image size and number of concurrent workers. -* **Process Pool:** The `ProcessPoolExecutor` manages the lifecycle of worker processes. Using it within a `with` statement (as done in `main.py` and `gui/processing_handler.py`) ensures proper shutdown and resource release for the pool itself. - --------------------------------- -13. Known Limitations & Edge Cases --------------------------------- - -* **Configuration:** - * Validation (`_validate_configs`) is primarily structural (key presence, basic types), not deeply logical (e.g., doesn't check if regex patterns are *sensible*). - * Regex compilation errors in `_compile_regex_patterns` are logged as warnings but don't prevent `Configuration` initialization, potentially leading to unexpected classification later. - * `_fnmatch_to_regex` helper only handles basic `*` and `?` wildcards. Complex fnmatch patterns might not translate correctly. -* **AssetProcessor:** - * Heavily reliant on correct filename patterns and rules defined in presets. Ambiguous or incorrect patterns lead to misclassification. - * Potential for high memory usage when processing very large images, especially with many workers. - * Error handling within `process()` is per-asset; a failure during map processing for one asset marks the whole asset as failed, without attempting other maps for that asset. No partial recovery within an asset. - * Gloss->Roughness inversion assumes gloss map is single channel or convertible to grayscale. - * `predict_output_structure` and `get_detailed_file_predictions` use simplified logic (e.g., assuming PNG output, highest resolution only) and may not perfectly match final output names/formats in all cases. - * Filename sanitization (`_sanitize_filename`) is basic and might not cover all edge cases for all filesystems. -* **CLI (`main.py`):** - * Preset existence check (`{preset}.json`) happens only in the main process before workers start. - * Blender executable finding logic relies on `config.py` path being valid or `blender` being in the system PATH. -* **GUI Concurrency (`gui/processing_handler.py`):** - * Cancellation (`request_cancel`) is not immediate for tasks already running in worker processes. It prevents new tasks and stops processing results from completed futures once the flag is checked. -* **General:** - * Limited input format support (ZIP archives, folders). Internal file formats limited by OpenCV (`cv2.imread`, `cv2.imwrite`). Optional `OpenEXR` package recommended for full EXR support. - * Error messages propagated from workers might lack full context in some edge cases. \ No newline at end of file diff --git a/Deprecated/documentation_plan.md b/Deprecated/documentation_plan.md deleted file mode 100644 index d5a1bf9..0000000 --- a/Deprecated/documentation_plan.md +++ /dev/null @@ -1,70 +0,0 @@ -# Asset Processor Tool Documentation Plan - -This document outlines the proposed structure for the documentation of the Asset Processor Tool, based on the content from `readme.md` and `documentation.txt`. The goal is to create a clear, modular, and comprehensive documentation set within a new `Documentation` directory. - -## Proposed Directory Structure - -``` -Documentation/ -├── 00_Overview.md -├── 01_User_Guide/ -│ ├── 01_Introduction.md -│ ├── 02_Features.md -│ ├── 03_Installation.md -│ ├── 04_Configuration_and_Presets.md -│ ├── 05_Usage_GUI.md -│ ├── 06_Usage_CLI.md -│ ├── 07_Usage_Monitor.md -│ ├── 08_Usage_Blender.md -│ ├── 09_Output_Structure.md -│ └── 10_Docker.md -└── 02_Developer_Guide/ - ├── 01_Architecture.md - ├── 02_Codebase_Structure.md - ├── 03_Key_Components.md - ├── 04_Configuration_System_and_Presets.md - ├── 05_Processing_Pipeline.md - ├── 06_GUI_Internals.md - ├── 07_Monitor_Internals.md - ├── 08_Blender_Integration_Internals.md - ├── 09_Development_Workflow.md - ├── 10_Coding_Conventions.md - └── 11_Debugging_Notes.md -``` - -## File Content Breakdown - -### `Documentation/00_Overview.md` - -* Project purpose, scope, and intended audience. -* High-level summary of the tool's functionality. -* Table of Contents for the entire documentation set. - -### `Documentation/01_User_Guide/` - -* **`01_Introduction.md`**: Brief welcome and purpose for users. -* **`02_Features.md`**: Detailed list of user-facing features. -* **`03_Installation.md`**: Requirements and step-by-step installation instructions. -* **`04_Configuration_and_Presets.md`**: Explains user-level configuration options (`config.py` settings relevant to users) and how to select and understand presets. -* **`05_Usage_GUI.md`**: Guide on using the Graphical User Interface, including descriptions of panels, controls, and workflow. -* **`06_Usage_CLI.md`**: Guide on using the Command-Line Interface, including arguments and examples. -* **`07_Usage_Monitor.md`**: Guide on setting up and using the Directory Monitor for automated processing. -* **`08_Usage_Blender.md`**: Explains the user-facing aspects of the Blender integration. -* **`09_Output_Structure.md`**: Describes the structure and contents of the generated asset library. -* **`10_Docker.md`**: Instructions for building and running the tool using Docker. - -### `Documentation/02_Developer_Guide/` - -* **`01_Architecture.md`**: High-level technical architecture, core components, and their relationships. -* **`02_Codebase_Structure.md`**: Detailed breakdown of key files and directories within the project. -* **`03_Key_Components.md`**: In-depth explanation of major classes and modules (`AssetProcessor`, `Configuration`, GUI Handlers, etc.). -* **`04_Configuration_System_and_Presets.md`**: Technical details of the configuration loading and merging process, the structure of preset JSON files, and guidance on creating/modifying presets for developers. -* **`05_Processing_Pipeline.md`**: Step-by-step technical breakdown of the asset processing logic within the `AssetProcessor` class. -* **`06_GUI_Internals.md`**: Technical details of the GUI implementation, including threading, signals/slots, and background task management. -* **`07_Monitor_Internals.md`**: Technical details of the Directory Monitor implementation using `watchdog`. -* **`08_Blender_Integration_Internals.md`**: Technical details of how the Blender scripts are executed and interact with the processed assets. -* **`09_Development_Workflow.md`**: Guidance for developers on contributing, setting up a development environment, and modifying specific parts of the codebase. -* **`10_Coding_Conventions.md`**: Overview of the project's coding standards, object-oriented approach, type hinting, logging, and error handling. -* **`11_Debugging_Notes.md`**: Advanced internal details, state management, error propagation, concurrency models, resource management, and known limitations/edge cases. - -This plan provides a solid foundation for organizing the existing documentation and serves as a roadmap for creating the new markdown files. \ No newline at end of file diff --git a/Deprecated/readme.md b/Deprecated/readme.md deleted file mode 100644 index bb5fb03..0000000 Binary files a/Deprecated/readme.md and /dev/null differ diff --git a/Deprecated/readme.md.bak b/Deprecated/readme.md.bak deleted file mode 100644 index 5f45ead..0000000 --- a/Deprecated/readme.md.bak +++ /dev/null @@ -1,356 +0,0 @@ -# Asset Processor Tool vX.Y - -## Overview - -This tool processes 3D asset source files (texture sets, models, etc., provided as ZIP archives or folders) into a standardized library format. It uses configurable presets to interpret different asset sources and automates tasks like file classification, image resizing, channel merging, and metadata generation. - -The tool offers both a Graphical User Interface (GUI) for interactive use and a Command-Line Interface (CLI) for batch processing and scripting. - -This tool is currently work in progress, rewritting features from an original proof of concept, original script can be found at `Deprecated-POC/` for reference - -## Features - -* **Preset-Driven:** Uses JSON presets (`presets/`) to define rules for different asset suppliers (e.g., `Poliigon.json`). -* **Dual Interface:** Provides both a user-friendly GUI and a powerful CLI. -* **Parallel Processing:** Utilizes multiple CPU cores for faster processing of multiple assets (configurable via `--workers` in CLI or GUI control). -* **Multi-Asset Input Handling:** Correctly identifies and processes multiple distinct assets contained within a single input ZIP or folder, creating separate outputs for each. -* **File Classification:** Automatically identifies map types (Color, Normal, Roughness, etc.), models, explicitly marked extra files, and unrecognised files based on preset rules. - * **Variant Handling:** Map types listed in `RESPECT_VARIANT_MAP_TYPES` (in `config.py`, e.g., `"COL"`) will *always* receive a numeric suffix (`-1`, `-2`, etc.). The numbering priority is determined primarily by the order of keywords listed in the preset's `map_type_mapping`. Alphabetical sorting of filenames is used only as a tie-breaker for files matching the exact same keyword pattern. Other map types will *never* receive a suffix. - * **16-bit Prioritization:** Correctly identifies 16-bit variants defined in preset `bit_depth_variants` (e.g., `*_NRM16.tif`), prioritizes them, and ignores the corresponding 8-bit version (marked as `Ignored` in GUI). -* **Map Processing:** - * Resizes texture maps to configured power of two resolutions (e.g., 4K, 2K, 1K), avoiding upscaling. - * Handles Glossiness map inversion to Roughness. - * Applies bit-depth rules (`respect` source or `force_8bit`). - * Saves maps in appropriate formats. Map types listed in `FORCE_LOSSLESS_MAP_TYPES` (in `config.py`, e.g., `"NRM"`, `"DISP"`) are *always* saved in a lossless format (PNG for 8-bit, configured 16-bit format like EXR/PNG for 16-bit), overriding other rules. For other map types, if the output is 8-bit and the resolution meets or exceeds `RESOLUTION_THRESHOLD_FOR_JPG` (in `config.py`), the output is forced to JPG. Otherwise, the format is based on input type and target bit depth: JPG inputs yield JPG outputs (8-bit); TIF inputs yield PNG/EXR (based on target bit depth and config); other inputs use configured formats (PNG/EXR). Merged maps follow similar logic, checking `FORCE_LOSSLESS_MAP_TYPES` first, then the threshold for 8-bit targets, then using the highest format from inputs (EXR > TIF > PNG > JPG hierarchy, with TIF adjusted to PNG/EXR based on target bit depth). - * Calculates basic image statistics (Min/Max/Mean) for a reference resolution. - * Calculates and stores the relative aspect ratio change string in metadata. -* **Channel Merging:** Combines channels from different maps into packed textures (e.g., NRMRGH) based on preset rules. -* **Metadata Generation:** Creates a `metadata.json` file for each asset containing details about maps, category, archetype, aspect ratio change, processing settings, etc. **Aspect Ratio Metadata:** Calculates the relative aspect ratio change during resizing and stores it in the `metadata.json` file (`aspect_ratio_change_string`). The format indicates if the aspect is unchanged (`EVEN`), scaled horizontally (`X150`, `X110`, etc.), scaled vertically (`Y150`, `Y125`, etc.) -* **Output Organization:** Creates a clean, structured output directory (`///`). -* **Skip/Overwrite:** Can skip processing if the output already exists or force reprocessing with the `--overwrite` flag (CLI) or checkbox (GUI). -* **Blender Integration:** Optionally runs Blender scripts (`create_nodegroups.py`, `create_materials.py`) after asset processing to automate node group and material creation in specified `.blend` files. Available via both CLI and GUI. -* **GUI Features:** - * Drag-and-drop input for assets (ZIPs/folders). - * Integrated preset editor panel for managing `.json` presets. - * Configurable output directory field with a browse button (defaults to path in `config.py`). - * Enhanced live preview table showing predicted file status (Mapped, Model, Extra, Unrecognised, Ignored, Error) based on the selected processing preset. - * Toggleable preview mode (via View menu) to switch between detailed file preview and a simple list of input assets. - * Toggleable log console panel (via View menu) displaying application log messages within the GUI. - * Progress bar, cancellation button, and clear queue button. - * **Blender Post-Processing Controls:** Checkbox to enable/disable Blender script execution and input fields with browse buttons to specify the target `.blend` files for node group and material creation (defaults configurable in `config.py`). -* **Responsive GUI:** Utilizes background threads (`QThread`) for processing (`ProcessPoolExecutor`) and file preview generation (`ThreadPoolExecutor`), ensuring the user interface remains responsive during intensive operations. -* **Optimized Classification:** Pre-compiles regular expressions from presets for faster file identification during classification. -* **Docker Support:** Includes a `Dockerfile` for containerized execution. - -## Directory Structure - -``` -Asset_processor_tool/ -│ -├── main.py # CLI Entry Point & processing orchestrator -├── monitor.py # Directory monitoring script for automated processing -├── asset_processor.py # Core class handling single asset processing pipeline -├── configuration.py # Class for loading and accessing configuration -├── config.py # Core settings definition (output paths, resolutions, merge rules etc.) -│ -├── blenderscripts/ # Scripts for integration with Blender -│ └── create_nodegroups.py # Script to create node groups from processed assets -│ └── create_materials.py # Script to create materials linking to node groups -│ -├── gui/ # Contains files related to the Graphical User Interface -│ ├── main_window.py # Main GUI application window and layout -│ ├── processing_handler.py # Handles background processing logic for the GUI -│ ├── prediction_handler.py # Handles background file prediction/preview for the GUI -│ -├── Presets/ # Preset definition files -│ ├── _template.json # Template for creating new presets -│ └── Poliigon.json # Example preset for Poliigon assets -│ -├── Testfiles/ # Directory containing example input assets for testing -│ -├── Tickets/ # Directory for issue and feature tracking (Markdown files) -│ ├── _template.md # Template for creating new tickets -│ └── Ticket-README.md # Explanation of the ticketing system -│ -├── requirements.txt # Python package dependencies for standard execution -├── requirements-docker.txt # Dependencies specifically for the Docker environment -├── Dockerfile # Instructions for building the Docker container image -└── readme.md # This documentation file -``` - -* **Core Logic:** `main.py`, `monitor.py`, `asset_processor.py`, `configuration.py`, `config.py` -* **Blender Integration:** `blenderscripts/` directory -* **GUI:** `gui/` directory -* **Configuration:** `config.py`, `Presets/` directory -* **Dependencies:** `requirements.txt`, `requirements-docker.txt` -* **Containerization:** `Dockerfile` -* **Documentation/Planning:** `readme.md`, `Project Notes/` directory -* **Issue/Feature Tracking:** `Tickets/` directory (see `Tickets/README.md`) -* **Testing:** `Testfiles/` directory - -## Architecture - -This section provides a higher-level overview of the tool's internal structure and design, intended for developers or users interested in the technical implementation. - -### Core Components - -The tool is primarily built around several key Python modules: - -* `config.py`: Defines core, global settings (output paths, resolutions, default behaviors, format rules, Blender executable path, default Blender file paths, etc.) that are generally not supplier-specific. -* `Presets/*.json`: Supplier-specific JSON files defining rules for interpreting source assets (filename patterns, map type keywords, model identification, etc.). -* `configuration.py` **(**`Configuration` **class)**: Responsible for loading the core `config.py` settings and merging them with a selected preset JSON file. Crucially, it also **pre-compiles** regular expression patterns defined in the preset (e.g., for map keywords, extra files, 16-bit variants) upon initialization. This pre-compilation significantly speeds up the file classification process. -* `asset_processor.py` **(**`AssetProcessor` **class)**: Contains the core logic for processing a *single* asset. It orchestrates the pipeline steps: workspace setup, extraction, file classification, metadata determination, map processing, channel merging, metadata file generation, and output organization. -* `main.py`: Serves as the entry point for the Command-Line Interface (CLI). It handles argument parsing, sets up logging, manages the parallel processing pool, calls `AssetProcessor` for each input asset via a wrapper function, and optionally triggers Blender script execution after processing. -* `gui/`: Contains modules related to the Graphical User Interface (GUI), built using PySide6. -* `monitor.py`: Implements the directory monitoring functionality for automated processing. - -### Parallel Processing (CLI & GUI) - -To accelerate the processing of multiple assets, the tool utilizes Python's `concurrent.futures.ProcessPoolExecutor`. - -* Both `main.py` (for CLI) and `gui/processing_handler.py` (for GUI background tasks) create a process pool. -* The actual processing for each asset is delegated to the `main.process_single_asset_wrapper` function. This wrapper is executed in a separate worker process within the pool. -* The wrapper function is responsible for instantiating the `Configuration` and `AssetProcessor` classes for the specific asset being processed in that worker. This isolates each asset's processing environment. -* Results (success, skip, failure, error messages) are communicated back from the worker processes to the main coordinating script (either `main.py` or `gui/processing_handler.py`). - -### Asset Processing Pipeline (`AssetProcessor` class) - -The `AssetProcessor` class executes a sequence of steps for each asset: - - 1. `_setup_workspace()`: Creates a temporary directory for processing. - 2. `_extract_input()`: Extracts the input ZIP archive or copies the input folder contents into the temporary workspace. - 3. `_inventory_and_classify_files()`: This is a critical step that scans the workspace and classifies each file based on rules defined in the loaded `Configuration` (which includes the preset). It uses the pre-compiled regex patterns for efficiency. Key logic includes: - * Identifying files explicitly marked for the `Extra/` folder. - * Identifying model files. - * Matching potential texture maps against keyword patterns. - * Identifying and prioritizing 16-bit variants (e.g., `_NRM16.tif`) over their 8-bit counterparts based on `source_naming.bit_depth_variants` patterns. Ignored 8-bit files are tracked. - * Handling map variants (e.g., multiple Color maps) by assigning suffixes (`-1`, `-2`) based on the `RESPECT_VARIANT_MAP_TYPES` setting in `config.py` and the order of keywords defined in the preset's `map_type_mapping`. - * Classifying any remaining files as 'Unrecognised' (which are also moved to the `Extra/` folder). - 4. `_determine_base_metadata()`: Determines the asset's base name, category (Texture, Asset, Decal), and archetype (e.g., Wood, Metal) based on classified files and preset rules (`source_naming`, `asset_category_rules`, `archetype_rules`). - 5. **Skip Check**: If `overwrite` is false, checks if the final output directory and metadata file already exist. If so, processing for this asset stops early. - 6. `_process_maps()`: Iterates through classified texture maps. For each map: - * Loads the image data (handling potential Gloss->Roughness inversion). - * Resizes the map to each target resolution specified in `config.py`, avoiding upscaling. - * Determines the output bit depth based on `MAP_BIT_DEPTH_RULES` (`respect` source or `force_8bit`). - * Determines the output file format (`.jpg`, `.png`, `.exr`) based on a combination of factors: - * The `RESOLUTION_THRESHOLD_FOR_JPG` (forces JPG for 8-bit maps above the threshold). - * The original input file format (e.g., `.jpg` inputs tend to produce `.jpg` outputs if 8-bit and below threshold). - * The target bit depth (16-bit outputs use configured `OUTPUT_FORMAT_16BIT_PRIMARY` or `_FALLBACK`). - * Configured 8-bit format (`OUTPUT_FORMAT_8BIT`). - * The `FORCE_LOSSLESS_MAP_TYPES` list in `config.py` (overrides all other logic for specified map types, ensuring PNG/EXR output). - * Saves the processed map for each resolution, applying appropriate compression/quality settings. Includes fallback logic if saving in the primary format fails (e.g., EXR -> PNG). - * Calculates basic image statistics (Min/Max/Mean) for a reference resolution (`CALCULATE_STATS_RESOLUTION`) and determines the aspect ratio change string (e.g., "EVEN", "X150", "Y075") stored in the metadata. - 7. `_merge_maps()`: Combines channels from different processed maps into new textures (e.g., NRMRGH) based on `MAP_MERGE_RULES` defined in `config.py`. It determines the output format for merged maps similarly to `_process_maps` (checking `FORCE_LOSSLESS_MAP_TYPES` first, then threshold, then input hierarchy), considering the formats of the input maps involved. - 8. `_generate_metadata_file()`: Collects all gathered information (asset name, maps present, resolutions, stats, aspect ratio change, etc.) and writes it to the `metadata.json` file. - 9. `_organize_output_files()`: Moves the processed maps, merged maps, models, metadata file, and any 'Extra'/'Unrecognised'/'Ignored' files from the temporary workspace to the final structured output directory (`///`). -10. `_cleanup_workspace()`: Removes the temporary workspace directory. - -### GUI Architecture (`gui/`) - -The GUI provides an interactive way to use the tool and manage presets. - -* **Framework**: Built using `PySide6`, the official Python bindings for the Qt framework. -* **Main Window (**`main_window.py`**)**: Defines the main application window, which includes: - * An integrated preset editor panel (using `QSplitter`). - * A processing panel with drag-and-drop support, output directory selection, a file preview table, and processing controls. - * **Blender Post-Processing Controls:** A group box containing a checkbox to enable/disable Blender script execution and input fields with browse buttons for specifying the target `.blend` files for node group and material creation. -* **Threading Model**: To prevent the UI from freezing during potentially long operations, background tasks are run in separate `QThread`s: - * `ProcessingHandler` **(**`processing_handler.py`**)**: Manages the execution of the main processing pipeline (using `ProcessPoolExecutor` and `main.process_single_asset_wrapper`, similar to the CLI) and the optional Blender script execution in a background thread. Receives the target output directory and Blender integration settings from the main window. - * `PredictionHandler` **(**`prediction_handler.py`**)**: Manages the generation of file previews in a background thread using a `ThreadPoolExecutor` to parallelize prediction across multiple assets. It calls `AssetProcessor.get_detailed_file_predictions()`, which performs extraction and classification. -* **Communication**: Qt's **signal and slot mechanism** is used for communication between the background threads (`ProcessingHandler`, `PredictionHandler`) and the main GUI thread (`MainWindow`). For example, signals are emitted to update the progress bar, populate the preview table, and report completion status or errors. A custom `QtLogHandler` redirects Python log messages to the UI console via signals. -* **Preset Editor**: The editor allows creating, modifying, and saving preset JSON files directly within the GUI. Changes are tracked, and users are prompted to save before closing or loading another preset if changes are pending. Includes an optional, toggleable log console panel at the top. - -### Monitor Architecture (`monitor.py`) - -The `monitor.py` script enables automated processing of assets dropped into a designated input directory. - -* **File System Watching**: Uses the `watchdog` library (specifically `PollingObserver` for cross-platform compatibility) to monitor the specified `INPUT_DIR`. -* **Event Handling**: A custom `ZipHandler` detects `on_created` events for `.zip` files. -* **Filename Parsing**: It expects filenames in the format `[preset]_filename.zip` and uses a regular expression (`PRESET_FILENAME_REGEX`) to extract the `preset` name. -* **Preset Validation**: Checks if the extracted preset name corresponds to a valid `.json` file in the `Presets/` directory. -* **Processing Trigger**: If the filename format and preset are valid, it calls the `main.run_processing` function (the same core logic used by the CLI) to process the detected ZIP file using the extracted preset. -* **File Management**: Moves the source ZIP file to either a `PROCESSED_DIR` (on success/skip) or an `ERROR_DIR` (on failure or invalid preset) after the processing attempt. - -### Error Handling - -* Custom exception classes (`ConfigurationError`, `AssetProcessingError`) are defined and used to signal specific types of errors during configuration loading or asset processing. -* Standard Python logging is used throughout the application (CLI, GUI, Monitor, Core Logic) to record information, warnings, and errors. Log levels can be configured. -* Worker processes in the processing pool capture exceptions and report them back to the main process for logging and status updates. - -## Requirements - -* Python 3.8+ -* Required Python Packages (see `requirements.txt`): - * `opencv-python` (for image processing) - * `numpy` (for numerical operations) - * `PySide6` (only needed for the GUI) -* Optional Python Packages: - * `OpenEXR` (provides more robust EXR file handling, recommended if processing EXR sources) -* **Blender:** A working installation of Blender is required for the optional Blender integration features. The path to the executable should be configured in `config.py` or available in the system's PATH. - -Install dependencies using pip: - -```bash -pip install -r requirements.txt -``` - -(For GUI, ensure PySide6 is included or install separately: `pip install PySide6`) - -## Configuration - -The tool's behavior is controlled by two main configuration components: - -1. `config.py`**:** Defines core, global settings: - * `OUTPUT_BASE_DIR`: Default root directory for processed assets. - * `DEFAULT_ASSET_CATEGORY`: Fallback category ("Texture", "Asset", "Decal"). - * `IMAGE_RESOLUTIONS`: Dictionary mapping resolution keys (e.g., "4K") to pixel dimensions. - * `RESPECT_VARIANT_MAP_TYPES`: List of map type strings (e.g., `["COL"]`) that should always receive a numeric suffix (`-1`, `-2`, etc.) based on preset order, even if only one variant exists. - * `TARGET_FILENAME_PATTERN`: Format string for output filenames. - * `MAP_MERGE_RULES`: List defining how to merge channels (e.g., creating NRMRGH). - * `ARCHETYPE_RULES`: Rules for determining asset usage archetype (e.g., Wood, Metal). - * `RESOLUTION_THRESHOLD_FOR_JPG`: Dimension threshold (pixels) above which 8-bit maps are forced to JPG format, overriding other format logic. - * `FORCE_LOSSLESS_MAP_TYPES`: List of map type strings (e.g., `["NRM", "DISP"]`) that should *always* be saved losslessly (PNG/EXR), overriding the JPG threshold and other format logic. - * `BLENDER_EXECUTABLE_PATH`: Path to the Blender executable (required for Blender integration). - * `DEFAULT_NODEGROUP_BLEND_PATH`: Default path to the .blend file for node group creation (used by GUI if not specified). - * `DEFAULT_MATERIALS_BLEND_PATH`: Default path to the .blend file for material creation (used by GUI if not specified). - * ... and other processing parameters (JPEG quality, PNG compression, 16-bit/8-bit output formats, etc.). -2. `presets/*.json`**:** Define supplier-specific rules. Each JSON file represents a preset (e.g., `Poliigon.json`). Key sections include: - * `supplier_name`: Name of the asset source. - * `map_type_mapping`: A list of dictionaries defining rules to map source filename keywords/patterns to standard map types. Each dictionary should have `"target_type"` (e.g., `"COL"`, `"NRM"`) and `"keywords"` (a list of source filename patterns like `["_col*", "_color"]`). For map types listed in `config.py`'s `RESPECT_VARIANT_MAP_TYPES`, the numbering priority (`-1`, `-2`, etc.) is determined primarily by the order of the keywords within the `"keywords"` list for the matching rule. Alphabetical sorting of filenames is used only as a secondary tie-breaker for files matching the exact same keyword pattern. Other map types do not receive suffixes. - * `bit_depth_variants`: Dictionary mapping standard map types (e.g., `"NRM"`) to fnmatch patterns used to identify their high bit-depth source files (e.g., `"*_NRM16*.tif"`). These take priority over standard keyword matches, and the corresponding 8-bit version will be ignored. - * `bit_depth_rules`: Specifies whether to `respect` source bit depth or `force_8bit` for specific map types (defined in `config.py`). - * `model_patterns`: Regex patterns to identify model files (e.g., `*.fbx`, `*.obj`). - * `move_to_extra_patterns`: Regex patterns for files to move directly to the `Extra/` output folder. - * `source_naming_convention`: Defines separator and indices for extracting base name/archetype from source filenames. - * `asset_category_rules`: Keywords/patterns to identify specific asset categories (e.g., "Decal"). - - Use `presets/_template.json` as a starting point for creating new presets. - -## Usage - -### 1. Graphical User Interface (GUI) - -* **Run:** - - ```bash - python -m gui.main_window - ``` - *(Note: Run this command from the project root directory)* -* **Interface:** - * **Menu Bar:** Contains a "View" menu to toggle visibility of the Log Console and enable/disable the detailed file preview. - * **Preset Editor Panel (Left):** - * Optional **Log Console:** A text area at the top displaying application log messages (toggle via View menu). - * **Preset List:** Allows creating, deleting, loading, editing, and saving presets. Select a preset here to load it into the editor tabs below. - * **Preset Editor Tabs:** Edit preset details ("General & Naming", "Mapping & Rules"). - * **Processing Panel (Right):** - * **Preset Selector:** Select the preset to use for *processing* the current queue. - * **Output Directory:** Displays the target output directory. Defaults to the path in `config.py`. Use the "Browse..." button to select a different directory. - * **Drag and Drop Area:** Drag asset ZIP files or folders here to add them to the queue. - * **Preview Table:** Displays information about the assets in the queue. Behavior depends on the "Disable Detailed Preview" option in the View menu: - * **Detailed Preview (Default):** Shows all files found within the dropped assets, their predicted classification status (Mapped, Model, Extra, Unrecognised, Ignored, Error), predicted output name (if applicable), and other details based on the selected *processing* preset. Rows are color-coded by status. - * **Simple View (Preview Disabled):** Shows only the list of top-level input asset paths (ZIPs/folders) added to the queue. - * **Progress Bar:** Shows the overall processing progress. - * **Blender Post-Processing:** A group box containing a checkbox to enable/disable the optional Blender script execution. When enabled, input fields and browse buttons appear to specify the `.blend` files for node group and material creation. These fields default to the paths configured in `config.py`. - * **Options & Controls (Bottom):** - * `Overwrite Existing`: Checkbox to force reprocessing if output already exists. - * `Workers`: Spinbox to set the number of assets to process concurrently. - * `Clear Queue`: Button to remove all assets from the queue and clear the preview. - * `Start Processing`: Button to begin processing all assets in the queue. - * `Cancel`: Button to attempt stopping ongoing processing. - * **Status Bar:** Displays messages about the current state, errors, or completion. - -### 2. Command-Line Interface (CLI) - -* **Run:** - - ```bash - python main.py [OPTIONS] INPUT_PATH [INPUT_PATH ...] - ``` -* **Arguments:** - * `INPUT_PATH`: One or more paths to input ZIP files or folders. - * `-p PRESET`, `--preset PRESET`: (Required) Name of the preset to use (e.g., `Poliigon`). - * `-o OUTPUT_DIR`, `--output-dir OUTPUT_DIR`: Override the `OUTPUT_BASE_DIR` set in `config.py`. - * `-w WORKERS`, `--workers WORKERS`: Number of parallel processes (default: auto-detected based on CPU cores). - * `--overwrite`: Force reprocessing and overwrite existing output. - * `-v`, `--verbose`: Enable detailed DEBUG level logging. - * `--nodegroup-blend NODEGROUP_BLEND`: Path to the .blend file for creating/updating node groups. Overrides `config.py` default. If provided, triggers node group script execution after processing. - * `--materials-blend MATERIALS_BLEND`: Path to the .blend file for creating/updating materials. Overrides `config.py` default. If provided, triggers material script execution after processing. -* **Example:** - - ```bash - python main.py "C:/Downloads/WoodFine001.zip" -p Poliigon -o "G:/Assets/Processed" --workers 4 --overwrite --nodegroup-blend "G:/Blender/Libraries/NodeGroups.blend" --materials-blend "G:/Blender/Libraries/Materials.blend" - ``` - -### 3. Directory Monitor (Automated Processing) - -* **Run:** - - ```bash - python monitor.py - ``` -* **Functionality:** This script continuously monitors a specified input directory for new `.zip` files. When a file matching the expected format `[preset]_filename.zip` appears, it automatically triggers the processing pipeline using the extracted preset name. **Note:** The directory monitor currently does *not* support the optional Blender script execution. This feature is only available via the CLI and GUI. -* **Configuration (Environment Variables):** - * `INPUT_DIR`: Directory to monitor for new ZIP files (default: `/data/input`). - * `OUTPUT_DIR`: Base directory for processed asset output (default: `/data/output`). - * `PROCESSED_DIR`: Directory where successfully processed/skipped source ZIPs are moved (default: `/data/processed`). - * `ERROR_DIR`: Directory where source ZIPs that failed processing are moved (default: `/data/error`). - * `LOG_LEVEL`: Logging verbosity (e.g., `INFO`, `DEBUG`) (default: `INFO`). - * `POLL_INTERVAL`: How often to check the input directory (seconds) (default: `5`). - * `PROCESS_DELAY`: Delay after detecting a file before processing starts (seconds) (default: `2`). - * `NUM_WORKERS`: Number of parallel workers for processing (default: auto-detected). -* **Output:** - * Logs processing activity to the console. - * Processed assets are created in the `OUTPUT_DIR` following the standard structure. - * The original input `.zip` file is moved to `PROCESSED_DIR` on success/skip or `ERROR_DIR` on failure. - - ### 4. Blender Node Group Creation Script (`blenderscripts/create_nodegroups.py`) - * **Purpose:** This script, designed to be run *within* Blender (either manually or triggered by `main.py`/GUI), scans processed assets and creates/updates PBR node groups in the active `.blend` file. - * **Execution:** Typically run via the Asset Processor tool's CLI or GUI after asset processing. Can also be run manually in Blender's Text Editor. - * **Prerequisites (for manual run):** - * A library of assets processed by this tool, located at a known path. - * A Blender file containing two template node groups named exactly `Template_PBRSET` and `Template_PBRTYPE`. - * **Configuration (Inside the script for manual run):** - * `PROCESSED_ASSET_LIBRARY_ROOT`: **Must be updated** within the script to point to the base output directory where the processed supplier folders (e.g., `Poliigon/`) are located. This is overridden by the tool when run via CLI/GUI. - * **Functionality:** Reads metadata, creates/updates node groups, loads textures, sets up nodes, applies metadata-driven settings (aspect ratio, stats, highest resolution), and sets asset previews. Includes an explicit save command at the end. - - ### 5. Blender Material Creation Script (`blenderscripts/create_materials.py`) - * **Purpose:** This script, designed to be run *within* Blender (either manually or triggered by `main.py`/GUI), scans processed assets and creates/updates materials in the active `.blend` file that link to the PBRSET node groups created by `create_nodegroups.py`. - * **Execution:** Typically run via the Asset Processor tool's CLI or GUI after asset processing. Can also be run manually in Blender's Text Editor. - * **Prerequisites (for manual run):** - * A library of assets processed by this tool, located at a known path. - * A `.blend` file containing the PBRSET node groups created by `create_nodegroups.py`. - * A template material in the *current* Blender file named `Template_PBRMaterial` that uses nodes and contains a Group node labeled `PLACEHOLDER_NODE_LABEL`. - * **Configuration (Inside the script for manual run):** - * `PROCESSED_ASSET_LIBRARY_ROOT`: **Must be updated** within the script to point to the base output directory where the processed supplier folders (e.g., `Poliigon/`) are located. This is overridden by the tool when run via CLI/GUI. - * `NODEGROUP_BLEND_FILE_PATH`: **Must be updated** within the script to point to the `.blend` file containing the PBRSET node groups. This is overridden by the tool when run via CLI/GUI. - * `TEMPLATE_MATERIAL_NAME`, `PLACEHOLDER_NODE_LABEL`, `MATERIAL_NAME_PREFIX`, `PBRSET_GROUP_PREFIX`, etc., can be adjusted if needed. - * **Functionality:** Reads metadata, creates/updates materials by copying the template, links the corresponding PBRSET node group from the specified `.blend` file, marks materials as assets, copies tags, sets custom previews, and sets viewport properties based on metadata. Includes an explicit save command at the end. - -## Processing Pipeline (Simplified) - - 1. **Extraction:** Input ZIP/folder contents are extracted/copied to a temporary workspace. - 2. **Classification:** Files are scanned and classified (map, model, extra, ignored) using preset rules. - 3. **Metadata Determination:** Asset name, category, and archetype are determined. - 4. **Skip Check:** If output exists and overwrite is off, processing stops here. - 5. **Map Processing:** Identified maps are loaded, resized, converted (bit depth, format), and saved. Gloss maps are inverted if needed. Stats are calculated. - 6. **Merging:** Channels are merged according to preset rules and saved. - 7. **Metadata Generation:** `metadata.json` is created with all collected information. - 8. **Output Organization:** Processed files are moved to the final structured output directory. - 9. **Cleanup:** The temporary workspace is removed. -10. **Optional Blender Script Execution:** If configured via CLI or GUI, Blender is launched in the background to run `create_nodegroups.py` and `create_materials.py` on specified `.blend` files, using the processed asset output directory as input. - -## Output Structure - -Processed assets are saved to: `///` - -Each asset directory typically contains: - -* Processed texture maps (e.g., `AssetName_Color_4K.png`, `AssetName_NRM_2K.exr`). -* Merged texture maps (e.g., `AssetName_NRMRGH_4K.png`). -* Model files (if present in source). -* `metadata.json`: Detailed information about the asset and processing. -* `Extra/` (subdirectory): Contains source files that were not classified as standard maps or models. This includes files explicitly matched by `move_to_extra_patterns` in the preset (e.g., previews, documentation) as well as any other unrecognised files. - -## Docker - -A `Dockerfile` and `requirements-docker.txt` are provided for building a container image to run the processor in an isolated environment. Build and run using standard Docker commands. \ No newline at end of file diff --git a/Documentation/00_Overview.md b/Documentation/00_Overview.md index a8d56e1..177ecf0 100644 --- a/Documentation/00_Overview.md +++ b/Documentation/00_Overview.md @@ -12,9 +12,9 @@ This documentation strictly excludes details on environment setup, dependency in ## Architecture and Codebase Summary -For developers interested in contributing, the tool's architecture is designed around a **Core Processing Engine** (`asset_processor.py`) that handles the pipeline for single assets, supported by a **Configuration System** (`configuration.py` and `config.py` with `Presets/*.json`). Multiple interfaces are provided: a **Graphical User Interface** (`gui/`), a **Command-Line Interface** (`main.py`), and a **Directory Monitor** (`monitor.py`). Optional **Blender Integration** (`blenderscripts/`) is also included. +For developers interested in contributing, the tool's architecture centers on a **Core Processing Engine** (`processing_engine.py`) which initializes and runs a **Pipeline Orchestrator** (`processing/pipeline/orchestrator.py::PipelineOrchestrator`). This orchestrator executes a defined sequence of **Processing Stages** (located in `processing/pipeline/stages/`) based on a **Hierarchical Rule System** (`rule_structure.py`) and a **Configuration System** (`configuration.py` loading `config/app_settings.json` and `Presets/*.json`). The **Graphical User Interface** (`gui/`) has been significantly refactored: `MainWindow` (`main_window.py`) acts as a coordinator, delegating tasks to specialized widgets (`MainPanelWidget`, `PresetEditorWidget`, `LogConsoleWidget`) and background handlers (`RuleBasedPredictionHandler`, `LLMPredictionHandler`, `LLMInteractionHandler`, `AssetRestructureHandler`). The **Directory Monitor** (`monitor.py`) now processes archives asynchronously using a thread pool and utility functions (`utils/prediction_utils.py`, `utils/workspace_utils.py`). The **Command-Line Interface** entry point (`main.py`) primarily launches the GUI, with core CLI functionality currently non-operational. Optional **Blender Integration** (`blenderscripts/`) remains. A new `utils/` directory houses shared helper functions. -The codebase is organized into key directories and files reflecting these components. The `gui/` directory contains all GUI-related code, `Presets/` holds configuration presets, and `blenderscripts/` contains scripts for Blender interaction. The core logic resides in files like `asset_processor.py`, `configuration.py`, `config.py`, `main.py`, and `monitor.py`. The processing pipeline involves steps such as file classification, map processing, channel merging, and metadata generation. +The codebase reflects this structure. The `gui/` directory contains the refactored UI components, `utils/` holds shared utilities, `processing/pipeline/` contains the orchestrator and individual processing stages, `Presets/` contains JSON presets, and `blenderscripts/` holds Blender scripts. Core logic resides in `processing_engine.py`, `processing/pipeline/orchestrator.py`, `configuration.py`, `rule_structure.py`, `monitor.py`, and `main.py`. The processing pipeline, initiated by `processing_engine.py` and executed by the `PipelineOrchestrator`, relies entirely on the input `SourceRule` and static configuration. Each stage in the pipeline operates on an `AssetProcessingContext` object (`processing/pipeline/asset_context.py`) to perform specific tasks like map processing, channel merging, and metadata generation. ## Table of Contents diff --git a/Documentation/01_User_Guide/04_Configuration_and_Presets.md b/Documentation/01_User_Guide/04_Configuration_and_Presets.md index 16fcc22..0748067 100644 --- a/Documentation/01_User_Guide/04_Configuration_and_Presets.md +++ b/Documentation/01_User_Guide/04_Configuration_and_Presets.md @@ -2,16 +2,60 @@ This document explains how to configure the Asset Processor Tool and use presets. -## Core Settings (`config.py`) +## Application Settings (`config/app_settings.json`) -The tool's behavior is controlled by core settings defined in `config.py`. While primarily for developers, some settings are important for users to be aware of: +The tool's core settings are now stored in `config/app_settings.json`. This JSON file contains the base configuration for the application. -* `OUTPUT_BASE_DIR`: The default root directory where processed assets will be saved. -* `IMAGE_RESOULTIONS`: Defines the target resolutions for processed texture maps (e.g., 4K, 2K). -* `BLENDER_EXECUTABLE_PATH`: The path to your Blender installation, required for optional Blender integration. -* Other settings control aspects like default asset category, filename patterns, map merge rules, and output formats. +The `configuration.py` module is responsible for loading the settings from `app_settings.json` (including loading and saving the JSON content), merging them with the rules from the selected preset file, and providing the base configuration via the `load_base_config()` function. Note that the old `config.py` file has been deleted. -These settings can often be overridden via the GUI or CLI arguments. +The `app_settings.json` file is structured into several key sections, including: +* `FILE_TYPE_DEFINITIONS`: Defines known file types (like different texture maps, models, etc.) and their properties. Each definition now includes a `"standard_type"` key for aliasing to a common type (e.g., "COL" for color maps, "NRM" for normal maps), an `"is_grayscale"` boolean property, and a `"bit_depth_rule"` key specifying how to handle bit depth for this file type. The separate `MAP_BIT_DEPTH_RULES` section has been removed. For users creating or editing presets, it's important to note that internal mapping rules (like `Map_type_Mapping.target_type` within a preset's `FileRule`) now directly use the main keys from these `FILE_TYPE_DEFINITIONS` (e.g., `"MAP_COL"`, `"MAP_RGH"`), not just the `standard_type` aliases. +* `ASSET_TYPE_DEFINITIONS`: Defines known asset types (like Surface, Model, Decal) and their properties. +* `MAP_MERGE_RULES`: Defines how multiple input maps can be merged into a single output map (e.g., combining Normal and Roughness into one). + +### LLM Predictor Settings + +For users who wish to utilize the experimental LLM Predictor feature, the following settings are available in `config/llm_settings.json`: + +* `llm_endpoint_url`: The URL of the LLM API endpoint. For local LLMs like LM Studio or Ollama, this will typically be `http://localhost:/v1`. Consult your LLM server documentation for the exact endpoint. +* `llm_api_key`: The API key required to access the LLM endpoint. Some local LLM servers may not require a key, in which case this can be left empty. +* `llm_model_name`: The name of the specific LLM model to use for prediction. This must match a model available at your specified endpoint. +* `llm_temperature`: Controls the randomness of the LLM's output. Lower values (e.g., 0.1-0.5) make the output more deterministic and focused, while higher values (e.g., 0.6-1.0) make it more creative and varied. For prediction tasks, lower temperatures are generally recommended. +* `llm_request_timeout`: The maximum time (in seconds) to wait for a response from the LLM API. Adjust this based on the performance of your LLM server and the complexity of the requests. + +Note that the `llm_predictor_prompt` and `llm_predictor_examples` settings are also present in `config/llm_settings.json`. These define the instructions and examples provided to the LLM for prediction. While they can be viewed here, they are primarily intended for developer reference and tuning the LLM's behavior, and most users will not need to modify them directly via the file. These settings are editable via the LLM Editor panel in the main GUI when the LLM interpretation mode is selected. + +## Application Preferences (`config/app_settings.json` overrides) + +You can modify user-overridable application settings using the built-in GUI editor. These settings are loaded from `config/app_settings.json` and saved as overrides in `config/user_settings.json`. Access it via the **Edit** -> **Preferences...** menu. + +This editor provides a tabbed interface to view and change various application behaviors. The tabs include: +* **General:** Basic settings like output base directory and temporary file prefix. +* **Output & Naming:** Settings controlling output directory and filename patterns, and how variants are handled. +* **Image Processing:** Settings related to image resolution definitions, compression levels, and format choices. +* **Map Merging:** Configuration for how multiple input maps are combined into single output maps. +* **Postprocess Scripts:** Paths to default Blender files for post-processing. + +Note that this editor focuses on user-specific overrides of core application settings. **Asset Type Definitions, File Type Definitions, and Supplier Settings are managed in a separate Definitions Editor.** + +Any changes made through the Preferences editor require an application restart to take effect. + +*(Ideally, a screenshot of the Application Preferences editor would be included here.)* + +## Definitions Editor (`config/asset_type_definitions.json`, `config/file_type_definitions.json`, `config/suppliers.json`) + +Core application definitions that are separate from general user preferences are managed in the dedicated Definitions Editor. This includes defining known asset types, file types, and configuring settings specific to different suppliers. Access it via the **Edit** -> **Edit Definitions...** menu. + +The editor is organized into three tabs: +* **Asset Type Definitions:** Define the different categories of assets (e.g., Surface, Model, Decal). For each asset type, you can configure its description, a color for UI representation, and example usage strings. +* **File Type Definitions:** Define the specific types of files the tool recognizes (e.g., MAP_COL, MAP_NRM, MODEL). For each file type, you can configure its description, a color, example keywords/patterns, a standard type alias, bit depth handling rules, whether it's grayscale, and an optional keybind for quick assignment in the GUI. +* **Supplier Settings:** Configure settings that are specific to assets originating from different suppliers. Currently, this includes the "Normal Map Type" (OpenGL or DirectX) used for normal maps from that supplier. + +Each tab presents a list of the defined items on the left (Asset Types, File Types, or Suppliers). Selecting an item in the list displays its configurable details on the right. Buttons are provided to add new definitions or remove existing ones. + +Changes made in the Definitions Editor are saved directly to their respective configuration files (`config/asset_type_definitions.json`, `config/file_type_definitions.json`, and `config/suppliers.json`). Some changes may require an application restart to take full effect in processing logic. + +*(Ideally, screenshots of the Definitions Editor tabs would be included here.)* ## Preset Files (`presets/*.json`) @@ -20,8 +64,18 @@ Preset files define supplier-specific rules for interpreting asset source files. * Presets are located in the `presets/` directory. * Each preset is a JSON file named after the supplier (e.g., `Poliigon.json`). * Presets contain rules based on filename patterns and keywords to identify map types, models, and other files. -* They also define how variants (like different resolutions or bit depths) are handled and how asset names and categories are determined from the source filename. +* They also define how variants (like different resolutions or bit depths) are handled and how asset names and categories are determined from the source filename. When defining `map_type_mapping` rules within a preset, the `target_type` field must now use a valid key from the `FILE_TYPE_DEFINITIONS` in `config/app_settings.json` (e.g., `"MAP_AO"` instead of a custom alias like `"AO"`). -When processing assets, you must specify which preset to use. The tool then loads the core settings from `config.py` and merges them with the rules from the selected preset to determine how to process the input. +When processing assets, you must specify which preset to use. The tool then loads the core settings from `config/app_settings.json` and merges them with the rules from the selected preset to determine how to process the input. -A template preset file (`presets/_template.json`) is provided as a base for creating new presets. \ No newline at end of file +A template preset file (`presets/_template.json`) is provided as a base for creating new presets. +## Global Output Path Configuration + +The structure and naming of the output files generated by the tool are now controlled by two global settings defined exclusively in `config/app_settings.json`: + +* `OUTPUT_DIRECTORY_PATTERN`: Defines the directory structure where processed assets will be saved. +* `OUTPUT_FILENAME_PATTERN`: Defines the naming convention for the individual output files within the generated directory. + +**Important:** These settings are global and apply to all processing tasks, regardless of the selected preset. They are **not** part of individual preset files and cannot be modified using the Preset Editor. You can view and edit these patterns in the main application preferences (**Edit** -> **Preferences...**). + +These patterns use special tokens (e.g., `[assetname]`, `[maptype]`) that are replaced with actual values during processing. For a detailed explanation of how these patterns work together, the available tokens, and examples, please refer to the [Output Structure](./09_Output_Structure.md) section of the User Guide. \ No newline at end of file diff --git a/Documentation/01_User_Guide/05_Usage_GUI.md b/Documentation/01_User_Guide/05_Usage_GUI.md index 80bb826..b63be1a 100644 --- a/Documentation/01_User_Guide/05_Usage_GUI.md +++ b/Documentation/01_User_Guide/05_Usage_GUI.md @@ -12,24 +12,51 @@ python -m gui.main_window ## Interface Overview -* **Menu Bar:** The "View" menu allows you to toggle the visibility of the Log Console and the Detailed File Preview. +* **Menu Bar:** The "Edit" menu contains options to configure application settings and definitions: + * **Preferences...:** Opens the Application Preferences editor for user-overridable settings (saved to `config/user_settings.json`). + * **Edit Definitions...:** Opens the Definitions Editor for managing Asset Type Definitions, File Type Definitions, and Supplier Settings (saved to their respective files). +The "View" menu allows you to toggle the visibility of the Log Console and the Detailed File Preview. * **Preset Editor Panel (Left):** * **Optional Log Console:** Displays application logs (toggle via View menu). * **Preset List:** Create, delete, load, edit, and save presets. On startup, the "-- Select a Preset --" item is explicitly selected. You must select a specific preset from this list to load it into the editor below, enable the detailed file preview, and enable the "Start Processing" button. * **Preset Editor Tabs:** Edit the details of the selected preset. * **Processing Panel (Right):** - * **Preset Selector:** Choose the preset to use for *processing* the current queue. - * **Output Directory:** Set the output path (defaults to `config.py`, use "Browse...") + * **Preset Selector:** Choose the preset to use for *processing* the current queue. (Note: LLM interpretation is now initiated via the right-click context menu in the Preview Table). + * **Output Directory:** Set the output path (defaults to `config/app_settings.json`, use "Browse...") * **Drag and Drop Area:** Add asset `.zip`, `.rar`, `.7z` files, or folders by dragging and dropping them here. - * **Preview Table:** Shows queued assets. Initially, this area displays a message prompting you to select a preset. Once a preset is selected from the Preset List, the detailed file preview will load here. The mode of the preview depends on the "View" menu: - * **Detailed Preview (Default):** Lists all files, predicted status (`Mapped`, `Model`, `Extra`, `Unrecognised`, `Ignored`, `Error`), output name, etc., based on the selected *processing* preset. Text colors are applied to cells based on the status of the individual file they represent. Rows use alternating background colors per asset group for visual separation. - * **Simple View (Preview Disabled):** Lists only top-level input asset paths. + * **Preview Table:** Shows queued assets in a hierarchical view (Source -> Asset -> File). Assets (files, directories, archives) added via drag-and-drop appear immediately in the table. This table is interactive: + * **Editable Fields:** The 'Name' field for Assets and the 'Target Asset', 'Supplier', 'Asset Type', and 'Item Type' fields for all items can be edited directly in the table. + * Editing an **Asset Name** automatically updates the 'Target Asset' field for all its child files. + * The **Item Type** field is a text input with auto-suggestions based on available types. + * **Drag-and-Drop Re-parenting:** File rows can be dragged and dropped onto different Asset rows to change their parent asset association. + * **Right-Click Context Menu:** Right-clicking on Source, Asset, or File rows brings up a context menu: + * **Re-interpret selected source:** This sub-menu allows re-running the prediction process for the selected source item(s) using either a specific preset or the LLM predictor. The available presets and the "LLM" option are listed dynamically. This replaces the previous standalone "Re-interpret Selected with LLM" button. +* **Keybinds for Item Management:** When items are selected in the Preview Table, the following keybinds can be used: + * `Ctrl + C`: Sets the file type of selected items to Color/Albedo (`MAP_COL`). + * `Ctrl + R`: Toggles the file type of selected items between Roughness (`MAP_ROUGH`) and Glossiness (`MAP_GLOSS`). + * `Ctrl + N`: Sets the file type of selected items to Normal (`MAP_NRM`). + * `Ctrl + M`: Toggles the file type of selected items between Metalness (`MAP_METAL`) and Reflection/Specular (`MAP_REFL`). + * `Ctrl + D`: Sets the file type of selected items to Displacement/Height (`MAP_DISP`). + * `Ctrl + E`: Sets the file type of selected items to Extra (`EXTRA`). + * `Ctrl + X`: Sets the file type of selected items to Ignore (`FILE_IGNORE`). + * `F2`: Prompts to set the asset name for all selected items. This name propagates to the `AssetRule` name or the `FileRule` `target_asset_name_override` for the files under the selected assets. If individual files are selected, it will affect their `target_asset_name_override`. + * **Prediction Population:** If a valid preset is selected in the Preset Selector (or if re-interpretation is triggered), the table populates with prediction results as they become available. If no preset is selected, added items show empty prediction fields. + * **Columns:** The table displays columns: Name, Target Asset, Supplier, Asset Type, Item Type. The "Target Asset" column stretches to fill available space. + * **Coloring:** The *text color* of file items is determined by their Item Type (colors defined in `config/app_settings.json`). The *background color* of file items is a 30% darker shade of their parent asset's background, helping to visually group files within an asset. Asset rows themselves may use alternating background colors based on the application theme. * **Progress Bar:** Shows overall processing progress. -* **Blender Post-Processing:** Checkbox to enable Blender scripts. If enabled, shows fields and browse buttons for target `.blend` files (defaults from `config.py`). +* **Blender Post-Processing:** Checkbox to enable Blender scripts. If enabled, shows fields and browse buttons for target `.blend` files (defaults from `config/app_settings.json`). * **Options & Controls (Bottom):** * `Overwrite Existing`: Checkbox to force reprocessing. * `Workers`: Spinbox for concurrent processes. * `Clear Queue`: Button to clear the queue and preview. - * `Start Processing`: Button to start processing the queue. This button is disabled until a valid preset is selected from the Preset List. + * `Start Processing`: Button to start processing the queue. This button is enabled as long as there are items listed in the Preview Table. When clicked, any items that do not have a value assigned in the "Target Asset" column will be automatically ignored for that processing run. * `Cancel`: Button to attempt stopping processing. -* **Status Bar:** Displays current status, errors, and completion messages. \ No newline at end of file +* **Status Bar:** Displays current status, errors, and completion messages. During LLM processing, the status bar will show messages indicating the progress of the LLM requests. + +## GUI Configuration Editor + +Access the GUI Configuration Editor via the **Edit** -> **Preferences...** menu. This dialog allows you to directly edit the `config/app_settings.json` file, which contains the core application settings. The editor uses a tabbed layout (e.g., "General", "Output & Naming") to organize settings. + +Any changes made in the GUI Configuration Editor require you to restart the application for them to take effect. + +*(Ideally, a screenshot of the GUI Configuration Editor would be included here.)* \ No newline at end of file diff --git a/Documentation/01_User_Guide/09_Output_Structure.md b/Documentation/01_User_Guide/09_Output_Structure.md index ce703ae..a45a4b1 100644 --- a/Documentation/01_User_Guide/09_Output_Structure.md +++ b/Documentation/01_User_Guide/09_Output_Structure.md @@ -2,20 +2,65 @@ This document describes the directory structure and contents of the processed assets generated by the Asset Processor Tool. -Processed assets are saved to: `///` +Processed assets are saved to a location determined by two global settings, `OUTPUT_DIRECTORY_PATTERN` and `OUTPUT_FILENAME_PATTERN`, defined in `config/app_settings.json`. These settings can be overridden by the user via `config/user_settings.json`. -* ``: The base output directory configured in `config.py` or specified via CLI/GUI. -* ``: The name of the asset supplier, determined from the preset used. -* ``: The name of the processed asset, determined from the source filename based on preset rules. +* `OUTPUT_DIRECTORY_PATTERN`: Defines the directory structure *within* the Base Output Directory. +* `OUTPUT_FILENAME_PATTERN`: Defines the naming convention for individual files *within* the directory created by `OUTPUT_DIRECTORY_PATTERN`. + +These patterns use special tokens (explained below) that are replaced with actual values during processing. You can configure these patterns via the main application preferences (**Edit** -> **Preferences...** -> **Output & Naming** tab). They are global settings and are not part of individual presets. + +### Available Tokens + +The following tokens can be used in both `OUTPUT_DIRECTORY_PATTERN` and `OUTPUT_FILENAME_PATTERN`. Note that some tokens make more sense in one pattern than the other (e.g., `[maptype]` and `[ext]` are typically used in the filename pattern). + +* `[Assettype]`: The type of asset (e.g., `Texture`, `Model`, `Surface`). +* `[supplier]`: The supplier name (from the preset, e.g., `Poliigon`). +* `[assetname]`: The main asset name (e.g., `RustyMetalPanel`). +* `[resolution]`: Texture resolution (e.g., `1k`, `2k`, `4k`). +* `[ext]`: The output file extension (e.g., `png`, `jpg`, `exr`). (Primarily for filename pattern) +* `[IncrementingValue]` or `[####]`: A numerical value that increments based on existing directories matching the `OUTPUT_DIRECTORY_PATTERN` in the output base path. The number of `#` characters determines the zero-padding (e.g., `[###]` -> `001`, `002`). If `[IncrementingValue]` is used, it defaults to 4 digits of padding (`0001`, `0002`). +* `[Date]`: Current date (`YYYYMMDD`). +* `[Time]`: Current time (`HHMMSS`). +* `[Sha5]`: The first 5 characters of the SHA-256 hash of the original input source file (e.g., the source zip archive). +* `[ApplicationPath]`: Absolute path to the application directory. +* `[maptype]`: The standardized map type identifier (e.g., `COL` for Color/Albedo, `NRM` for Normal, `RGH` for Roughness). This is derived from the `standard_type` defined in the application's `FILE_TYPE_DEFINITIONS` (managed in `config/file_type_definitions.json` via the Definitions Editor) and may include a variant suffix if applicable. (Primarily for filename pattern) +* `[dimensions]`: Pixel dimensions (e.g., `2048x2048`). +* `[bitdepth]`: Output bit depth (e.g., `8bit`, `16bit`). +* `[category]`: Asset category determined by preset rules. +* `[archetype]`: Asset archetype determined by preset rules. +* `[variant]`: Asset variant identifier determined by preset rules. +* `[source_filename]`: The original filename of the source file being processed. +* `[source_basename]`: The original filename without the extension. +* `[source_dirname]`: The directory containing the original source file. + +### Example Output Paths + +The final output path is constructed by combining the Base Output Directory (set in Preferences or via CLI) with the results of the two patterns. + +**Example 1:** + +* Base Output Directory: `/home/user/ProcessedAssets` +* `OUTPUT_DIRECTORY_PATTERN`: `[supplier]/[assetname]/[resolution]` +* `OUTPUT_FILENAME_PATTERN`: `[assetname]_[maptype]_[resolution].[ext]` +* Resulting Path for an Albedo map: `/home/user/ProcessedAssets/Poliigon/WoodFloor001/4k/WoodFloor001_Albedo_4k.png` + +**Example 2:** + +* Base Output Directory: `Output` (relative path) +* `OUTPUT_DIRECTORY_PATTERN`: `[Assettype]/[category]/[assetname]` +* `OUTPUT_FILENAME_PATTERN`: `[maptype].[ext]` +* Resulting Path for a Normal map: `Output/Texture/Wood/WoodFloor001/Normal.exr` + +The `` (the root folder where processing output starts) is configured separately via the GUI (**Edit** -> **Preferences...** -> **General** tab -> **Output Base Directory**) or the `--output` CLI argument. The `OUTPUT_DIRECTORY_PATTERN` defines the structure *within* this base directory, and `OUTPUT_FILENAME_PATTERN` defines the filenames within that structure. ## Contents of Each Asset Directory Each asset directory contains the following: -* Processed texture maps (e.g., `AssetName_Color_4K.png`, `AssetName_NRM_2K.exr`). These are the resized, format-converted, and bit-depth adjusted texture files. -* Merged texture maps (e.g., `AssetName_NRMRGH_4K.png`). These are maps created by combining channels from different source maps based on the configured merge rules. +* Processed texture maps (e.g., `WoodFloor_Albedo_4k.png`, `MetalPanel_Normal_2k.exr`). The exact filenames depend on the `OUTPUT_FILENAME_PATTERN`. These are the resized, format-converted, and bit-depth adjusted texture files. +* Merged texture maps (e.g., `WoodFloor_Combined_4k.png`). The exact filenames depend on the `OUTPUT_FILENAME_PATTERN`. These are maps created by combining channels from different source maps based on the configured merge rules. * Model files (if present in the source asset). -* `metadata.json`: A JSON file containing detailed information about the asset and the processing that was performed. This includes details about the maps, resolutions, formats, bit depths, merged map details, calculated image statistics, aspect ratio change information, asset category and archetype, the source preset used, and a list of ignored source files. This file is intended for use by downstream tools or scripts (like the Blender integration scripts). -* `Extra/` (subdirectory): Contains source files that were not classified as maps or models but were explicitly marked to be moved to the extra directory based on preset rules (e.g., previews, documentation files). +* `metadata.json`: A JSON file containing detailed information about the asset and the processing that was performed. This includes details about the maps (resolutions, formats, bit depths, and for roughness maps, a `derived_from_gloss_filename: true` flag if it was inverted from an original gloss map), merged map details, calculated image statistics, aspect ratio change information, asset category and archetype, the source preset used, and a list of ignored source files. This file is intended for use by downstream tools or scripts (like the Blender integration scripts). +* `EXTRA/` (subdirectory): Contains source files not classified as maps or models but marked as "EXTRA" by preset rules (e.g., previews, documentation). These files are placed in an `EXTRA` folder *within* the directory generated by `OUTPUT_DIRECTORY_PATTERN`. * `Unrecognised/` (subdirectory): Contains source files that were not classified as maps, models, or explicitly marked as extra, and were not ignored. * `Ignored/` (subdirectory): Contains source files that were explicitly ignored during processing (e.g., an 8-bit Normal map when a 16-bit variant exists and is prioritized). \ No newline at end of file diff --git a/Documentation/01_User_Guide/11_Usage_Autotest.md b/Documentation/01_User_Guide/11_Usage_Autotest.md new file mode 100644 index 0000000..d4ee010 --- /dev/null +++ b/Documentation/01_User_Guide/11_Usage_Autotest.md @@ -0,0 +1,83 @@ +# User Guide: Usage - Automated GUI Testing (`autotest.py`) + +This document explains how to use the `autotest.py` script for automated sanity checks of the Asset Processor Tool's GUI-driven workflow. + +## Overview + +The `autotest.py` script provides a way to run predefined test scenarios headlessly (without displaying the GUI). It simulates the core user actions: loading an asset, selecting a preset, allowing rules to be predicted, processing the asset, and then checks the results against expectations. This is primarily intended as a developer tool for regression testing and ensuring core functionality remains stable. + +## Running the Autotest Script + +From the project root directory, you can run the script using Python: + +```bash +python autotest.py [OPTIONS] +``` + +### Command-Line Options + +The script accepts several command-line arguments to configure the test run. If not provided, they use predefined default values. + +* `--zipfile PATH_TO_ZIP`: + * Specifies the path to the input asset `.zip` file to be used for the test. + * Default: `TestFiles/BoucleChunky001.zip` +* `--preset PRESET_NAME`: + * Specifies the name of the preset to be selected and used for rule prediction and processing. + * Default: `Dinesen` +* `--expectedrules PATH_TO_JSON`: + * Specifies the path to a JSON file containing the expected rule structure that should be generated after the preset is applied to the input asset. + * Default: `TestFiles/test-BoucleChunky001.json` +* `--outputdir PATH_TO_DIR`: + * Specifies the directory where the processed assets will be written. + * Default: `TestFiles/TestOutputs/DefaultTestOutput` +* `--search "SEARCH_TERM"` (optional): + * A string to search for within the application logs generated during the test run. If found, matching log lines (with context) will be highlighted. + * Default: None +* `--additional-lines NUM_LINES` (optional): + * When using `--search`, this specifies how many lines of context before and after each matching log line should be displayed. + * Default: `0` + +**Example Usage:** + +```bash +# Run with default test files and settings +python autotest.py + +# Run with specific test files and search for a log message +python autotest.py --zipfile TestFiles/MySpecificAsset.zip --preset MyPreset --expectedrules TestFiles/MySpecificAsset_rules.json --outputdir TestFiles/TestOutputs/MySpecificOutput --search "Processing complete for asset" +``` + +## `TestFiles` Directory + +The autotest script relies on a directory named `TestFiles` located in the project root. This directory should contain: + +* **Test Asset `.zip` files:** The actual asset archives used as input for tests (e.g., `default_test_asset.zip`, `MySpecificAsset.zip`). +* **Expected Rules `.json` files:** JSON files defining the expected rule structure for a given asset and preset combination (e.g., `default_test_asset_rules.json`, `MySpecificAsset_rules.json`). The structure of this file is detailed in the main autotest plan (`AUTOTEST_GUI_PLAN.md`). +* **`TestOutputs/` subdirectory:** This is the default parent directory where the autotest script will create specific output folders for each test run (e.g., `TestFiles/TestOutputs/DefaultTestOutput/`). + +## Test Workflow + +When executed, `autotest.py` performs the following steps: + +1. **Initialization:** Parses command-line arguments and initializes the main application components headlessly. +2. **Load Expected Rules:** Loads the `expected_rules.json` file. +3. **Load Asset:** Loads the specified `.zip` file into the application. +4. **Select Preset:** Selects the specified preset. This triggers the internal rule prediction process. +5. **Await Prediction:** Waits for the rule prediction to complete. +6. **Compare Rules:** Retrieves the predicted rules from the application and compares them against the loaded expected rules. If there's a mismatch, the test typically fails at this point. +7. **Start Processing:** If the rules match, it initiates the asset processing pipeline, directing output to the specified output directory. +8. **Await Processing:** Waits for all backend processing tasks to complete. +9. **Check Output:** Verifies the existence of the output directory and lists its contents. Basic checks ensure some output was generated. +10. **Analyze Logs:** Retrieves logs from the application. If a search term was provided, it filters and displays relevant log portions. It also checks for Python tracebacks, which usually indicate a failure. +11. **Report Result:** Prints a summary of the test outcome (success or failure) and exits with an appropriate status code (0 for success, 1 for failure). + +## Interpreting Results + +* **Console Output:** The script will log its progress and the results of each step to the console. +* **Log Analysis:** Pay attention to the log output, especially if a `--search` term was used or if any tracebacks are reported. +* **Exit Code:** + * `0`: Test completed successfully. + * `1`: Test failed at some point (e.g., rule mismatch, processing error, traceback found). +* **Output Directory:** Inspect the contents of the specified output directory to manually verify the processed assets if needed. + +This automated test helps ensure the stability of the core processing logic when driven by GUI-equivalent actions. \ No newline at end of file diff --git a/Documentation/02_Developer_Guide/01_Architecture.md b/Documentation/02_Developer_Guide/01_Architecture.md index 911fba3..24d888d 100644 --- a/Documentation/02_Developer_Guide/01_Architecture.md +++ b/Documentation/02_Developer_Guide/01_Architecture.md @@ -6,41 +6,76 @@ This document provides a high-level overview of the Asset Processor Tool's archi The Asset Processor Tool is designed to process 3D asset source files into a standardized library format. Its high-level architecture consists of: -1. **Core Processing Engine (`AssetProcessor`):** The central component responsible for orchestrating the asset processing pipeline for a single input asset. -2. **Configuration System (`Configuration`):** Handles loading core settings and merging them with supplier-specific rules defined in JSON presets. -3. **Multiple Interfaces:** Provides different ways to interact with the tool: +1. **Core Processing Initiation (`processing_engine.py`):** The `ProcessingEngine` class acts as the entry point for an asset processing task. It initializes and runs a `PipelineOrchestrator`. +2. **Pipeline Orchestration (`processing/pipeline/orchestrator.py`):** The `PipelineOrchestrator` manages a sequence of discrete processing stages. It creates an `AssetProcessingContext` for each asset and passes this context through each stage. +3. **Processing Stages (`processing/pipeline/stages/`):** Individual modules, each responsible for a specific task in the pipeline (e.g., filtering files, processing maps, merging channels, organizing output). They operate on the `AssetProcessingContext`. +4. **Prediction System:** Responsible for analyzing input files and generating the initial `SourceRule` hierarchy with predicted values. This system utilizes a base handler (`gui/base_prediction_handler.py::BasePredictionHandler`) with specific implementations: + * **Rule-Based Predictor (`gui/prediction_handler.py::RuleBasedPredictionHandler`):** Uses predefined rules from presets to classify files and determine initial processing parameters. + * **LLM Predictor (`gui/llm_prediction_handler.py::LLMPredictionHandler`):** An experimental alternative that uses a Large Language Model (LLM) to interpret file contents and context to predict processing parameters. +5. **Configuration System (`Configuration`):** Handles loading core settings (including centralized type definitions and LLM-specific configuration) and merging them with supplier-specific rules defined in JSON presets and the persistent `config/suppliers.json` file. +6. **Multiple Interfaces:** Provides different ways to interact with the tool: * Graphical User Interface (GUI) - * Command-Line Interface (CLI) + * Command-Line Interface (CLI) - *Note: The primary CLI execution logic (`run_cli` in `main.py`) is currently non-functional/commented out post-refactoring.* * Directory Monitor for automated processing. -These interfaces exchange data structures containing information about each file and asset set being processed. -4. **Optional Integration:** Includes scripts and logic for integrating with external software, specifically Blender, to automate material and node group creation. +The GUI acts as the primary source of truth for processing rules, coordinating the generation and management of the `SourceRule` hierarchy before sending it to the `ProcessingEngine`. It accumulates prediction results from multiple input sources before updating the view. The Monitor interface can also generate `SourceRule` objects (using `utils/prediction_utils.py`) to bypass the GUI for automated workflows. +7. **Optional Integration:** Includes scripts (`blenderscripts/`) for integrating with Blender. Logic for executing these scripts was intended to be centralized in `utils/blender_utils.py`, but this utility has not yet been implemented. + +## Hierarchical Rule System + +A key addition to the architecture is the **Hierarchical Rule System**, which provides a dynamic layer of configuration that can override the static settings loaded from presets. This system is represented by a hierarchy of rule objects: `SourceRule`, `AssetRule`, and `FileRule`. + +* **SourceRule:** Represents rules applied at the top level, typically corresponding to an entire input source (e.g., a ZIP file or folder). +* **AssetRule:** Represents rules applied to a specific asset within a source (a source can contain multiple assets). +* **FileRule:** Represents rules applied to individual files within an asset. + +This hierarchy allows for fine-grained control over processing parameters. The GUI's prediction logic generates this hierarchy with initial predicted values for overridable fields based on presets and file analysis. The `ProcessingEngine` (via the `PipelineOrchestrator` and its stages) then operates *solely* on the explicit values provided in this `SourceRule` object and static configuration, without internal prediction or fallback logic. ## Core Components -* `config.py`: Defines core, global settings and constants. -* `Presets/*.json`: Supplier-specific JSON files defining rules for file interpretation and processing. -* `configuration.py` (`Configuration` class): Loads `config.py` settings and merges them with a selected preset, pre-compiling regex patterns for efficiency. -* `asset_processor.py` (`AssetProcessor` class): Contains the core logic for processing a *single* asset through the defined pipeline steps. This component works with data structures containing detailed information about the asset and its files. -* `main.py`: The entry point for the Command-Line Interface (CLI). It handles argument parsing, logging, parallel processing orchestration, and triggering Blender scripts. It orchestrates the processing of multiple assets by interacting with the `AssetProcessor` for individual assets and manages the overall CLI execution flow. -* `gui/`: Directory containing modules for the Graphical User Interface (GUI), built with PySide6. The GUI interacts with the core processing logic indirectly via dedicated handler classes (`ProcessingHandler`, `PredictionHandler`) running in separate threads. Data structures, such as input paths, configuration details, processing progress updates, and file prediction results, are passed between the GUI, these handlers, and the `AssetProcessor` using thread-safe mechanisms like Qt signals and slots. -* `monitor.py`: Implements the directory monitoring feature using `watchdog`. +* `config/app_settings.json`: Defines core, global settings, constants, and centralized definitions for allowed asset and file types (`ASSET_TYPE_DEFINITIONS`, `FILE_TYPE_DEFINITIONS`), including metadata like colors and descriptions. This replaces the old `config.py` file. +* `config/suppliers.json`: A persistent JSON file storing known supplier names for GUI auto-completion. +* `Presets/*.json`: Supplier-specific JSON files defining rules for file interpretation and initial prediction. +* `configuration.py` (`Configuration` class): Loads `config/app_settings.json` settings and merges them with a selected preset, pre-compiling regex patterns for efficiency. This static configuration is used by the processing pipeline. +* `rule_structure.py`: Defines the `SourceRule`, `AssetRule`, and `FileRule` dataclasses used to represent the hierarchical processing rules. +* `gui/`: Directory containing modules for the Graphical User Interface (GUI), built with PySide6. The `MainWindow` (`main_window.py`) acts as a coordinator, orchestrating interactions between various components. Key GUI components include: + * `main_panel_widget.py::MainPanelWidget`: Contains the primary controls for loading sources, selecting presets, viewing/editing rules, and initiating processing. + * `preset_editor_widget.py::PresetEditorWidget`: Provides the interface for managing presets. + * `log_console_widget.py::LogConsoleWidget`: Displays application logs. + * `unified_view_model.py::UnifiedViewModel`: Implements the `QAbstractItemModel` for the hierarchical rule view, holding `SourceRule` data and managing display logic (coloring, etc.). Caches configuration data for performance. + * `rule_hierarchy_model.py::RuleHierarchyModel`: A simpler model used internally by the `UnifiedViewModel` to manage the `SourceRule` data structure. + * `delegates.py`: Contains custom `QStyledItemDelegate` implementations for inline editing in the rule view. + * `asset_restructure_handler.py::AssetRestructureHandler`: Handles complex model updates when a file's target asset is changed via the GUI, ensuring the `SourceRule` hierarchy is correctly modified. + * `base_prediction_handler.py::BasePredictionHandler`: Abstract base class for prediction logic. + * `prediction_handler.py::RuleBasedPredictionHandler`: Generates the initial `SourceRule` hierarchy based on presets and file analysis. Inherits from `BasePredictionHandler`. + * `llm_prediction_handler.py::LLMPredictionHandler`: Experimental predictor using an LLM. Inherits from `BasePredictionHandler`. + * `llm_interaction_handler.py::LLMInteractionHandler`: Manages communication with the LLM service for the LLM predictor. +* `processing_engine.py` (`ProcessingEngine` class): The entry-point class that initializes and runs the `PipelineOrchestrator` for a given `SourceRule` and `Configuration`. +* `processing/pipeline/orchestrator.py` (`PipelineOrchestrator` class): Manages the sequence of processing stages, creating and passing an `AssetProcessingContext` through them. +* `processing/pipeline/asset_context.py` (`AssetProcessingContext` class): A dataclass holding all data and state for the processing of a single asset, passed between stages. +* `processing/pipeline/stages/`: Directory containing individual processing stage modules, each handling a specific part of the pipeline (e.g., `IndividualMapProcessingStage`, `MapMergingStage`). +* `main.py`: The main entry point for the application. Primarily launches the GUI. Contains commented-out/non-functional CLI logic (`run_cli`). +* `monitor.py`: Implements the directory monitoring feature using `watchdog`. It now processes archives asynchronously using a `ThreadPoolExecutor`, leveraging `utils.prediction_utils.py` for rule generation and `utils.workspace_utils.py` for workspace management before invoking the `ProcessingEngine`. * `blenderscripts/`: Contains Python scripts designed to be executed *within* Blender for post-processing tasks. +* `utils/`: Directory containing utility modules: + * `workspace_utils.py`: Contains functions like `prepare_processing_workspace` for handling temporary directories and archive extraction. + * `prediction_utils.py`: Contains functions like `generate_source_rule_from_archive` used by the monitor for rule-based prediction. + * `blender_utils.py`: (Intended location for Blender script execution logic, currently not implemented). -## Processing Pipeline (Simplified) +## Processing Pipeline (Simplified Overview) -The core processing engine (`AssetProcessor`) executes a series of steps for each asset: +The asset processing pipeline, initiated by `processing_engine.py` and managed by `PipelineOrchestrator`, executes a series of stages for each asset defined in the `SourceRule`. An `AssetProcessingContext` object carries data between stages. The typical sequence is: -1. Extraction of input to a temporary workspace. -2. Classification of files (map, model, extra, ignored, unrecognised) using preset rules. -3. Determination of base metadata (asset name, category, archetype). -4. Skip check if output exists and overwrite is not forced. -5. Processing of maps (resize, format/bit depth conversion, inversion, stats calculation). -6. Merging of channels based on rules. -7. Generation of `metadata.json` file. -8. Organization of processed files into the final output structure. -9. Cleanup of the temporary workspace. -10. (Optional) Execution of Blender scripts for post-processing. +1. **Supplier Determination**: Identify the effective supplier. +2. **Asset Skip Logic**: Check if the asset should be skipped. +3. **Metadata Initialization**: Set up initial asset metadata. +4. **File Rule Filtering**: Determine which files to process. +5. **Pre-Map Processing**: + * Gloss-to-Roughness Conversion. + * Alpha Channel Extraction. + * Normal Map Green Channel Inversion. +6. **Individual Map Processing**: Handle individual maps (scaling, variants, stats, naming). +7. **Map Merging**: Combine channels from different maps. +8. **Metadata Finalization & Save**: Generate and save `metadata.json` (temporarily). +9. **Output Organization**: Copy all processed files to final output locations. -This architecture allows for a modular design, separating configuration, core processing logic, and different interfaces. The data structures flowing through this pipeline carry detailed information about individual files and asset sets. Examples include lists of input file paths, configuration objects, dictionaries summarizing processing outcomes, and detailed lists of file predictions. Parallel processing is utilized for efficiency, and background threads keep the GUI responsive. - -**Note on Data Passing:** Major changes to the data passing mechanisms between the GUI, Main (CLI orchestration), and `AssetProcessor` modules are currently being planned. These changes are expected to involve new data structures and updated interaction patterns to convey detailed specifications for datasets/asset-sets and processing instructions for individual files. The documentation in this section, particularly regarding data flow, will require significant review and updates once the plan for these changes is finalized. \ No newline at end of file +External steps like workspace preparation/cleanup and optional Blender script execution bracket this core pipeline. This architecture allows for a modular design, separating configuration, rule generation/management, and core processing execution. \ No newline at end of file diff --git a/Documentation/02_Developer_Guide/02_Codebase_Structure.md b/Documentation/02_Developer_Guide/02_Codebase_Structure.md index 3cdabac..a969bee 100644 --- a/Documentation/02_Developer_Guide/02_Codebase_Structure.md +++ b/Documentation/02_Developer_Guide/02_Codebase_Structure.md @@ -4,61 +4,90 @@ This document outlines the key files and directories within the Asset Processor ``` Asset_processor_tool/ -├── asset_processor.py # Core class handling single asset processing pipeline -├── config.py # Core settings definition (output paths, resolutions, merge rules etc.) -├── configuration.py # Class for loading and accessing configuration (merges config.py and presets) -├── detailed_documentation_plan.md # (Existing file, potentially outdated) +├── configuration.py # Class for loading and accessing configuration (merges app_settings.json and presets) ├── Dockerfile # Instructions for building the Docker container image -├── documentation_plan.md # Plan for the new documentation structure (this plan) -├── documentation.txt # Original developer documentation (to be migrated) -├── main.py # CLI Entry Point & processing orchestrator -├── monitor.py # Directory monitoring script for automated processing -├── readme.md # Original main documentation file (to be migrated) -├── readme.md.bak # Backup of readme.md +├── main.py # Main application entry point (primarily GUI launcher) +├── monitor.py # Directory monitoring script for automated processing (async) +├── processing_engine.py # Core class handling single asset processing based on SourceRule ├── requirements-docker.txt # Dependencies specifically for the Docker environment ├── requirements.txt # Python package dependencies for standard execution +├── rule_structure.py # Dataclasses for hierarchical rules (SourceRule, AssetRule, FileRule) ├── blenderscripts/ # Scripts for integration with Blender │ ├── create_materials.py # Script to create materials linking to node groups │ └── create_nodegroups.py # Script to create node groups from processed assets -├── Deprecated-POC/ # Directory containing original proof of concept scripts -│ ├── Blender-MaterialsFromNodegroups.py -│ ├── Blender-NodegroupsFromPBRSETS.py -│ └── Standalonebatcher-Main.py -├── Documentation/ # New directory for organized documentation (this structure) +├── config/ # Directory for configuration files +│ ├── app_settings.json # Core settings, constants, and type definitions +│ └── suppliers.json # Persistent list of known supplier names for GUI auto-completion +├── Deprecated/ # Contains old code, documentation, and POC scripts +│ ├── ... +├── Documentation/ # Directory for organized documentation (this structure) │ ├── 00_Overview.md │ ├── 01_User_Guide/ │ └── 02_Developer_Guide/ -├── gui/ # Contains files related to the Graphical User Interface -│ ├── main_window.py # Main GUI application window and layout -│ ├── processing_handler.py # Handles background processing logic for the GUI -│ ├── prediction_handler.py # Handles background file prediction/preview for the GUI -│ ├── preview_table_model.py # Model and proxy for the GUI's preview table -│ └── ... # Other GUI components -├── Presets/ # Preset definition files +├── gui/ # Contains files related to the Graphical User Interface (PySide6) +│ ├── asset_restructure_handler.py # Handles model updates for target asset changes +│ ├── base_prediction_handler.py # Abstract base class for prediction logic +│ ├── config_editor_dialog.py # Dialog for editing configuration files +│ ├── delegates.py # Custom delegates for inline editing in rule view +│ ├── llm_interaction_handler.py # Manages communication with LLM service +│ ├── llm_prediction_handler.py # LLM-based prediction handler +│ ├── log_console_widget.py # Widget for displaying logs +│ ├── main_panel_widget.py # Main panel containing core GUI controls +│ ├── main_window.py # Main GUI application window (coordinator) +│ ├── prediction_handler.py # Rule-based prediction handler +│ ├── preset_editor_widget.py # Widget for managing presets +│ ├── preview_table_model.py # Model for the (deprecated?) preview table +│ ├── rule_editor_widget.py # Widget containing the rule hierarchy view and editor +│ ├── rule_hierarchy_model.py # Internal model for rule hierarchy data +│ └── unified_view_model.py # QAbstractItemModel for the rule hierarchy view +├── llm_prototype/ # Files related to the experimental LLM predictor prototype +│ ├── ... +├── Presets/ # Preset definition files (JSON) │ ├── _template.json # Template for creating new presets │ ├── Poliigon.json # Example preset for Poliigon assets │ └── ... # Other presets -├── Project Notes/ # Directory for issue and feature tracking (Markdown files) -│ ├── ... # Various planning and note files -└── Testfiles/ # Directory containing example input assets for testing - └── ... # Example asset ZIPs +├── ProjectNotes/ # Directory for developer notes, plans, etc. (Markdown files) +│ ├── ... +├── PythonCheatsheats/ # Utility Python reference files +│ ├── ... +├── Testfiles/ # Directory containing example input assets for testing +│ ├── ... +├── Tickets/ # Directory for issue and feature tracking (Markdown files) +│ ├── ... +└── utils/ # Utility modules shared across the application + ├── prediction_utils.py # Utilities for prediction (e.g., used by monitor) + └── workspace_utils.py # Utilities for managing processing workspaces ``` **Key Files and Directories:** -* `asset_processor.py`: Contains the `AssetProcessor` class, the core logic for processing a single asset through the pipeline. Includes methods for classification, map processing, merging, metadata generation, and output organization. Also provides methods for predicting output structure used by the GUI. -* `configuration.py`: Defines the `Configuration` class. Responsible for loading core settings from `config.py` and merging them with a specified preset JSON file (`Presets/*.json`). Pre-compiles regex patterns from presets for efficiency. -* `config.py`: Stores global default settings, constants, and core rules (e.g., standard map types, default resolutions, merge rules, output format rules, Blender paths). -* `main.py`: Entry point for the Command-Line Interface (CLI). Handles argument parsing, logging setup, parallel processing orchestration (using `concurrent.futures.ProcessPoolExecutor`), calls `AssetProcessor` via a wrapper function, and optionally triggers Blender scripts. -* `monitor.py`: Implements the automated directory monitoring feature using the `watchdog` library. Contains the `ZipHandler` class to detect new ZIP files and trigger processing via `main.run_processing`. -* `gui/`: Directory containing all code related to the Graphical User Interface (GUI), built with PySide6. - * `main_window.py`: Defines the `MainWindow` class, the main application window structure, UI layout, event handling, and menu setup. Manages GUI-specific logging (`QtLogHandler`). - * `processing_handler.py`: Defines the `ProcessingHandler` class (runs on a `QThread`). Manages the execution of the main asset processing pipeline and Blender script execution in the background. - * `prediction_handler.py`: Defines the `PredictionHandler` class (runs on a `QThread`). Manages background file analysis/preview generation. - * `preview_table_model.py`: Defines `PreviewTableModel` and `PreviewSortFilterProxyModel` for managing and displaying data in the GUI's preview table. +* `config/`: Directory containing configuration files. + * `app_settings.json`: Stores global default settings, constants, core rules, and centralized definitions for allowed asset and file types (`ASSET_TYPE_DEFINITIONS`, `FILE_TYPE_DEFINITIONS`) used for validation, GUI elements, and coloring. Replaces the old `config.py`. + * `suppliers.json`: A JSON file storing a persistent list of known supplier names, used by the GUI for auto-completion. +* `configuration.py`: Defines the `Configuration` class. Responsible for loading core settings from `config/app_settings.json` and merging them with a specified preset JSON file (`Presets/*.json`). Pre-compiles regex patterns from presets for efficiency. An instance of this class is passed to the `ProcessingEngine`. +* `rule_structure.py`: Defines the `SourceRule`, `AssetRule`, and `FileRule` dataclasses. These structures represent the hierarchical processing rules and are the primary data contract passed from the rule generation layer (GUI, Monitor) to the processing engine. +* `processing_engine.py`: Defines the `ProcessingEngine` class. This is the core component that executes the processing pipeline for a single asset based *solely* on a provided `SourceRule` object and the static `Configuration`. It contains no internal prediction or fallback logic. +* `main.py`: Main entry point for the application. Primarily responsible for initializing and launching the GUI (`gui.main_window.MainWindow`). Contains non-functional/commented-out CLI logic (`run_cli`). +* `monitor.py`: Implements the automated directory monitoring feature using `watchdog`. It now processes detected archives asynchronously using a `ThreadPoolExecutor`. It utilizes `utils.prediction_utils.generate_source_rule_from_archive` for rule-based prediction and `utils.workspace_utils.prepare_processing_workspace` for workspace setup before invoking the `ProcessingEngine`. +* `gui/`: Directory containing all code related to the Graphical User Interface (GUI), built with PySide6. The `MainWindow` acts as a coordinator, delegating functionality to specialized widgets and handlers. + * `main_window.py`: Defines the `MainWindow` class. Acts as the main application window and coordinator, connecting signals and slots between different GUI components. + * `main_panel_widget.py`: Defines `MainPanelWidget`, containing the primary user controls (source loading, preset selection, rule view/editor integration, processing buttons). + * `preset_editor_widget.py`: Defines `PresetEditorWidget` for managing presets (loading, saving, editing). + * `log_console_widget.py`: Defines `LogConsoleWidget` for displaying application logs within the GUI. + * `rule_editor_widget.py`: Defines `RuleEditorWidget`, which houses the `QTreeView` for displaying the rule hierarchy. + * `unified_view_model.py`: Defines `UnifiedViewModel` (`QAbstractItemModel`) for the rule hierarchy view. Holds `SourceRule` data, manages display logic (coloring), handles inline editing requests, and caches configuration data for performance. + * `rule_hierarchy_model.py`: Defines `RuleHierarchyModel`, a simpler internal model used by `UnifiedViewModel` to manage the underlying `SourceRule` data structure. + * `delegates.py`: Contains custom `QStyledItemDelegate` implementations used by the `UnifiedViewModel` to provide appropriate inline editors (e.g., dropdowns, text boxes) for different rule attributes. + * `asset_restructure_handler.py`: Defines `AssetRestructureHandler`. Handles the complex logic of modifying the `SourceRule` hierarchy when a user changes a file's target asset via the GUI, ensuring data integrity. Triggered by signals from the model. + * `base_prediction_handler.py`: Defines the abstract `BasePredictionHandler` class, providing a common interface and threading (`QRunnable`) for prediction tasks. + * `prediction_handler.py`: Defines `RuleBasedPredictionHandler` (inherits from `BasePredictionHandler`). Generates the initial `SourceRule` hierarchy with predicted values based on input files and the selected preset rules. Runs in a background thread. + * `llm_prediction_handler.py`: Defines `LLMPredictionHandler` (inherits from `BasePredictionHandler`). Experimental handler using an LLM for prediction. Runs in a background thread. + * `llm_interaction_handler.py`: Defines `LLMInteractionHandler`. Manages the communication details (API calls, etc.) with the LLM service, used by `LLMPredictionHandler`. +* `utils/`: Directory containing shared utility modules. + * `workspace_utils.py`: Provides functions for managing processing workspaces, such as creating temporary directories and extracting archives (`prepare_processing_workspace`). Used by `main.py` (ProcessingTask) and `monitor.py`. + * `prediction_utils.py`: Provides utility functions related to prediction, such as generating a `SourceRule` from an archive (`generate_source_rule_from_archive`), used by `monitor.py`. * `blenderscripts/`: Contains Python scripts (`create_nodegroups.py`, `create_materials.py`) designed to be executed *within* Blender for post-processing. -* `Presets/`: Contains supplier-specific configuration files in JSON format. +* `Presets/`: Contains supplier-specific configuration files in JSON format, used by the `RuleBasedPredictionHandler` for initial rule generation. * `Testfiles/`: Contains example input assets for testing purposes. * `Tickets/`: Directory for issue and feature tracking using Markdown files. - -**Note on Data Passing:** As mentioned in the Architecture documentation, major changes to the data passing mechanisms between the GUI, Main (CLI orchestration), and `asset_processor` modules are currently being planned. The descriptions of module interactions and data flow within this document reflect the current state and will require review and updates once the plan for these changes is finalized. \ No newline at end of file +* `Deprecated/`: Contains older code, documentation, and proof-of-concept scripts that are no longer actively used. \ No newline at end of file diff --git a/Documentation/02_Developer_Guide/03_Key_Components.md b/Documentation/02_Developer_Guide/03_Key_Components.md index 6db1cda..95f1c17 100644 --- a/Documentation/02_Developer_Guide/03_Key_Components.md +++ b/Documentation/02_Developer_Guide/03_Key_Components.md @@ -2,69 +2,247 @@ This document describes the major classes and modules that form the core of the Asset Processor Tool. -## `AssetProcessor` (`asset_processor.py`) +## Core Processing Architecture -The `AssetProcessor` class is the central engine of the tool. It is responsible for processing a *single* input asset (either a ZIP archive or a folder) through the entire pipeline. Its key responsibilities include: +The asset processing pipeline has been refactored into a staged architecture, managed by an orchestrator. -* Setting up and cleaning up a temporary workspace for processing. -* Extracting or copying input files to the workspace. -* Inventorying and classifying files based on configured rules (maps, models, extra, ignored, unrecognised). -* Determining asset metadata such as name, category, and archetype. -* Processing texture maps (resizing, format/bit depth conversion, handling Gloss->Roughness inversion, calculating statistics). -* Merging channels from different maps according to merge rules. -* Generating the `metadata.json` file containing details about the processed asset. -* Organizing the final output files into the structured library directory. -* Providing methods (`get_detailed_file_predictions`) used by the GUI for previewing file classification. +### `ProcessingEngine` (`processing_engine.py`) + +The `ProcessingEngine` class serves as the primary entry point for initiating an asset processing task. Its main responsibilities are: + +* Initializing a `PipelineOrchestrator` instance. +* Providing the `PipelineOrchestrator` with the global `Configuration` object and a predefined list of processing stages. +* Invoking the orchestrator's `process_source_rule()` method with the input `SourceRule`, workspace path, output path, and other processing parameters. +* Managing a top-level temporary directory for the engine's operations if needed, though individual stages might also use sub-temporary directories via the `AssetProcessingContext`. + +It no longer contains the detailed logic for each processing step (like map manipulation, merging, etc.) directly. Instead, it delegates these tasks to the orchestrator and its stages. + +### `PipelineOrchestrator` (`processing/pipeline/orchestrator.py`) + +The `PipelineOrchestrator` class is responsible for managing the execution of the asset processing pipeline. Its key functions include: + +* Receiving a `SourceRule` object, `Configuration`, and a list of `ProcessingStage` objects. +* For each `AssetRule` within the `SourceRule`: + * Creating an `AssetProcessingContext` instance. + * Sequentially executing each registered `ProcessingStage`, passing the `AssetProcessingContext` to each stage. + * Handling exceptions that occur within stages and managing the overall status of asset processing (processed, skipped, failed). +* Managing a temporary directory for the duration of a `SourceRule` processing, which is made available to stages via the `AssetProcessingContext`. + +### `AssetProcessingContext` (`processing/pipeline/asset_context.py`) + +The `AssetProcessingContext` is a dataclass that acts as a stateful container for all data related to the processing of a single `AssetRule`. An instance of this context is created by the `PipelineOrchestrator` for each asset and is passed through each processing stage. Key information it holds includes: + +* The input `SourceRule` and the current `AssetRule`. +* Paths: `workspace_path`, `engine_temp_dir`, `output_base_path`. +* The `Configuration` object. +* `effective_supplier`: Determined by an early stage. +* `asset_metadata`: A dictionary to accumulate metadata about the asset. +* `processed_maps_details`: Stores details about individually processed maps (paths, dimensions, etc.). +* `merged_maps_details`: Stores details about merged maps. +* `files_to_process`: A list of `FileRule` objects to be processed for the current asset. +* `loaded_data_cache`: For caching loaded image data within an asset's processing. +* `status_flags`: For signaling conditions like `skip_asset` or `asset_failed`. +* `incrementing_value`, `sha5_value`: Optional values for path generation. + +Each stage reads from and writes to this context, allowing data and state to flow through the pipeline. + +### `Processing Stages` (`processing/pipeline/stages/`) + +The actual processing logic is broken down into a series of discrete stages, each inheriting from `ProcessingStage` (`processing/pipeline/stages/base_stage.py`). Each stage implements an `execute(context: AssetProcessingContext)` method. Key stages include (in typical execution order): + +* **`SupplierDeterminationStage`**: Determines the effective supplier. +* **`AssetSkipLogicStage`**: Checks if the asset processing should be skipped. +* **`MetadataInitializationStage`**: Initializes basic asset metadata. +* **`FileRuleFilterStage`**: Filters `FileRule`s to decide which files to process. +* **`GlossToRoughConversionStage`**: Handles gloss-to-roughness map inversion. +* **`AlphaExtractionToMaskStage`**: Extracts alpha channels to create masks. +* **`NormalMapGreenChannelStage`**: Inverts normal map green channels if required. +* **`IndividualMapProcessingStage`**: Processes individual maps (POT scaling, resolution variants, color conversion, stats, aspect ratio, filename conventions). +* **`MapMergingStage`**: Merges map channels based on rules. +* **`MetadataFinalizationAndSaveStage`**: Collects all metadata and saves `metadata.json` to a temporary location. +* **`OutputOrganizationStage`**: Copies all processed files and metadata to the final output directory structure. + +## `Rule Structure` (`rule_structure.py`) + +This module defines the data structures used to represent the hierarchical processing rules: + +* `SourceRule`: A dataclass representing rules applied at the source level. It contains nested `AssetRule` objects. +* `AssetRule`: A dataclass representing rules applied at the asset level. It contains nested `FileRule` objects. +* `FileRule`: A dataclass representing rules applied at the file level. + +These classes hold specific rule parameters (e.g., `supplier_identifier`, `asset_type`, `asset_type_override`, `item_type`, `item_type_override`, `target_asset_name_override`, `resolution_override`, `channel_merge_instructions`). Attributes like `asset_type` and `item_type_override` now use string types, which are validated against centralized lists in `config/app_settings.json`. These structures support serialization (Pickle, JSON) to allow them to be passed between different parts of theapplication, including across process boundaries. The `PipelineOrchestrator` and its stages heavily rely on the information within these rule objects, passed via the `AssetProcessingContext`. ## `Configuration` (`configuration.py`) The `Configuration` class manages the tool's settings. It is responsible for: -* Loading the core default settings defined in `config.py`. +* Loading the core default settings defined in `config/app_settings.json` (e.g., `FILE_TYPE_DEFINITIONS`, `ASSET_TYPE_DEFINITIONS`, `image_resolutions`, `map_merge_rules`, `output_filename_pattern`). * Loading the supplier-specific rules from a selected preset JSON file (`Presets/*.json`). * Merging the core settings and preset rules into a single, unified configuration object. * Validating the loaded configuration to ensure required settings are present. -* Pre-compiling regular expression patterns defined in the preset for efficient file classification by the `AssetProcessor`. +* Pre-compiling regular expression patterns defined in the preset for efficient file classification by the prediction handlers. -An instance of the `Configuration` class is typically created once per asset processing task (within a worker process) to ensure isolated and correct settings for each asset. +An instance of the `Configuration` class is typically created once per application run (or per processing batch) and passed to the `ProcessingEngine`, which then makes it available to the `PipelineOrchestrator` and subsequently to each stage via the `AssetProcessingContext`. -## `MainWindow` (`gui/main_window.py`) +## GUI Components (`gui/`) -The `MainWindow` class is the main application window for the Graphical User Interface (GUI). It handles the overall UI layout and user interaction: +The GUI has been refactored into several key components: -* Sets up the main window structure, including panels for the preset editor and processing controls. -* Manages the layout of UI elements like the drag-and-drop area, preview table, buttons, and status bar. -* Connects user actions (button clicks, drag/drop events) to corresponding handler methods (slots). -* Interacts with background processing and prediction handlers (`ProcessingHandler`, `PredictionHandler`) via Qt signals and slots to update the UI safely from background threads. -* Manages the GUI-specific logging handler (`QtLogHandler`) to display logs in the UI console. +### `MainWindow` (`gui/main_window.py`) -## `ProcessingHandler` (`gui/processing_handler.py`) +The `MainWindow` class acts as the main application window and **coordinator** for the GUI. Its primary responsibilities now include: -The `ProcessingHandler` class is designed to run in a separate `QThread` within the GUI. Its purpose is to manage the execution of the main asset processing pipeline and optional Blender scripts in the background, preventing the GUI from freezing. It: +* Setting up the main window structure (using a `QSplitter`) and menu bar. +* Instantiating and arranging the major GUI widgets: + * `PresetEditorWidget` (providing selector and JSON editor parts) + * `LLMEditorWidget` (for LLM settings) + * `MainPanelWidget` (containing the rule view and processing controls) + * `LogConsoleWidget` +* **Layout Management:** Placing the preset selector statically and using a `QStackedWidget` to switch between the `PresetEditorWidget`'s JSON editor and the `LLMEditorWidget`. +* **Editor Switching:** Handling the `preset_selection_changed_signal` from `PresetEditorWidget` to switch the stacked editor view (`_on_preset_selection_changed` slot). +* Connecting signals and slots between widgets, models (`UnifiedViewModel`), and handlers (`LLMInteractionHandler`, `AssetRestructureHandler`). +* Managing the overall application state related to GUI interactions (e.g., enabling/disabling controls). +* Handling top-level actions like loading sources (drag-and-drop), initiating predictions (`update_preview`), and starting the processing task (`_on_process_requested`). +* Managing background prediction threads (Rule-Based via `QThread`, LLM via `LLMInteractionHandler`). +* Implementing slots (`_on_rule_hierarchy_ready`, `_on_llm_prediction_ready_from_handler`, `_on_prediction_error`, `_handle_prediction_completion`) to update the model/view when prediction results/errors arrive. -* Manages a `concurrent.futures.ProcessPoolExecutor` to run individual asset processing tasks (`AssetProcessor.process()`) in separate worker processes. -* Submits processing tasks to the pool and monitors their completion. -* Communicates progress, status updates, and results back to the `MainWindow` using Qt signals. -* Handles the execution of Blender scripts via subprocess calls after asset processing is complete. -* Provides logic for cancelling ongoing processing tasks (though cancellation of already running worker processes is not immediate). +### `MainPanelWidget` (`gui/main_panel_widget.py`) -## `PredictionHandler` (`gui/prediction_handler.py`) +This widget contains the central part of the GUI, including: -The `PredictionHandler` class also runs in a separate `QThread` in the GUI. It is responsible for generating file classification previews in the background without blocking the UI. It: +* Controls for loading source files/directories. +* The preset selection dropdown. +* Buttons for initiating prediction and processing. +* The `RuleEditorWidget` which houses the hierarchical rule view. -* Calls methods on the `AssetProcessor` (specifically `get_detailed_file_predictions`) to analyze input files and predict their classification and output names based on the selected processing preset. -* Uses a `ThreadPoolExecutor` for potentially concurrent prediction tasks. -* Sends the prediction results back to the `MainWindow` via Qt signals to update the preview table. +### `PresetEditorWidget` (`gui/preset_editor_widget.py`) -## `ZipHandler` (`monitor.py`) +This widget provides the interface for managing presets: -The `ZipHandler` is a custom event handler used by the `monitor.py` script, built upon the `watchdog` library. It is responsible for: +* Loading, saving, and editing preset files (`Presets/*.json`). +* Displaying preset rules and settings in a tabbed JSON editor. +* Providing the preset selection list (`QListWidget`) including the "LLM Interpretation" option. +* **Refactored:** Exposes its selector (`selector_container`) and JSON editor (`json_editor_container`) as separate widgets for use by `MainWindow`. +* Emits `preset_selection_changed_signal` when the selection changes. -* Detecting file system events, specifically the creation of new `.zip` files, in the monitored input directory. -* Validating the filename format of detected ZIPs to extract the intended preset name. -* Triggering the main asset processing logic (`main.run_processing`) for valid new ZIP files. -* Managing the movement of processed source ZIP files to 'processed' or 'error' directories. +### `LogConsoleWidget` (`gui/log_console_widget.py`) -These key components work together to provide the tool's functionality, separating concerns and utilizing concurrency for performance and responsiveness. +This widget displays application logs within the GUI: -**Note on Data Passing:** As mentioned in the Architecture documentation, major changes to the data passing mechanisms between the GUI, Main (CLI orchestration), and `AssetProcessor` modules are currently being planned. The descriptions of module interactions and data flow within this document reflect the current state and will require review and updates once the plan for these changes is finalized. \ No newline at end of file +* Provides a text area for log messages. +* Integrates with Python's `logging` system via a custom `QtLogHandler`. +* Can be shown/hidden via the main window's "View" menu. + +### `LLMEditorWidget` (`gui/llm_editor_widget.py`) + +A new widget dedicated to editing LLM settings: + +* Provides a tabbed interface ("Prompt Settings", "API Settings") to edit `config/llm_settings.json`. +* Allows editing the main prompt, managing examples (add/delete/edit JSON), and configuring API details (URL, key, model, temperature, timeout). +* Loads settings via `load_settings()` and saves them using `_save_settings()` (which calls `configuration.save_llm_config()`). +* Placed within `MainWindow`'s `QStackedWidget`. + +### `UnifiedViewModel` (`gui/unified_view_model.py`) + +The `UnifiedViewModel` implements a `QAbstractItemModel` for use with Qt's model-view architecture. It is specifically designed to: + +* Wrap a list of `SourceRule` objects and expose their hierarchical structure (Source -> Asset -> File) to a `QTreeView` (the Unified Hierarchical View). +* Provide methods (`data`, `index`, `parent`, `rowCount`, `columnCount`, `flags`, `setData`) required by `QAbstractItemModel` to allow the `QTreeView` to display the rule hierarchy and support inline editing of specific attributes (e.g., `supplier_override`, `asset_type_override`, `item_type_override`, `target_asset_name_override`). +* Handle requests for data editing (`setData`) by validating input and updating the underlying `RuleHierarchyModel`. **Note:** Complex restructuring logic (e.g., moving files between assets when `target_asset_name_override` changes) is now delegated to the `AssetRestructureHandler`. +* Determine row background colors based on the `asset_type` and `item_type`/`item_type_override` using color metadata from the `Configuration`. +* Hold the `SourceRule` data (via `RuleHierarchyModel`) that is the single source of truth for the GUI's processing rules. +* Cache configuration data (`ASSET_TYPE_DEFINITIONS`, `FILE_TYPE_DEFINITIONS`, color maps) during initialization for improved performance in the `data()` method. +* Includes the `update_rules_for_sources` method, which intelligently merges new prediction results into the existing model data, preserving user overrides where possible. + +### `RuleHierarchyModel` (`gui/rule_hierarchy_model.py`) + +A simpler, non-Qt model used internally by `UnifiedViewModel` to manage the list of `SourceRule` objects and provide methods for accessing and modifying the hierarchy. + +### `AssetRestructureHandler` (`gui/asset_restructure_handler.py`) + +This handler contains the complex logic required to modify the `SourceRule` hierarchy when a file's target asset is changed via the GUI's `UnifiedViewModel`. It: + +* Is triggered by a signal (`targetAssetOverrideChanged`) from the `UnifiedViewModel`. +* Uses dedicated methods on the `RuleHierarchyModel` (`moveFileRule`, `createAssetRule`, `removeAssetRule`) to safely move `FileRule` objects between `AssetRule`s, creating or removing `AssetRule`s as needed. +* Ensures data consistency during these potentially complex restructuring operations. + +### `Delegates` (`gui/delegates.py`) + +This module contains custom `QStyledItemDelegate` implementations used by the Unified Hierarchical View (`QTreeView`) to provide inline editors for specific data types or rule attributes. Examples include delegates for: + +* `ComboBoxDelegate`: For selecting from predefined lists of allowed asset and file types, sourced from the `Configuration` (originally from `config/app_settings.json`). +* `LineEditDelegate`: For free-form text editing, such as the `target_asset_name_override`. +* `SupplierSearchDelegate`: For the "Supplier" column. Provides a `QLineEdit` with auto-completion suggestions loaded from `config/suppliers.json` and handles adding/saving new suppliers. + +These delegates handle the presentation and editing of data within the tree view cells, interacting with the `UnifiedViewModel` to get and set data. + +## Prediction Handlers (`gui/`) + +Prediction logic is handled by classes inheriting from a common base class, running in background threads. + +### `BasePredictionHandler` (`gui/base_prediction_handler.py`) + +An abstract base class (`QRunnable`) for prediction handlers. It defines the common structure and signals (`prediction_signal`) used by specific predictor implementations. It's designed to be run in a `QThreadPool`. + +### `RuleBasedPredictionHandler` (`gui/prediction_handler.py`) + +This class (inheriting from `BasePredictionHandler`) is responsible for generating the initial `SourceRule` hierarchy using predefined rules from presets. It: + +* Takes an input source identifier, file list, and `Configuration` object. +* Analyzes files based on regex patterns and rules defined in the loaded preset. +* Constructs a `SourceRule` hierarchy with predicted values. +* Emits the `prediction_signal` with the generated `SourceRule` object. + +### `LLMPredictionHandler` (`gui/llm_prediction_handler.py`) + +An experimental predictor (inheriting from `BasePredictionHandler`) that uses a Large Language Model (LLM). It: + +* Takes an input source identifier, file list, and `Configuration` object. +* Interacts with the `LLMInteractionHandler` to send data to the LLM and receive predictions. +* **Parses the LLM's JSON response**: It expects a specific two-part JSON structure (see `12_LLM_Predictor_Integration.md`). It first sanitizes the response (removing comments/markdown) and then parses the JSON. +* **Constructs `SourceRule`**: It groups files based on the `proposed_asset_group_name` from the JSON, assigns the final `asset_type` using the `asset_group_classifications` map, and builds the complete `SourceRule` hierarchy. +* Emits the `prediction_signal` with the generated `SourceRule` object or `error_signal` on failure. + +### `LLMInteractionHandler` (`gui/llm_interaction_handler.py`) + +This class now acts as the central manager for LLM prediction tasks: + +* **Manages the LLM prediction queue** and processes items sequentially. +* **Loads LLM configuration** directly from `config/llm_settings.json` and `config/app_settings.json`. +* **Instantiates and manages** the `LLMPredictionHandler` and its `QThread`. +* **Handles LLM task state** (running/idle) and signals changes to the GUI. +* Receives results/errors from `LLMPredictionHandler` and **emits signals** (`llm_prediction_ready`, `llm_prediction_error`, `llm_status_update`, `llm_processing_state_changed`) to `MainWindow`. + +## Utility Modules (`utils/`) + +Common utility functions have been extracted into separate modules: + +### `workspace_utils.py` + +Contains functions related to managing the processing workspace: + +* `prepare_processing_workspace`: Creates temporary directories, extracts archive files (ZIP, RAR, 7z), and returns the path to the prepared workspace. Used by `main.ProcessingTask` and `monitor.py`. + +### `prediction_utils.py` + +Contains utility functions supporting prediction tasks: + +* `generate_source_rule_from_archive`: A helper function used by `monitor.py` to perform rule-based prediction directly on an archive file without needing the full GUI setup. It extracts files temporarily, runs prediction logic similar to `RuleBasedPredictionHandler`, and returns a `SourceRule`. + +## Monitor (`monitor.py`) + +The `monitor.py` script implements the directory monitoring feature. It has been refactored to: + +* Use `watchdog` to detect new archive files in the input directory. +* Use a `ThreadPoolExecutor` to process detected archives asynchronously in a `_process_archive_task` function. +* Within the task, it: + * Loads the necessary `Configuration`. + * Calls `utils.prediction_utils.generate_source_rule_from_archive` to get the `SourceRule`. + * Calls `utils.workspace_utils.prepare_processing_workspace` to set up the workspace. + * Instantiates and runs the `ProcessingEngine` (which in turn uses the `PipelineOrchestrator`). + * Handles moving the source archive to 'processed' or 'error' directories. + * Cleans up the workspace. + +## Summary + +These key components, along with the refactored GUI structure and new utility modules, work together to provide the tool's functionality. The architecture emphasizes separation of concerns (configuration, rule generation, processing, UI), utilizes background processing for responsiveness (GUI prediction, Monitor tasks), and relies on the `SourceRule` object as the central data structure passed between different stages of the workflow. The processing core is now a staged pipeline managed by the `PipelineOrchestrator`, enhancing modularity and maintainability. \ No newline at end of file diff --git a/Documentation/02_Developer_Guide/04_Configuration_System_and_Presets.md b/Documentation/02_Developer_Guide/04_Configuration_System_and_Presets.md index 63a5cfb..1c1b4f8 100644 --- a/Documentation/02_Developer_Guide/04_Configuration_System_and_Presets.md +++ b/Documentation/02_Developer_Guide/04_Configuration_System_and_Presets.md @@ -2,34 +2,194 @@ This document provides technical details about the configuration system and the structure of preset files for developers working on the Asset Processor Tool. -## Configuration Flow +## Configuration System Overview -The tool utilizes a two-tiered configuration system: +The tool's configuration is managed by the `configuration.py` module and loaded from several JSON files, providing a layered approach for defaults, user overrides, definitions, and source-specific presets. -1. **Core Settings (`config.py`):** This Python module defines global default settings, constants, and core rules that apply generally across different asset sources. Examples include default output paths, standard image resolutions, map merge rules, output format rules, Blender executable paths, and default map types. -2. **Preset Files (`Presets/*.json`):** These JSON files define supplier-specific rules and overrides. They contain patterns (often regular expressions) to interpret filenames, classify map types, handle variants, define naming conventions, and specify other source-specific behaviors. +### Configuration Files -## `Configuration` Class (`configuration.py`) +The tool's configuration is loaded from several JSON files, providing a layered approach for defaults, user overrides, definitions, and source-specific presets. -The `Configuration` class is responsible for loading, merging, and preparing the configuration settings for use by the `AssetProcessor`. +1. **Application Settings (`config/app_settings.json`):** This JSON file defines the core global default settings, constants, and rules that apply generally across different asset sources (e.g., the global `OUTPUT_DIRECTORY_PATTERN` and `OUTPUT_FILENAME_PATTERN`, standard image resolutions, map merge rules, output format rules, Blender paths, temporary directory prefix, initial scaling mode, merge dimension mismatch strategy). See the [User Guide: Output Structure](../01_User_Guide/09_Output_Structure.md#available-tokens) for a list of available tokens for these patterns. + * *Note:* `ASSET_TYPE_DEFINITIONS` and `FILE_TYPE_DEFINITIONS` are no longer stored here; they have been moved to dedicated files. -* **Initialization:** An instance is created with a specific `preset_name`. -* **Loading:** - * It loads the core settings from `config.py` using `importlib.util`. - * It loads the specified preset JSON file from the `Presets/` directory. -* **Merging:** The loaded core settings and preset rules are merged into a single configuration object accessible via instance attributes. Preset values generally override core settings where applicable. -* **Validation (`_validate_configs`):** Performs basic structural validation on the loaded settings, checking for the presence of required keys and basic data types (e.g., ensuring `map_type_mapping` is a list of dictionaries). -* **Regex Compilation (`_compile_regex_patterns`):** A crucial step for performance. It iterates through the regex patterns defined in the preset (for extra files, models, bit depth variants, map keywords) and compiles them using `re.compile` (mostly case-insensitive). These compiled regex objects are stored as instance attributes (e.g., `self.compiled_map_keyword_regex`) for fast matching during file classification. It uses a helper (`_fnmatch_to_regex`) for basic wildcard (`*`, `?`) conversion in patterns. +2. **User Settings (`config/user_settings.json`):** This optional JSON file allows users to override specific settings defined in `config/app_settings.json`. If this file exists, its values for corresponding keys will take precedence over the base application settings. This file is primarily managed through the GUI's Application Preferences Editor. -An instance of `Configuration` is created within each worker process (`main.process_single_asset_wrapper`) to ensure that each concurrently processed asset uses the correct, isolated configuration based on the specified preset. +3. **Asset Type Definitions (`config/asset_type_definitions.json`):** This dedicated JSON file contains the definitions for different asset types (e.g., Surface, Model, Decal), including their descriptions, colors for UI representation, and example usage strings. + +4. **File Type Definitions (`config/file_type_definitions.json`):** This dedicated JSON file contains the definitions for different file types (specifically texture maps and models), including descriptions, colors for UI representation, examples of keywords/patterns, a standard alias (`standard_type`), bit depth handling rules (`bit_depth_rule`), a grayscale flag (`is_grayscale`), and an optional GUI keybind (`keybind`). + * **`keybind` Property:** Each file type object within `FILE_TYPE_DEFINITIONS` can optionally include a `keybind` property. This property accepts a single character string (e.g., `"C"`, `"R"`) representing the keyboard key. In the GUI, this key (typically combined with `Ctrl`) is used as a shortcut to set or toggle the corresponding file type for selected items in the Preview Table. + *Example:* + ```json + "MAP_COL": { + "description": "Color/Albedo Map", + "color": "#ffaa00", + "examples": ["_col.", "_basecolor.", "albedo", "diffuse"], + "standard_type": "COL", + "bit_depth_rule": "force_8bit", + "is_grayscale": false, + "keybind": "C" + }, + ``` + Note: The `bit_depth_rule` property in `FILE_TYPE_DEFINITIONS` is the primary source for determining bit depth handling for a given map type. + +5. **Supplier Settings (`config/suppliers.json`):** This JSON file stores settings specific to different asset suppliers. It is now structured as a dictionary where keys are supplier names and values are objects containing supplier-specific configurations. + * **Structure:** + ```json + { + "SupplierName1": { + "setting_key1": "value", + "setting_key2": "value" + }, + "SupplierName2": { + "setting_key1": "value" + } + } + ``` + * **`normal_map_type` Property:** A key setting within each supplier's object is `normal_map_type`, specifying whether normal maps from this supplier use "OpenGL" or "DirectX" conventions. + *Example:* + ```json + { + "Poliigon": { + "normal_map_type": "DirectX" + }, + "Dimensiva": { + "normal_map_type": "OpenGL" + } + } + ``` + +6. **LLM Settings (`config/llm_settings.json`):** This JSON file contains settings specifically related to the LLM predictor, such as the API endpoint, model name, prompt template, and examples. These settings are managed through the GUI using the `LLMEditorWidget`. + +7. **Preset Files (`Presets/*.json`):** These JSON files define source-specific rules and overrides. They contain patterns to interpret filenames, classify map types, handle variants, define naming conventions, and specify other source-specific behaviors. Preset settings override values from `app_settings.json` and `user_settings.json` where applicable. + + +### Configuration Loading and Access + +The `configuration.py` module contains the `Configuration` class and standalone functions for loading and saving settings. + +* **`Configuration` Class:** This is the primary class used by the processing engine and other core components. When initialized with a `preset_name`, it loads settings in the following order, with later files overriding earlier ones for shared keys: + 1. `config/app_settings.json` (Base Defaults) + 2. `config/user_settings.json` (User Overrides - if exists) + 3. `config/asset_type_definitions.json` (Asset Type Definitions) + 4. `config/file_type_definitions.json` (File Type Definitions) + 5. `config/llm_settings.json` (LLM Settings) + 6. `Presets/{preset_name}.json` (Preset Overrides) + + The loaded settings are merged into internal dictionaries, and most are accessible via instance properties (e.g., `config.output_base_dir`, `config.llm_endpoint_url`, `config.get_asset_type_definitions()`). Regex patterns defined in the merged configuration are pre-compiled for performance. + +* **`load_base_config()` function:** This standalone function is primarily used by the GUI for initial setup and displaying default/user-overridden settings before a specific preset is selected. It loads and merges the following files: + 1. `config/app_settings.json` + 2. `config/user_settings.json` (if exists) + 3. `config/asset_type_definitions.json` + 4. `config/file_type_definitions.json` + + It returns a single dictionary containing the combined settings and definitions. + +* **Saving Functions:** + * `save_base_config(settings_dict)`: Saves the provided dictionary to `config/app_settings.json`. (Used less frequently now for user-driven saves). + * `save_user_config(settings_dict)`: Saves the provided dictionary to `config/user_settings.json`. Used by `ConfigEditorDialog`. + * `save_llm_config(settings_dict)`: Saves the provided dictionary to `config/llm_settings.json`. Used by `LLMEditorWidget`. + +## Supplier Management (`config/suppliers.json`) + +A file, `config/suppliers.json`, is used to store a persistent list of known supplier names. This file is a simple JSON array of strings. + +* **Purpose:** Provides a list of suggestions for the "Supplier" field in the GUI's Unified View, enabling auto-completion. +* **Management:** The GUI's `SupplierSearchDelegate` is responsible for loading this list on startup, adding new, unique supplier names entered by the user, and saving the updated list back to the file. + +## GUI Configuration Editors + +The GUI provides dedicated editors for modifying configuration files: + +* **`ConfigEditorDialog` (`gui/config_editor_dialog.py`):** Edits user-configurable application settings. +* **`LLMEditorWidget` (`gui/llm_editor_widget.py`):** Edits the LLM-specific settings. + +### `ConfigEditorDialog` (`gui/config_editor_dialog.py`) + +The GUI includes a dedicated editor for modifying user-configurable settings. This is implemented in `gui/config_editor_dialog.py`. + +* **Purpose:** Provides a user-friendly interface for viewing the effective application settings (defaults + user overrides + definitions) and editing the user-specific overrides. +* **Implementation:** The dialog loads the effective settings using `load_base_config()`. It presents relevant settings in a tabbed layout ("General", "Output & Naming", etc.). When saving, it now performs a **granular save**: it loads the current content of `config/user_settings.json`, identifies only the settings that were changed by the user during the current dialog session (by comparing against the initial state), updates only those specific values in the loaded `user_settings.json` content, and saves the modified content back to `config/user_settings.json` using `save_user_config()`. This preserves any other settings in `user_settings.json` that were not touched. The dialog displays definitions from `asset_type_definitions.json` and `file_type_definitions.json` but does not save changes to these files. +* **Limitations:** Currently, editing complex fields like `IMAGE_RESOLUTIONS` or the full details of `MAP_MERGE_RULES` via the UI is not fully supported for saving to `user_settings.json`. + +### `LLMEditorWidget` (`gui/llm_editor_widget.py`) + +* **Purpose:** Provides a user-friendly interface for viewing and editing the LLM settings defined in `config/llm_settings.json`. +* **Implementation:** Uses tabs for "Prompt Settings" and "API Settings". Allows editing the prompt, managing examples, and configuring API details. When saving, it also performs a **granular save**: it loads the current content of `config/llm_settings.json`, identifies only the settings changed by the user in the current session, updates only those values, and saves the modified content back to `config/llm_settings.json` using `configuration.save_llm_config()`. ## Preset File Structure (`Presets/*.json`) Preset files are the primary way to adapt the tool to new asset sources. Developers should use `Presets/_template.json` as a starting point. Key fields include: * `supplier_name`: The name of the asset source (e.g., `"Poliigon"`). Used for output directory naming. -* `map_type_mapping`: A list of dictionaries, each mapping source filename patterns/keywords to a standard internal map type (defined in `config.py`). - * `target_type`: The standard internal map type (e.g., `"COL"`, `"NRM"`). +* `map_type_mapping`: A list of dictionaries, each mapping source filename patterns/keywords to a specific file type. The `target_type` for this mapping **must** be a key from the `FILE_TYPE_DEFINITIONS` now located in `config/file_type_definitions.json`. + * `target_type`: The specific file type key from `FILE_TYPE_DEFINITIONS` (e.g., `"MAP_COL"`, `"MAP_NORM_GL"`, `"MAP_RGH"`). This replaces previous alias-based systems. The common aliases like "COL" or "NRM" are now derived from the `standard_type` property within `FILE_TYPE_DEFINITIONS` but are not used directly for `target_type`. + * `keywords`: A list of filename patterns (regex or fnmatch-style wildcards) used to identify this map type. The order of keywords within this list, and the order of dictionaries in the `map_type_mapping` list, determines the priority for assigning variant suffixes (`-1`, `-2`, etc.) when multiple files match the same `target_type`. +* `bit_depth_variants`: A dictionary mapping standard map types (e.g., `"NRM"`) to a pattern identifying its high bit-depth variant (e.g., `"*_NRM16*.tif"`). Files matching these patterns are prioritized over their standard counterparts. +* `map_bit_depth_rules`: Defines how to handle the bit depth of source maps. Can specify a default behavior (`"respect"` or `"force_8bit"`) and overrides for specific map types. +* `model_patterns`: A list of regex patterns to identify model files (e.g., `".*\\.fbx"`, `".*\\.obj"`). +* `move_to_extra_patterns`: A list of regex patterns for files that should be moved directly to the `Extra/` output subdirectory without further processing. +* `source_naming_convention`: Rules for extracting the base asset name and potentially the archetype from source filenames or directory structures (e.g., using separators and indices). +* `asset_category_rules`: Keywords or patterns used to determine the asset category (e.g., identifying `"Decal"` based on keywords). +* `archetype_rules`: Keywords or patterns used to determine the asset archetype (e.g., identifying `"Wood"` or `"Metal"`). + +Careful definition of these patterns and rules, especially the regex in `map_type_mapping`, `bit_depth_variants`, `model_patterns`, and `move_to_extra_patterns`, is essential for correct asset processing. + +**Note on Data Passing:** As mentioned in the Architecture documentation, major changes to the data passing mechanisms between the GUI, Main (CLI orchestration), and `AssetProcessor` modules are currently being planned. The descriptions of how configuration data is handled and passed within this document reflect the current state and will require review and updates once the plan for these changes is finalized. + +## Supplier Management (`config/suppliers.json`) + +A new file, `config/suppliers.json`, is used to store a persistent list of known supplier names. This file is a simple JSON array of strings. + +* **Purpose:** Provides a list of suggestions for the "Supplier" field in the GUI's Unified View, enabling auto-completion. +* **Management:** The GUI's `SupplierSearchDelegate` is responsible for loading this list on startup, adding new, unique supplier names entered by the user, and saving the updated list back to the file. + +## `Configuration` Class (`configuration.py`) + +The `Configuration` class is central to the new configuration system. It is responsible for loading, merging, and preparing the configuration settings for use by the `ProcessingEngine` and other components like the `PredictionHandler` and `LLMPredictionHandler`. + +* **Initialization:** An instance is created with a specific `preset_name`. +* **Loading:** + * It first loads the base application settings from `config/app_settings.json`. + * It then loads the LLM-specific settings from `config/llm_settings.json`. + * Finally, it loads the specified preset JSON file from the `Presets/` directory. +* **Merging & Access:** The base settings from `app_settings.json` are merged with the preset rules. LLM settings are stored separately. Most settings are accessible via instance properties (e.g., `config.llm_endpoint_url`). Preset values generally override the base settings where applicable. **Exception:** The `OUTPUT_DIRECTORY_PATTERN` and `OUTPUT_FILENAME_PATTERN` are loaded *only* from `app_settings.json` and are accessed via `config.output_directory_pattern` and `config.output_filename_pattern` respectively; they are not defined in or overridden by presets. +* **Validation (`_validate_configs`):** Performs basic structural validation on the loaded settings (base, LLM, and preset), checking for the presence of required keys and basic data types. Logs warnings for missing optional LLM keys. +* **Regex Compilation (`_compile_regex_patterns`):** Compiles regex patterns defined in the merged configuration (from base settings and the preset) for performance. Compiled regex objects are stored as instance attributes (e.g., `self.compiled_map_keyword_regex`). +* **LLM Settings Access:** The `Configuration` class provides direct property access (e.g., `config.llm_endpoint_url`, `config.llm_api_key`, `config.llm_model_name`, `config.llm_temperature`, `config.llm_request_timeout`, `config.llm_predictor_prompt`, `config.get_llm_examples()`) to allow components like the `LLMPredictionHandler` to easily access the necessary LLM configuration values loaded from `config/llm_settings.json`. + +An instance of `Configuration` is created within each worker process (`main.process_single_asset_wrapper`) to ensure that each concurrently processed asset uses the correct, isolated configuration based on the specified preset and the base application settings. The `LLMInteractionHandler` loads LLM settings directly using helper functions or file access, not the `Configuration` class. + +## GUI Configuration Editors + +The GUI provides dedicated editors for modifying configuration files: + +* **`ConfigEditorDialog` (`gui/config_editor_dialog.py`):** Edits the core `config/app_settings.json`. +* **`LLMEditorWidget` (`gui/llm_editor_widget.py`):** Edits the LLM-specific `config/llm_settings.json`. + +### `ConfigEditorDialog` (`gui/config_editor_dialog.py`) + +The GUI includes a dedicated editor for modifying the `config/app_settings.json` file. This is implemented in `gui/config_editor_dialog.py`. + +* **Purpose:** Provides a user-friendly interface for viewing and editing the core application settings defined in `app_settings.json`. +* **Implementation:** The dialog loads the JSON content of `app_settings.json`, presents it in a tabbed layout ("General", "Output & Naming", etc.) using standard GUI widgets mapped to the JSON structure, and saves the changes back to the file. The "Output & Naming" tab specifically handles the global `OUTPUT_DIRECTORY_PATTERN` and `OUTPUT_FILENAME_PATTERN`. It supports editing basic fields, tables for definitions (`FILE_TYPE_DEFINITIONS`, `ASSET_TYPE_DEFINITIONS`), and a list/detail view for merge rules (`MAP_MERGE_RULES`). The definitions tables include dynamic color editing features. +* **Limitations:** Currently, editing complex fields like `IMAGE_RESOLUTIONS` or the full details of `MAP_MERGE_RULES` via the UI is not fully supported. +* **Note:** Changes made through the `ConfigEditorDialog` are written directly to `config/app_settings.json` (using `save_base_config`) but require an application restart to be loaded and applied by the `Configuration` class during processing. + +### `LLMEditorWidget` (`gui/llm_editor_widget.py`) + +* **Purpose:** Provides a user-friendly interface for viewing and editing the LLM settings defined in `config/llm_settings.json`. +* **Implementation:** Uses tabs for "Prompt Settings" and "API Settings". Allows editing the prompt, managing examples, and configuring API details. +* **Persistence:** Saves changes directly to `config/llm_settings.json` using the `configuration.save_llm_config()` function. Changes are loaded by the `LLMInteractionHandler` the next time an LLM task is initiated. + +## Preset File Structure (`Presets/*.json`) + +Preset files are the primary way to adapt the tool to new asset sources. Developers should use `Presets/_template.json` as a starting point. Key fields include: + +* `supplier_name`: The name of the asset source (e.g., `"Poliigon"`). Used for output directory naming. +* `map_type_mapping`: A list of dictionaries, each mapping source filename patterns/keywords to a specific file type. The `target_type` for this mapping **must** be a key from `FILE_TYPE_DEFINITIONS` located in `config/app_settings.json`. + * `target_type`: The specific file type key from `FILE_TYPE_DEFINITIONS` (e.g., `"MAP_COL"`, `"MAP_NORM_GL"`, `"MAP_RGH"`). This replaces previous alias-based systems. The common aliases like "COL" or "NRM" are now derived from the `standard_type` property within `FILE_TYPE_DEFINITIONS` but are not used directly for `target_type`. * `keywords`: A list of filename patterns (regex or fnmatch-style wildcards) used to identify this map type. The order of keywords within this list, and the order of dictionaries in the `map_type_mapping` list, determines the priority for assigning variant suffixes (`-1`, `-2`, etc.) when multiple files match the same `target_type`. * `bit_depth_variants`: A dictionary mapping standard map types (e.g., `"NRM"`) to a pattern identifying its high bit-depth variant (e.g., `"*_NRM16*.tif"`). Files matching these patterns are prioritized over their standard counterparts. * `map_bit_depth_rules`: Defines how to handle the bit depth of source maps. Can specify a default behavior (`"respect"` or `"force_8bit"`) and overrides for specific map types. diff --git a/Documentation/02_Developer_Guide/05_Processing_Pipeline.md b/Documentation/02_Developer_Guide/05_Processing_Pipeline.md index 8233ab8..4dead92 100644 --- a/Documentation/02_Developer_Guide/05_Processing_Pipeline.md +++ b/Documentation/02_Developer_Guide/05_Processing_Pipeline.md @@ -1,82 +1,98 @@ -# Developer Guide: Processing Pipeline +Cl# Developer Guide: Processing Pipeline -This document details the step-by-step technical process executed by the `AssetProcessor` class (`asset_processor.py`) when processing a single asset. +This document details the step-by-step technical process executed by the asset processing pipeline, which is initiated by the [`ProcessingEngine`](processing_engine.py:73) class (`processing_engine.py`) and orchestrated by the [`PipelineOrchestrator`](processing/pipeline/orchestrator.py:36) (`processing/pipeline/orchestrator.py`). -The `AssetProcessor.process()` method orchestrates the following pipeline: +The [`ProcessingEngine.process()`](processing_engine.py:131) method serves as the main entry point. It initializes a [`PipelineOrchestrator`](processing/pipeline/orchestrator.py:36) instance, providing it with the application's [`Configuration`](configuration.py:68) object and predefined lists of pre-item and post-item processing stages. The [`PipelineOrchestrator.process_source_rule()`](processing/pipeline/orchestrator.py:95) method then manages the execution of these stages for each asset defined in the input [`SourceRule`](rule_structure.py:40). -1. **Workspace Setup (`_setup_workspace`)**: - * Creates a temporary directory using `tempfile.mkdtemp()` to isolate the processing of the current asset. +A crucial component in this architecture is the [`AssetProcessingContext`](processing/pipeline/asset_context.py:86) (`processing/pipeline/asset_context.py`). An instance of this dataclass is created for each [`AssetRule`](rule_structure.py:22) being processed. It acts as a stateful container, carrying all relevant data (source files, rules, configuration, intermediate results, metadata) and is passed sequentially through each stage. Each stage can read from and write to the context, allowing data to flow and be modified throughout the pipeline. -2. **Input Extraction (`_extract_input`)**: - * If the input is a supported archive type (.zip, .rar, .7z), it's extracted into the temporary workspace using the appropriate library (`zipfile`, `rarfile`, or `py7zr`). - * If the input is a directory, its contents are copied into the temporary workspace. - * Includes basic error handling for invalid or password-protected archives. +The pipeline execution for each asset follows this general flow: -3. **File Inventory and Classification (`_inventory_and_classify_files`)**: - * Scans the contents of the temporary workspace. - * Uses the pre-compiled regex patterns from the loaded `Configuration` object to classify each file. - * Classification follows a multi-pass approach for priority: - * Explicitly marked `Extra/` files (using `move_to_extra_patterns` regex). - * Model files (using `model_patterns` regex). - * Potential Texture Maps (matching `map_type_mapping` keyword patterns). - * Standalone 16-bit variants check (using `bit_depth_variants` patterns). - * Prioritization of 16-bit variants over their 8-bit counterparts (marking the 8-bit version as `Ignored`). - * Final classification of remaining potential maps. - * Remaining files are classified as `Unrecognised` (and typically moved to `Extra/` later). - * Stores the classification results (including source path, determined map type, potential variant suffix, etc.) in `self.classified_files`. - * Sorts potential map variants based on preset rule order, keyword order within the rule, and finally alphabetical path to determine suffix assignment priority (`-1`, `-2`, etc.). +1. **Pre-Item Stages:** A sequence of stages executed once per asset before the core item processing loop. These stages typically perform initial setup, filtering, and asset-level transformations. +2. **Core Item Processing Loop:** The [`PipelineOrchestrator`](processing/pipeline/orchestrator.py:36) iterates through a list of "processing items" (individual files or merge tasks) prepared by a dedicated stage. For each item, a sequence of core processing stages is executed. +3. **Post-Item Stages:** A sequence of stages executed once per asset after the core item processing loop is complete. These stages handle final tasks like organizing output files and saving metadata. -4. **Base Metadata Determination (`_determine_base_metadata`, `_determine_single_asset_metadata`)**: - * Determines the base asset name using `source_naming_convention` rules from the `Configuration` (separators, indices), with fallbacks to common prefixes or the input name. Handles multiple distinct assets within a single input source. - * Determines the asset category (`Texture`, `Asset`, `Decal`) based on the presence of model files or `decal_keywords` in the `Configuration`. - * Determines the asset archetype (e.g., `Wood`, `Metal`) by matching keywords from `archetype_rules` (in `Configuration`) against file stems or the determined base name. - * Stores this preliminary metadata. +## Pipeline Stages -5. **Skip Check**: - * If the `overwrite` flag (passed during initialization) is `False`, the tool checks if the final output directory for the determined asset name already exists and contains a `metadata.json` file. - * If both exist, processing for this specific asset is skipped, marked as "skipped", and the pipeline moves to the next asset (if processing multiple assets from one source) or finishes. +The stages are executed in the following order for each asset: -6. **Map Processing (`_process_maps`)**: - * Iterates through the files classified as texture maps for the current asset. - * Loads the image using `cv2.imread` (handling grayscale and unchanged flags). Converts BGR to RGB internally for consistency (except for saving non-EXR formats). - * Handles Glossiness-to-Roughness inversion if necessary (loads gloss, inverts `1.0 - img/norm`, prioritizes gloss source if both exist). - * Resizes the image to target resolutions defined in `IMAGE_RESOULTIONS` (from `Configuration`) using `cv2.resize` (`INTER_LANCZOS4` for downscaling). Upscaling is generally avoided by checks. - * Determines the output bit depth based on `MAP_BIT_DEPTH_RULES` (`respect` vs `force_8bit`). - * Determines the output file format (`.jpg`, `.png`, `.exr`) based on a hierarchy of rules: - * `FORCE_LOSSLESS_MAP_TYPES` list (overrides other logic). - * `RESOLUTION_THRESHOLD_FOR_JPG` (forces JPG for large 8-bit maps). - * Source format, target bit depth, and configured defaults (`OUTPUT_FORMAT_16BIT_PRIMARY`, `OUTPUT_FORMAT_8BIT`). - * Converts the NumPy array data type appropriately before saving (e.g., float to uint8/uint16 with scaling). - * Saves the processed map using `cv2.imwrite` (converting RGB back to BGR if saving to non-EXR formats). Includes fallback logic (e.g., attempting PNG if saving 16-bit EXR fails). - * Calculates image statistics (Min/Max/Mean) using `_calculate_image_stats` on normalized float64 data for the `CALCULATE_STATS_RESOLUTION`. - * Determines the aspect ratio change string (e.g., `"EVEN"`, `"X150"`) using `_normalize_aspect_ratio_change`. - * Stores details about each processed map (path, resolution, format, stats, etc.) in `processed_maps_details_asset`. +### Pre-Item Stages -7. **Map Merging (`_merge_maps_from_source`)**: - * Iterates through the `MAP_MERGE_RULES` defined in the `Configuration`. - * Identifies the required *source* map files needed as input for each merge rule based on the classified files. - * Determines common resolutions available across the required input maps. - * Loads the necessary source map channels for each common resolution (using a helper `_load_and_transform_source` which includes caching). - * Converts inputs to normalized float32 (0-1). - * Injects default channel values (from rule `defaults`) if an input channel is missing. - * Merges channels using `cv2.merge`. - * Determines output bit depth and format based on rules (similar logic to `_process_maps`, considering input properties). Handles potential JPG 16-bit conflict by forcing 8-bit. - * Saves the merged map using the `_save_image` helper (includes data type/color space conversions and fallback). - * Stores details about each merged map in `merged_maps_details_asset`. +These stages are executed sequentially once for each asset before the core item processing loop begins. -8. **Metadata File Generation (`_generate_metadata_file`)**: - * Collects all determined information for the current asset: base metadata, details from `processed_maps_details_asset` and `merged_maps_details_asset`, list of ignored files, source preset used, etc. - * Writes this collected data into the `metadata.json` file within the temporary workspace using `json.dump`. +1. **[`SupplierDeterminationStage`](processing/pipeline/stages/supplier_determination.py:6)** (`processing/pipeline/stages/supplier_determination.py`): + * **Responsibility**: Determines the effective supplier for the asset based on the [`SourceRule`](rule_structure.py:40)'s `supplier_override`, `supplier_identifier`, and validation against configured suppliers. + * **Context Interaction**: Sets `context.effective_supplier` and may set a `supplier_error` flag in `context.status_flags`. -9. **Output Organization (`_organize_output_files`)**: - * Creates the final structured output directory: `///`. - * Creates subdirectories `Extra/`, `Unrecognised/`, and `Ignored/` within the asset directory. - * Moves the processed maps, merged maps, model files, `metadata.json`, and files classified as Extra, Unrecognised, or Ignored from the temporary workspace into their respective locations in the final output directory structure. +2. **[`AssetSkipLogicStage`](processing/pipeline/stages/asset_skip_logic.py:5)** (`processing/pipeline/stages/asset_skip_logic.py`): + * **Responsibility**: Checks if the entire asset should be skipped based on conditions like a missing/invalid supplier, a "SKIP" status in asset metadata, or if the asset is already processed and overwrite is disabled. + * **Context Interaction**: Sets the `skip_asset` flag and `skip_reason` in `context.status_flags` if the asset should be skipped. -10. **Workspace Cleanup (`_cleanup_workspace`)**: - * Removes the temporary workspace directory and its contents using `shutil.rmtree()`. This is called within a `finally` block to ensure cleanup is attempted even if errors occur during processing. +3. **[`MetadataInitializationStage`](processing/pipeline/stages/metadata_initialization.py:81)** (`processing/pipeline/stages/metadata_initialization.py`): + * **Responsibility**: Initializes the `context.asset_metadata` dictionary with base information derived from the [`AssetRule`](rule_structure.py:22), [`SourceRule`](rule_structure.py:40), and [`Configuration`](configuration.py:68). This includes asset name, IDs, source/output paths, timestamps, and initial status. + * **Context Interaction**: Populates `context.asset_metadata`. Initializes `context.processed_maps_details` and `context.merged_maps_details` as empty dictionaries (these are used internally by subsequent stages but are not directly part of the final `metadata.json` in their original form). -11. **(Optional) Blender Script Execution**: - * If triggered via CLI arguments (`--nodegroup-blend`, `--materials-blend`) or GUI controls, the orchestrator (`main.py` or `gui/processing_handler.py`) executes the corresponding Blender scripts (`blenderscripts/*.py`) using `subprocess.run` after the `AssetProcessor.process()` call completes successfully for an asset batch. See `Developer Guide: Blender Integration Internals` for more details. +4. **[`FileRuleFilterStage`](processing/pipeline/stages/file_rule_filter.py:10)** (`processing/pipeline/stages/file_rule_filter.py`): + * **Responsibility**: Filters the [`FileRule`](rule_structure.py:5) objects associated with the asset to determine which individual files should be considered for processing. It identifies and excludes files matching "FILE_IGNORE" rules based on their `item_type`. + * **Context Interaction**: Populates `context.files_to_process` with the list of [`FileRule`](rule_structure.py:5) objects that are not ignored. -**Note on Data Passing:** As mentioned in the Architecture documentation, major changes to the data passing mechanisms between the GUI, Main (CLI orchestration), and `AssetProcessor` modules are currently being planned. The descriptions of how data is processed and transformed within this pipeline reflect the current state and will require review and updates once the plan for these changes is finalized. \ No newline at end of file +5. **[`GlossToRoughConversionStage`](processing/pipeline/stages/gloss_to_rough_conversion.py:15)** (`processing/pipeline/stages/gloss_to_rough_conversion.py`): + * **Responsibility**: Identifies processed maps in `context.processed_maps_details` whose `internal_map_type` starts with "MAP_GLOSS". If found, it loads the temporary image data, inverts it using the shared utility function [`apply_common_map_transformations`](processing/utils/image_processing_utils.py), saves a new temporary roughness map ("MAP_ROUGH"), and updates the corresponding details in `context.processed_maps_details` (setting `internal_map_type` to "MAP_ROUGH") and the relevant [`FileRule`](rule_structure.py:5) in `context.files_to_process` (setting `item_type` to "MAP_ROUGH"). + * **Context Interaction**: Reads from and updates `context.processed_maps_details` (specifically `internal_map_type` and `temp_processed_file`) and `context.files_to_process` (specifically `item_type`). + +6. **[`AlphaExtractionToMaskStage`](processing/pipeline/stages/alpha_extraction_to_mask.py:16)** (`processing/pipeline/stages/alpha_extraction_to_mask.py`): + * **Responsibility**: If no mask map is explicitly defined for the asset (as a [`FileRule`](rule_structure.py:5) with `item_type="MAP_MASK"`), this stage searches `context.processed_maps_details` for a suitable source map (e.g., a "MAP_COL" with an alpha channel, based on its `internal_map_type`). If found, it extracts the alpha channel, saves it as a new temporary mask map, and adds a new [`FileRule`](rule_structure.py:5) (with `item_type="MAP_MASK"`) and corresponding details (with `internal_map_type="MAP_MASK"`) to the context. + * **Context Interaction**: Reads from `context.processed_maps_details`, adds a new [`FileRule`](rule_structure.py:5) to `context.files_to_process`, and adds a new entry to `context.processed_maps_details` (setting `internal_map_type`). + +7. **[`NormalMapGreenChannelStage`](processing/pipeline/stages/normal_map_green_channel.py:14)** (`processing/pipeline/stages/normal_map_green_channel.py`): + * **Responsibility**: Identifies processed normal maps in `context.processed_maps_details` (those with an `internal_map_type` starting with "MAP_NRM"). If the global `invert_normal_map_green_channel_globally` configuration is true, it loads the temporary image data, inverts the green channel using the shared utility function [`apply_common_map_transformations`](processing/utils/image_processing_utils.py), saves a new temporary modified normal map, and updates the `temp_processed_file` path in `context.processed_maps_details`. + * **Context Interaction**: Reads from and updates `context.processed_maps_details` (specifically `temp_processed_file` and `notes`). + +### Core Item Processing Loop + +The [`PipelineOrchestrator`](processing/pipeline/orchestrator.py:36) iterates through the `context.processing_items` list (populated by the [`PrepareProcessingItemsStage`](processing/pipeline/stages/prepare_processing_items.py:10)). For each item (either a [`FileRule`](rule_structure.py:5) for a regular map or a [`MergeTaskDefinition`](processing/pipeline/asset_context.py:16) for a merged map), the following stages are executed sequentially: + +1. **[`PrepareProcessingItemsStage`](processing/pipeline/stages/prepare_processing_items.py:10)** (`processing/pipeline/stages/prepare_processing_items.py`): + * **Responsibility**: (Executed once before the loop) Creates the `context.processing_items` list by combining [`FileRule`](rule_structure.py:5)s from `context.files_to_process` and [`MergeTaskDefinition`](processing/pipeline/asset_context.py:16)s derived from the global `map_merge_rules` configuration. It correctly accesses `map_merge_rules` from `context.config_obj` and validates each merge rule for the presence of `output_map_type` and a dictionary for `inputs`. Initializes `context.intermediate_results`. + * **Context Interaction**: Reads from `context.files_to_process` and `context.config_obj` (accessing `map_merge_rules`). Populates `context.processing_items` and initializes `context.intermediate_results`. + +2. **[`RegularMapProcessorStage`](processing/pipeline/stages/regular_map_processor.py:18)** (`processing/pipeline/stages/regular_map_processor.py`): + * **Responsibility**: (Executed per [`FileRule`](rule_structure.py:5) item) Checks if the `FileRule.item_type` starts with "MAP_". If not, the item is skipped. Otherwise, it loads the image data for the file, determines its potentially suffixed internal map type (e.g., "MAP_COL-1"), applies in-memory transformations (Gloss-to-Rough, Normal Green Invert) using the shared utility function [`apply_common_map_transformations`](processing/utils/image_processing_utils.py), and returns the processed image data and details in a [`ProcessedRegularMapData`](processing/pipeline/asset_context.py:23) object. The `internal_map_type` in the output reflects any transformations (e.g., "MAP_GLOSS" becomes "MAP_ROUGH"). + * **Context Interaction**: Reads from the input [`FileRule`](rule_structure.py:5) (checking `item_type`) and [`Configuration`](configuration.py:68). Returns a [`ProcessedRegularMapData`](processing/pipeline/asset_context.py:23) object which is stored in `context.intermediate_results`. + +3. **[`MergedTaskProcessorStage`](processing/pipeline/stages/merged_task_processor.py:68)** (`processing/pipeline/stages/merged_task_processor.py`): + * **Responsibility**: (Executed per [`MergeTaskDefinition`](processing/pipeline/asset_context.py:16) item) Validates that all input map types specified in the merge rule start with "MAP_". If not, the task is failed. It dynamically loads input images by looking up the required input map types (e.g., "MAP_NRM") in `context.processed_maps_details` and using the temporary file paths from their `saved_files_info`. It applies in-memory transformations to inputs using [`apply_common_map_transformations`](processing/utils/image_processing_utils.py), handles dimension mismatches (with fallback creation if configured and `source_dimensions` are available), performs the channel merging operation, and returns the merged image data and details in a [`ProcessedMergedMapData`](processing/pipeline/asset_context.py:35) object. The `output_map_type` of the merged map must also be "MAP_" prefixed in the configuration. + * **Context Interaction**: Reads from the input [`MergeTaskDefinition`](processing/pipeline/asset_context.py:16) (checking input map types), `context.workspace_path`, `context.processed_maps_details` (for input image data), and [`Configuration`](configuration.py:68). Returns a [`ProcessedMergedMapData`](processing/pipeline/asset_context.py:35) object which is stored in `context.intermediate_results`. + +4. **[`InitialScalingStage`](processing/pipeline/stages/initial_scaling.py:14)** (`processing/pipeline/stages/initial_scaling.py`): + * **Responsibility**: (Executed per item) Applies initial scaling (e.g., Power-of-Two downscaling) to the image data from the previous processing stage based on the `initial_scaling_mode` configuration. + * **Context Interaction**: Takes a [`InitialScalingInput`](processing/pipeline/asset_context.py:46) (containing image data and config) and returns an [`InitialScalingOutput`](processing/pipeline/asset_context.py:54) object, which updates the item's entry in `context.intermediate_results`. + +5. **[`SaveVariantsStage`](processing/pipeline/stages/save_variants.py:15)** (`processing/pipeline/stages/save_variants.py`): + * **Responsibility**: (Executed per item) Takes the final processed image data (potentially scaled) and configuration, and calls a utility to save the image to temporary files in various resolutions and formats as defined by the configuration. + * **Context Interaction**: Takes a [`SaveVariantsInput`](processing/pipeline/asset_context.py:61) object (which includes the "MAP_" prefixed `internal_map_type`). It uses the `get_filename_friendly_map_type` utility to convert this to a "standard type" (e.g., "COL") for output naming. Returns a [`SaveVariantsOutput`](processing/pipeline/asset_context.py:79) object containing details about the saved temporary files. The orchestrator stores these details, including the original "MAP_" prefixed `internal_map_type`, in `context.processed_maps_details` for the item. + +### Post-Item Stages + +These stages are executed sequentially once for each asset after the core item processing loop has finished for all items. + +1. **[`OutputOrganizationStage`](processing/pipeline/stages/output_organization.py:14)** (`processing/pipeline/stages/output_organization.py`): + * **Responsibility**: Determines the final output paths for all processed maps (including variants) and extra files based on configured patterns. It copies the temporary files generated by the core stages to these final destinations, creating directories as needed and respecting overwrite settings. + * **Context Interaction**: Reads from `context.processed_maps_details`, `context.files_to_process` (for 'EXTRA' files), `context.output_base_path`, and [`Configuration`](configuration.py:68). Updates entries in `context.processed_maps_details` with organization status. Populates `context.asset_metadata['maps']` with the final map structure: + * The `maps` object is a dictionary where keys are standard map types (e.g., "COL", "REFL"). + * Each entry contains a `variant_paths` dictionary, where keys are resolution strings (e.g., "8K", "4K") and values are the filenames of the map variants (relative to the asset's output directory). + It also populates `context.asset_metadata['final_output_files']` with a list of absolute paths to all generated files (this list itself is not saved in the final `metadata.json`). + +2. **[`MetadataFinalizationAndSaveStage`](processing/pipeline/stages/metadata_finalization_save.py:14)** (`processing/pipeline/stages/metadata_finalization_save.py`): + * **Responsibility**: Finalizes the `context.asset_metadata` (setting final status based on flags). It determines the save path for the metadata file based on configuration and patterns, serializes the `context.asset_metadata` (which now contains the structured `maps` data from `OutputOrganizationStage`) to JSON, and saves the `metadata.json` file. + * **Context Interaction**: Reads from `context.asset_metadata` (including the `maps` structure), `context.output_base_path`, and [`Configuration`](configuration.py:68). Before saving, it explicitly removes the `final_output_files` key from `context.asset_metadata`. The `processing_end_time` is also no longer added. The `metadata.json` file is written, and `context.asset_metadata` is updated with its final path and status. The older `processed_maps_details` and `merged_maps_details` from the context are not directly included in the JSON. + +## External Steps + +Certain steps are integral to the overall asset processing workflow but are handled outside the [`PipelineOrchestrator`](processing/pipeline/orchestrator.py:36)'s direct execution loop: + +* **Workspace Preparation and Cleanup**: Handled by the code that invokes [`ProcessingEngine.process()`](processing_engine.py:131) (e.g., `main.ProcessingTask`, `monitor._process_archive_task`), typically involving extracting archives and setting up temporary directories. The engine itself manages a sub-temporary directory (`engine_temp_dir`) for intermediate processing files. +* **Prediction and Rule Generation**: Performed before the [`ProcessingEngine`](processing_engine.py:73) is called. This involves analyzing source files and generating the [`SourceRule`](rule_structure.py:40) object with its nested [`AssetRule`](rule_structure.py:22)s and [`FileRule`](rule_structure.py:5)s, often involving prediction logic (potentially using LLMs). +* **Optional Blender Script Execution**: Can be triggered externally after successful processing to perform tasks like material setup in Blender using the generated output files and metadata. + +This staged pipeline provides a modular and extensible architecture for asset processing, with clear separation of concerns for each step. The [`AssetProcessingContext`](processing/pipeline/asset_context.py:86) ensures that data flows consistently between these stages. \ No newline at end of file diff --git a/Documentation/02_Developer_Guide/06_GUI_Internals.md b/Documentation/02_Developer_Guide/06_GUI_Internals.md index 28c7323..da90018 100644 --- a/Documentation/02_Developer_Guide/06_GUI_Internals.md +++ b/Documentation/02_Developer_Guide/06_GUI_Internals.md @@ -8,98 +8,214 @@ The GUI is built using `PySide6`, which provides Python bindings for the Qt fram ## Main Window (`gui/main_window.py`) -The `MainWindow` class is the central component of the GUI application. It is responsible for: +The `MainWindow` class acts as the central **coordinator** for the GUI application. It is responsible for: -* Defining the main application window structure and layout using PySide6 widgets. -* Arranging the Preset Editor panel (left) and the Processing panel (right). -* Setting up the menu bar, including the "View" menu for toggling the Log Console and Detailed File Preview. -* Connecting user interactions (button clicks, drag-and-drop events, checkbox states, spinbox values) to corresponding methods (slots) within the `MainWindow` or other handler classes. -* Managing the display of application logs in the UI console using a custom `QtLogHandler`. -* Interacting with background handlers (`ProcessingHandler`, `PredictionHandler`) via Qt signals and slots to ensure thread-safe updates to the UI during long-running operations. +* Setting up the main application window structure and menu bar, including actions to launch configuration and definition editors. +* **Layout:** Arranging the main GUI components using a `QSplitter`. + * **Left Pane:** Contains the preset selection controls (from `PresetEditorWidget`) permanently displayed at the top. Below this, a `QStackedWidget` switches between the preset JSON editor (also from `PresetEditorWidget`) and the `LLMEditorWidget`. + * **Right Pane:** Contains the `MainPanelWidget`. +* Instantiating and managing the major GUI widgets: + * `PresetEditorWidget` (`gui/preset_editor_widget.py`): Provides the preset selector and the JSON editor parts. + * `LLMEditorWidget` (`gui/llm_editor_widget.py`): Provides the editor for LLM settings (from `config/llm_settings.json`). + * `MainPanelWidget` (`gui/main_panel_widget.py`): Contains the rule hierarchy view and processing controls. + * `LogConsoleWidget` (`gui/log_console_widget.py`): Displays application logs. +* Instantiating key models and handlers: + * `UnifiedViewModel` (`gui/unified_view_model.py`): The model for the rule hierarchy view. + * `LLMInteractionHandler` (`gui/llm_interaction_handler.py`): Manages communication with the LLM service. + * `AssetRestructureHandler` (`gui/asset_restructure_handler.py`): Handles rule restructuring. +* Connecting signals and slots between these components to orchestrate the application flow. +* **Editor Switching:** Handling the `preset_selection_changed_signal` from `PresetEditorWidget` in its `_on_preset_selection_changed` slot. This slot: + * Switches the `QStackedWidget` (`editor_stack`) to display either the `PresetEditorWidget`'s JSON editor or the `LLMEditorWidget` based on the selected mode ("preset", "llm", "placeholder"). + * Calls `llm_editor_widget.load_settings()` when switching to LLM mode. + * Updates the window title. + * Triggers `update_preview()`. +* Handling top-level user interactions like drag-and-drop for loading sources (`add_input_paths`). This method now handles the "placeholder" state (no preset selected) by scanning directories or inspecting archives (ZIP) and creating placeholder `SourceRule`/`AssetRule`/`FileRule` objects to immediately populate the `UnifiedViewModel` with the file structure. +* Initiating predictions based on the selected preset mode (Rule-Based or LLM) when presets change or sources are added (`update_preview`). +* Starting the processing task (`_on_process_requested`): This slot now filters the `SourceRule` list obtained from the `UnifiedViewModel`, excluding sources where no asset has a `Target Asset` name assigned, before emitting the `start_backend_processing` signal. It also manages enabling/disabling controls. +* Managing the background prediction threads (`RuleBasedPredictionHandler` via `QThread`, `LLMPredictionHandler` via `LLMInteractionHandler`). +* Implementing slots to handle results from background tasks: + * `_on_rule_hierarchy_ready`: Handles results from `RuleBasedPredictionHandler`. + * `_on_llm_prediction_ready_from_handler`: Handles results from `LLMInteractionHandler`. + * `_on_prediction_error`: Handles errors from both prediction paths. + * `_handle_prediction_completion`: Centralized logic to track completion and update UI state after each prediction result or error. + * Slots to handle status and state changes from `LLMInteractionHandler`. ## Threading and Background Tasks -To keep the UI responsive during intensive operations like asset processing and file preview generation, the GUI utilizes background threads managed by `QThread`. +To keep the UI responsive, prediction tasks run in background threads managed by a `QThreadPool`. -* **`ProcessingHandler` (`gui/processing_handler.py`):** This class is designed to run in a separate `QThread`. It manages the execution of the main asset processing pipeline for multiple assets concurrently using `concurrent.futures.ProcessPoolExecutor`. It submits individual asset processing tasks to the pool and monitors their completion. It uses Qt signals to communicate progress updates, file status changes, and overall processing completion back to the `MainWindow` on the main UI thread. It also handles the execution of optional Blender scripts via subprocess calls after processing. This handler processes and utilizes data structures received from the core processing engine, such as status summaries. -* **`PredictionHandler` (`gui/prediction_handler.py`):** This class also runs in a separate `QThread`. It is responsible for generating the detailed file classification previews displayed in the preview table. It calls methods on the `AssetProcessor` (`get_detailed_file_predictions`) to perform the analysis in the background. It uses a `ThreadPoolExecutor` for potentially concurrent prediction tasks. Results are sent back to the `MainWindow` via Qt signals to update the preview table data. This handler works with data structures containing file prediction details. +* **`BasePredictionHandler` (`gui/base_prediction_handler.py`):** An abstract `QRunnable` base class defining the common interface and signals (`prediction_signal`, `status_signal`) for prediction tasks. +* **`RuleBasedPredictionHandler` (`gui/prediction_handler.py`):** Inherits from `BasePredictionHandler`. Runs as a `QRunnable` in the thread pool when a rule-based preset is selected. Generates the `SourceRule` hierarchy based on preset rules and emits `prediction_signal`. +* **`LLMPredictionHandler` (`gui/llm_prediction_handler.py`):** Inherits from `BasePredictionHandler`. Runs as a `QRunnable` in the thread pool when "- LLM Interpretation -" is selected. Interacts with `LLMInteractionHandler`, parses the response, generates the `SourceRule` hierarchy for a *single* input item, and emits `prediction_signal` and `status_signal`. +* **`LLMInteractionHandler` (`gui/llm_interaction_handler.py`):** Manages the communication with the LLM service. This handler itself may perform network operations but typically runs synchronously within the `LLMPredictionHandler`'s thread. + +*(Note: The actual processing via `ProcessingEngine` is now handled by `main.ProcessingTask`, which runs in a separate process managed outside the GUI's direct threading model, though the GUI initiates it).* ## Communication (Signals and Slots) -Communication between the main UI thread (`MainWindow`) and the background threads (`ProcessingHandler`, `PredictionHandler`) relies heavily on Qt's signals and slots mechanism. This is a thread-safe way for objects in different threads to communicate. +Communication between the `MainWindow` (main UI thread) and the background prediction tasks relies on Qt's signals and slots. -* Background handlers emit signals to indicate events (e.g., progress updated, file status changed, task finished). -* The `MainWindow` connects slots (methods) to these signals. When a signal is emitted, the connected slot is invoked on the thread that owns the receiving object (the main UI thread for `MainWindow`), ensuring UI updates happen safely. +* Prediction handlers (`RuleBasedPredictionHandler`, `LLMPredictionHandler`) emit signals from the `BasePredictionHandler`: + * `prediction_signal(source_id, source_rule_list)`: Indicates prediction for a source is complete. + * `status_signal(message)`: Provides status updates (primarily from LLM handler). +* The `MainWindow` connects slots to these signals: + * `prediction_signal` -> `MainWindow._handle_prediction_completion(source_id, source_rule_list)` + * `status_signal` -> `MainWindow._on_status_update(message)` (updates status bar) +* Signals from the `UnifiedViewModel` (`dataChanged`, `layoutChanged`) trigger updates in the `QTreeView`. +* Signals from the `UnifiedViewModel` (`targetAssetOverrideChanged`) trigger the `AssetRestructureHandler`. -## Preset Editor +## Preset Editor (`gui/preset_editor_widget.py`) -The GUI includes an integrated preset editor panel. This allows users to interactively create, load, modify, and save preset `.json` files directly within the application. The editor typically uses standard UI widgets to display and edit the key fields of the preset structure. +The `PresetEditorWidget` provides a dedicated interface for managing presets. It handles loading, displaying, editing, and saving preset `.json` files. -## Preview Table +* **Refactoring:** This widget has been refactored to expose its main components: + * `selector_container`: A `QWidget` containing the preset list (`QListWidget`) and New/Delete buttons. Used statically by `MainWindow`. + * `json_editor_container`: A `QWidget` containing the tabbed editor (`QTabWidget`) for preset JSON details and the Save/Save As buttons. Placed in `MainWindow`'s `QStackedWidget`. +* **Functionality:** Still manages the logic for populating the preset list, loading/saving presets, handling unsaved changes, and providing the editor UI for preset details. +* **Communication:** Emits `preset_selection_changed_signal(mode, preset_name)` when the user selects a preset, the LLM option, or the placeholder. This signal is crucial for `MainWindow` to switch the editor stack and trigger preview updates. +## LLM Settings Editor (`gui/llm_editor_widget.py`) -The `PreviewTableModel` receives a list of file prediction dictionaries from the `PredictionHandler` via the `prediction_results_ready` signal. This list contains dictionaries for each file with details such as original path, predicted asset name, status, and other relevant information. +This new widget provides a dedicated interface for editing LLM-specific settings stored in `config/llm_settings.json`. -The `PreviewTableModel` is designed to process and display this file prediction data. Instead of directly displaying the flat list, it processes and transforms the data into a structured list of rows (`self._table_rows`). This transformation involves: +* **Purpose:** Allows users to configure the LLM predictor's behavior without directly editing the JSON file. +* **Structure:** Uses a `QTabWidget` with two tabs: + * **"Prompt Settings":** Contains a `QPlainTextEdit` for the main prompt and a nested `QTabWidget` for managing examples (add/delete/edit JSON in `QTextEdit` widgets). + * **"API Settings":** Contains fields (`QLineEdit`, `QDoubleSpinBox`, `QSpinBox`) for endpoint URL, API key, model name, temperature, and timeout. +* **Functionality:** + * `load_settings()`: Reads `config/llm_settings.json` and populates the UI fields. Handles file not found or JSON errors. Called by `MainWindow` when switching to LLM mode. + * `_save_settings()`: Gathers data from the UI, validates example JSON, constructs the settings dictionary, and calls `configuration.save_llm_config()` to write back to the file. Emits `settings_saved` signal on success. + * Manages unsaved changes state and enables/disables the "Save LLM Settings" button accordingly. -1. **Grouping:** Files are grouped based on their `source_asset`. -2. **Separation:** Within each asset group, files are separated into `main_files` (Mapped, Model, Error) and `additional_files` (Ignored, Extra, Unrecognised, Unmatched Extra). -3. **Structuring Rows:** Rows are created for `self._table_rows` to represent the grouped data. Each row can contain information about a main file and/or an additional file, allowing for the display of additional files in a separate column aligned with the main files of the same asset. Empty rows are created if there are more additional files than main files for an asset to maintain alignment. +## Unified Hierarchical View -The `data()` method of the `PreviewTableModel` then accesses this structured `self._table_rows` list to provide data to the `QTableView` for display. It handles different columns and roles (Display, Tooltip, Foreground, and Background). +The core rule editing interface is built around a `QTreeView` managed within the `MainPanelWidget`, using a custom model and delegates. -* `Qt.ItemDataRole.ForegroundRole`: Used to set the text color of individual cells based on the status of the file they represent. Coloring is applied to cells corresponding to a main file based on the main file's status, and to cells in the "Additional Files" column based on the additional file's status. -* `Qt.ItemDataRole.BackgroundRole`: Used to provide alternating background colors based on the index of the asset group the row belongs to in a sorted list of unique assets, improving visual separation between different asset groups. +* **`UnifiedViewModel` (`gui/unified_view_model.py`):** Implements `QAbstractItemModel`. + * Wraps the `RuleHierarchyModel` to expose the `SourceRule` list (Source -> Asset -> File) to the `QTreeView`. + * Provides data for display and flags for editing. + * **Handles `setData` requests:** Validates input and updates the underlying `RuleHierarchyModel`. Crucially, it **delegates** complex restructuring (when `target_asset_name_override` changes) to the `AssetRestructureHandler` by emitting the `targetAssetOverrideChanged` signal. + * **Row Coloring:** Provides data for `Qt.ForegroundRole` (text color) based on the `item_type` and the colors defined in `config/app_settings.json`. Provides data for `Qt.BackgroundRole` based on calculating a 30% darker shade of the parent asset's background color. + * **Caching:** Caches configuration data (`ASSET_TYPE_DEFINITIONS`, `FILE_TYPE_DEFINITIONS`, color maps) in `__init__` for performance. + * **`update_rules_for_sources` Method:** Intelligently merges new prediction results or placeholder rules into the existing model data, preserving user overrides where applicable. + * *(Note: The previous concept of switching between "simple" and "detailed" display modes has been removed. The model always represents the full detailed structure.)* +* **`RuleHierarchyModel` (`gui/rule_hierarchy_model.py`):** A non-Qt model holding the actual list of `SourceRule` objects. Provides methods for accessing and modifying the hierarchy (used by `UnifiedViewModel` and `AssetRestructureHandler`). +* **`AssetRestructureHandler` (`gui/asset_restructure_handler.py`):** Contains the logic to modify the `RuleHierarchyModel` when a file's target asset is changed. It listens for the `targetAssetOverrideChanged` signal from the `UnifiedViewModel` and uses methods on the `RuleHierarchyModel` (`moveFileRule`, `createAssetRule`, `removeAssetRule`) to perform the restructuring safely. +* **`Delegates` (`gui/delegates.py`):** Custom `QStyledItemDelegate` implementations provide inline editors: + * **`ComboBoxDelegate`:** For selecting predefined types (from `Configuration`). + * **`LineEditDelegate`:** For free-form text editing. + * **`SupplierSearchDelegate`:** For supplier names with auto-completion (using `config/suppliers.json`). -The `PreviewSortFilterProxyModel` operates on this structured data, implementing a multi-level sort based on source asset, row type (main vs. additional-only), and file paths within those types. - - -### Preview Table Column Configuration - -The display and behavior of the columns in the `QTableView` are configured in `gui/main_window.py`. The current configuration is as follows: - -* **Column Order (from left to right):** - 1. Status - 2. Predicted Asset - 3. Details - 4. Original Path - 5. Additional Files -* **Column Resizing:** - * Status: Resizes to content. - * Predicted Asset: Resizes to content. - * Details: Resizes to content. - * Original Path: Resizes to content (fixed width behavior). - * Additional Files: Stretches to fill available space. - -**Data Flow Diagram:** +**Data Flow Diagram (GUI Rule Management - Refactored):** ```mermaid -graph LR - A[PredictionHandler] -- prediction_results_ready(flat_list) --> B(PreviewTableModel); - subgraph PreviewTableModel - C[set_data] -- Processes flat_list --> D{Internal Grouping & Transformation}; - D -- Creates --> E[_table_rows (Structured List)]; - F[data()] -- Reads from --> E; +graph TD + subgraph MainWindow [MainWindow Coordinator] + direction LR + MW_Input[User Input (Drag/Drop)] --> MW(MainWindow); + MW -- Owns/Manages --> Splitter(QSplitter); + MW -- Owns/Manages --> LLMIH(LLMInteractionHandler); + MW -- Owns/Manages --> ARH(AssetRestructureHandler); + MW -- Owns/Manages --> VM(UnifiedViewModel); + MW -- Owns/Manages --> LCW(LogConsoleWidget); + MW -- Initiates --> PredPool{Prediction Threads}; + MW -- Connects Signals --> VM; + MW -- Connects Signals --> ARH; + MW -- Connects Signals --> LLMIH; + MW -- Connects Signals --> PEW(PresetEditorWidget); + MW -- Connects Signals --> LLMEDW(LLMEditorWidget); end - B -- Provides data via data() --> G(QTableView via Proxy); - style B fill:#f9f,stroke:#333,stroke-width:2px - style C fill:#ccf,stroke:#333,stroke-width:1px - style D fill:#lightgrey,stroke:#333,stroke-width:1px - style E fill:#ccf,stroke:#333,stroke-width:1px - style F fill:#ccf,stroke:#333,stroke-width:1px + subgraph LeftPane [Left Pane Widgets] + direction TB + Splitter -- Adds Widget --> LPW(Left Pane Container); + LPW -- Contains --> PEW_Sel(PresetEditorWidget - Selector); + LPW -- Contains --> Stack(QStackedWidget); + Stack -- Contains --> PEW_Edit(PresetEditorWidget - JSON Editor); + Stack -- Contains --> LLMEDW; + end + + subgraph RightPane [Right Pane Widgets] + direction TB + Splitter -- Adds Widget --> MPW(MainPanelWidget); + MPW -- Contains --> TV(QTreeView - Rule View); + MPW_UI[UI Controls (Process Btn, etc)]; + MPW_UI --> MPW; + end + + subgraph Prediction [Background Prediction] + direction TB + PredPool -- Runs --> RBP(RuleBasedPredictionHandler); + PredPool -- Runs --> LLMP(LLMPredictionHandler); + LLMIH -- Manages/Starts --> LLMP; + RBP -- prediction_ready/error/status --> MW; + LLMIH -- llm_prediction_ready/error/status --> MW; + end + + subgraph ModelView [Model/View Components] + direction TB + TV -- Sets Model --> VM; + TV -- Displays Data From --> VM; + TV -- Uses Delegates --> Del(Delegates); + UserEdit[User Edits Rules] --> TV; + TV -- setData --> VM; + VM -- Wraps --> RHM(RuleHierarchyModel); + VM -- dataChanged/layoutChanged --> TV; + VM -- targetAssetOverrideChanged --> ARH; + ARH -- Modifies --> RHM; + Del -- Get/Set Data --> VM; + end + + %% MainWindow Interactions + MW_Input -- Triggers --> MW; + PEW -- preset_selection_changed_signal --> MW; + LLMEDW -- settings_saved --> MW; + MPW -- process_requested/etc --> MW; + MW -- _on_preset_selection_changed --> Stack; + MW -- _on_preset_selection_changed --> LLMEDW; + MW -- _handle_prediction_completion --> VM; + MW -- Triggers Processing --> ProcTask(main.ProcessingTask); + + %% Connections between subgraphs + PEW --> LPW; %% PresetEditorWidget parts are in Left Pane + LLMEDW --> Stack; %% LLMEditorWidget is in Stack + MPW --> Splitter; %% MainPanelWidget is in Right Pane + VM --> MW; + ARH --> MW; + LLMIH --> MW; + LCW --> MW; ``` -### Application Styling +## Application Styling -The application style is explicitly set to 'Fusion' in `gui/main_window.py` to provide a more consistent look and feel across different operating systems, particularly to address styling inconsistencies observed on Windows 11. A custom `QPalette` is also applied to the application to adjust default colors within the 'Fusion' style, specifically to change the background color of list-like widgets and potentially other elements from a default dark blue to a more neutral grey. +The application style is explicitly set to 'Fusion' in `gui/main_window.py`. A custom `QPalette` adjusts default colors. -## Logging +## Logging (`gui/log_console_widget.py`) -A custom `QtLogHandler` is used to redirect log messages from the standard Python `logging` module to a text area or console widget within the GUI, allowing users to see detailed application output and errors. +The `LogConsoleWidget` displays logs captured by a custom `QtLogHandler` from Python's `logging` module. ## Cancellation -The GUI provides a "Cancel" button to stop ongoing processing. The `ProcessingHandler` implements logic to handle cancellation requests. This typically involves setting an internal flag and attempting to shut down the `ProcessPoolExecutor`. However, it's important to note that this does not immediately terminate worker processes that are already executing; it primarily prevents new tasks from starting and stops processing results from completed futures once the cancellation flag is checked. +The GUI provides a "Cancel" button. Cancellation logic for the actual processing is now likely handled within the `main.ProcessingTask` or the code that manages it, as the `ProcessingHandler` has been removed. The GUI button would signal this external task manager. -**Note on Data Passing:** As mentioned in the Architecture documentation, major changes to the data passing mechanisms between the GUI, Main (CLI orchestration), and `AssetProcessor` modules are currently being planned. The descriptions of how data is handled and passed within the GUI and its interactions with background handlers reflect the current state and will require review and updates once the plan for these changes is finalized. \ No newline at end of file +## Application Preferences Editor (`gui/config_editor_dialog.py`) + +A dedicated dialog for editing user-overridable application settings. It loads base settings from `config/app_settings.json` and saves user overrides to `config/user_settings.json`. + +* **Functionality:** Provides a tabbed interface to edit various application settings, including general paths, output/naming patterns, image processing options (like resolutions and compression), and map merging rules. It no longer includes editors for Asset Type or File Type Definitions. +* **Integration:** Launched by `MainWindow` via the "Edit" -> "Preferences..." menu. +* **Persistence:** Saves changes to `config/user_settings.json`. Changes require an application restart to take effect in processing logic. + +The refactored GUI separates concerns into distinct widgets and handlers, coordinated by the `MainWindow`. Background tasks use `QThreadPool` and `QRunnable`. The `UnifiedViewModel` focuses on data presentation and simple edits, delegating complex restructuring to the `AssetRestructureHandler`. + +## Definitions Editor (`gui/definitions_editor_dialog.py`) + +A new dedicated dialog for managing core application definitions that are separate from general user preferences. + +* **Purpose:** Provides a structured UI for editing Asset Type Definitions, File Type Definitions, and Supplier Settings. +* **Structure:** Uses a `QTabWidget` with three tabs: + * **Asset Type Definitions:** Manages definitions from `config/asset_type_definitions.json`. Presents a list of asset types and allows editing their description, color, and examples. + * **File Type Definitions:** Manages definitions from `config/file_type_definitions.json`. Presents a list of file types and allows editing their description, color, examples, standard type, bit depth rule, grayscale status, and keybind. + * **Supplier Settings:** Manages settings from `config/suppliers.json`. Presents a list of suppliers and allows editing supplier-specific settings (e.g., Normal Map Type). +* **Integration:** Launched by `MainWindow` via the "Edit" -> "Edit Definitions..." menu. +* **Persistence:** Saves changes directly to the respective configuration files (`config/asset_type_definitions.json`, `config/file_type_definitions.json`, `config/suppliers.json`). Some changes may require an application restart. \ No newline at end of file diff --git a/Documentation/02_Developer_Guide/07_Monitor_Internals.md b/Documentation/02_Developer_Guide/07_Monitor_Internals.md index 1694099..6ddeaaa 100644 --- a/Documentation/02_Developer_Guide/07_Monitor_Internals.md +++ b/Documentation/02_Developer_Guide/07_Monitor_Internals.md @@ -4,30 +4,41 @@ This document provides technical details about the implementation of the Directo ## Overview -The `monitor.py` script provides an automated way to process assets by monitoring a specified input directory for new ZIP files. It is built using the `watchdog` library. +The `monitor.py` script provides an automated way to process assets by monitoring a specified input directory for new archive files. It has been refactored to use a `ThreadPoolExecutor` for asynchronous processing. ## Key Components -* **`watchdog` Library:** The script relies on the `watchdog` library for monitoring file system events. Specifically, it uses a `PollingObserver` to watch the `INPUT_DIR` for changes. -* **`ZipHandler` Class:** This is a custom event handler class defined within `monitor.py`. It inherits from a `watchdog` event handler class (likely `FileSystemEventHandler` or similar, though not explicitly stated in the source text, it's the standard pattern). Its primary method of interest is the one that handles file creation events (`on_created`). -* **`main.run_processing`:** The monitor script triggers the main asset processing logic by calling the `run_processing` function from the `main.py` module. +* **`watchdog` Library:** Used for monitoring file system events (specifically file creation) in the `INPUT_DIR`. A `PollingObserver` is typically used. +* **`concurrent.futures.ThreadPoolExecutor`:** Manages a pool of worker threads to process detected archives concurrently. The number of workers can often be configured (e.g., via `NUM_WORKERS` environment variable). +* **`_process_archive_task` Function:** The core function executed by the thread pool for each detected archive. It encapsulates the entire processing workflow for a single archive. +* **`utils.prediction_utils.generate_source_rule_from_archive`:** A utility function called by `_process_archive_task` to perform rule-based prediction directly on the archive file and generate the necessary `SourceRule` object. +* **`utils.workspace_utils.prepare_processing_workspace`:** A utility function called by `_process_archive_task` to create a temporary workspace and extract the archive contents into it. +* **`ProcessingEngine` (`processing_engine.py`):** The core engine instantiated and run within `_process_archive_task` to perform the actual asset processing based on the generated `SourceRule`. +* **`Configuration` (`configuration.py`):** Loaded within `_process_archive_task` based on the preset derived from the archive filename. -## Functionality Details +## Functionality Details (Asynchronous Workflow) -1. **Watching:** A `PollingObserver` is set up to monitor the directory specified by the `INPUT_DIR` environment variable. Polling is used, checking for changes at a frequency defined by `POLL_INTERVAL`. -2. **Event Handling:** The `ZipHandler` is attached to the observer. When a file is created in the monitored directory, the `on_created` method of the `ZipHandler` is triggered. -3. **ZIP File Detection:** The `on_created` method checks if the newly created file is a `.zip` file. -4. **Filename Parsing:** If it's a ZIP file, the script expects the filename to follow a specific format: `[preset]_filename.zip`. It uses a regular expression (`PRESET_FILENAME_REGEX`, likely defined in `config.py` or similar) to extract the `[preset]` part from the filename. -5. **Preset Validation:** It validates whether the extracted `preset` name corresponds to an existing preset JSON file in the `Presets/` directory. -6. **Triggering Processing:** If the preset is valid, the `monitor.py` script calls `main.run_processing`, passing the path to the detected ZIP file and the extracted preset name. This initiates the main asset processing pipeline for that single asset. A `PROCESS_DELAY` can be configured to wait before triggering processing, potentially allowing large files to finish copying. -7. **Source ZIP Management:** After the processing initiated by `main.run_processing` completes, the original source `.zip` file is moved to either the `PROCESSED_DIR` (if processing was successful or skipped) or the `ERROR_DIR` (if processing failed or the preset was invalid). +1. **Watching:** A `watchdog` observer monitors the `INPUT_DIR` for file creation events. +2. **Event Handling:** When a file is created, an event handler (e.g., `on_created` method) is triggered. +3. **Archive Detection:** The handler checks if the new file is a supported archive type (e.g., `.zip`, `.rar`, `.7z`). +4. **Filename Parsing:** If it's a supported archive, the script attempts to parse the filename to extract the intended preset name (e.g., using a regex like `[preset]_filename.ext`). +5. **Preset Validation:** The extracted preset name is validated against existing preset files (`Presets/*.json`). +6. **Task Submission:** If the preset is valid, the path to the archive file and the validated preset name are submitted as a task to the `ThreadPoolExecutor`, which will eventually run the `_process_archive_task` function with these arguments in a worker thread. +7. **`_process_archive_task` Execution (Worker Thread):** + * **Load Configuration:** Loads the `Configuration` object using the provided preset name. + * **Generate SourceRule:** Calls `utils.prediction_utils.generate_source_rule_from_archive`, passing the archive path and `Configuration`. This utility handles temporary extraction (if needed internally) and rule-based prediction, returning the `SourceRule`. + * **Prepare Workspace:** Calls `utils.workspace_utils.prepare_processing_workspace`, passing the archive path. This creates a unique temporary directory and extracts the archive contents. It returns the path to the prepared workspace. This step should ideally be wrapped in a `try...finally` block to ensure cleanup. + * **Instantiate Engine:** Creates an instance of the `ProcessingEngine`, passing the loaded `Configuration` and the prepared workspace path. + * **Run Processing:** Calls the `ProcessingEngine.process()` method, passing the generated `SourceRule`. + * **Handle Results:** Based on the success or failure of the processing, moves the original source archive file from `INPUT_DIR` to either `PROCESSED_DIR` or `ERROR_DIR`. + * **Cleanup Workspace:** Ensures the temporary workspace directory created by `prepare_processing_workspace` is removed (e.g., in the `finally` block). ## Configuration -The monitor's behavior is primarily controlled by environment variables, which are read by the `monitor.py` script. These include `INPUT_DIR`, `OUTPUT_DIR`, `PROCESSED_DIR`, `ERROR_DIR`, `LOG_LEVEL`, `POLL_INTERVAL`, and `NUM_WORKERS`. +The monitor's behavior is controlled by environment variables or configuration settings, likely including `INPUT_DIR`, `OUTPUT_DIR`, `PROCESSED_DIR`, `ERROR_DIR`, `LOG_LEVEL`, `POLL_INTERVAL`, and potentially `NUM_WORKERS` to control the size of the `ThreadPoolExecutor`. ## Limitations -* The current implementation of the directory monitor does *not* support triggering the optional Blender script execution after processing. This post-processing step is only available when running the tool via the CLI or GUI. +* The monitor likely still does *not* support triggering optional Blender script execution post-processing, as this integration point was complex and potentially removed or not yet reimplemented in the refactored workflow. -Understanding the interaction between `watchdog`, the `ZipHandler`, and the call to `main.run_processing` is key to debugging or modifying the directory monitoring functionality. \ No newline at end of file +Understanding the asynchronous nature, the role of the `ThreadPoolExecutor`, the `_process_archive_task` function, and the reliance on utility modules (`prediction_utils`, `workspace_utils`) is key to debugging or modifying the directory monitoring functionality. \ No newline at end of file diff --git a/Documentation/02_Developer_Guide/10_Coding_Conventions.md b/Documentation/02_Developer_Guide/10_Coding_Conventions.md index cd911fd..d89831d 100644 --- a/Documentation/02_Developer_Guide/10_Coding_Conventions.md +++ b/Documentation/02_Developer_Guide/10_Coding_Conventions.md @@ -20,10 +20,10 @@ This document outlines the coding conventions and general practices followed wit * Use Qt's signals and slots mechanism for communication between objects, especially across threads. * Run long-running or blocking tasks in separate `QThread`s to keep the main UI thread responsive. * Perform UI updates only from the main UI thread. -* **Configuration:** Core settings are managed in `config.py` (Python module). Supplier-specific rules are managed in JSON files (`Presets/`). The `Configuration` class handles loading and merging these. +* **Configuration:** Core application settings are defined in `config/app_settings.json`. Supplier-specific rules are managed in JSON files within the `Presets/` directory. The `Configuration` class (`configuration.py`) is responsible for loading `app_settings.json` and merging it with the selected preset file. * **File Paths:** Use `pathlib.Path` objects for handling file system paths. Avoid using string manipulation for path joining or parsing. * **Docstrings:** Write clear and concise docstrings for modules, classes, methods, and functions, explaining their purpose, arguments, and return values. -* **Comments:** Use comments to explain complex logic or non-obvious parts of the code. +* **Comments:** Use comments to explain complex logic or non-obvious parts of the code. Avoid obsolete comments (e.g., commented-out old code) and redundant comments (e.g., comments stating the obvious, like `# Import module` or `# Initialize variable`). The goal is to maintain clarity while minimizing unnecessary token usage for LLM tools. * **Imports:** Organize imports at the top of the file, grouped by standard library, third-party libraries, and local modules. * **Naming:** * Use `snake_case` for function and variable names. @@ -31,4 +31,51 @@ This document outlines the coding conventions and general practices followed wit * Use `UPPER_CASE` for constants. * Use a leading underscore (`_`) for internal or "protected" methods/attributes. +## Terminology and Data Standards + +To ensure consistency and clarity across the codebase, particularly concerning asset and file classifications, the following standards must be adhered to. These primarily revolve around definitions stored in `config/app_settings.json`. + +### `FILE_TYPE_DEFINITIONS` + +`FILE_TYPE_DEFINITIONS` in `config/app_settings.json` is the **single source of truth** for all file type identifiers used within the application. + +* **`FileRule.item_type` and `FileRule.item_type_override`**: When defining or interpreting `SourceRule` objects (and their constituent `FileRule` instances), the `item_type` and `item_type_override` attributes **must** always use a key directly from `FILE_TYPE_DEFINITIONS`. + * Example: `file_rule.item_type = "MAP_COL"` (for a color map) or `file_rule.item_type = "MODEL_FBX"` (for an FBX model). + +* **`standard_type` Property**: Each entry in `FILE_TYPE_DEFINITIONS` includes a `standard_type` property. This provides a common, often abbreviated, alias for the file type. + * Example: `FILE_TYPE_DEFINITIONS["MAP_COL"]["standard_type"]` might be `"COL"`. + * Example: `FILE_TYPE_DEFINITIONS["MAP_NORM_GL"]["standard_type"]` might be `"NRM"`. + +* **Removal of `STANDARD_MAP_TYPES`**: The global constant `STANDARD_MAP_TYPES` (previously in `config.py`) has been **removed**. Standard map type aliases (e.g., "COL", "NRM", "RGH") are now derived dynamically from the `standard_type` property of the relevant entry in `FILE_TYPE_DEFINITIONS`. + +* **`map_type` Usage**: + * **Filename Tokens**: When used as a token in output filename patterns (e.g., `[maptype]`), `map_type` is typically derived from the `standard_type` of the file's effective `item_type`. It may also include a variant suffix if applicable (e.g., "COL", "COL_var01"). + * **General Classification**: For precise classification within code logic or rules, developers should refer to the full `FILE_TYPE_DEFINITIONS` key (e.g., `"MAP_COL"`, `"MAP_METAL"`). The `standard_type` can be used for broader categorization or when a common alias is needed. + +* **`Map_type_Mapping.target_type` in Presets**: Within preset files (e.g., `Presets/Poliigon.json`), the `map_type_mapping` rules found inside a `FileRule`'s `map_processing_options` now use keys from `FILE_TYPE_DEFINITIONS` for the `target_type` field. + * Example: + ```json + // Inside a FileRule in a preset + "map_processing_options": { + "map_type_mapping": { + "source_type_pattern": ".*ambient occlusion.*", + "target_type": "MAP_AO", // Uses FILE_TYPE_DEFINITIONS key + "source_channels": "RGB" + } + } + ``` + This replaces old aliases like `"AO"` or `"OCC"`. + +* **`is_grayscale` Property**: `FILE_TYPE_DEFINITIONS` entries can now include an `is_grayscale` boolean property. This flag indicates whether the file type is inherently grayscale (e.g., a roughness map). It can be used by the processing engine to inform decisions about channel handling, compression, or specific image operations. + * Example: `FILE_TYPE_DEFINITIONS["MAP_RGH"]["is_grayscale"]` might be `true`. + +### `ASSET_TYPE_DEFINITIONS` + +Similarly, `ASSET_TYPE_DEFINITIONS` in `config/app_settings.json` is the **single source of truth** for all asset type identifiers. + +* **`AssetRule.asset_type`, `AssetRule.asset_type_override`, and `AssetRule.asset_category`**: When defining or interpreting `SourceRule` objects (and their constituent `AssetRule` instances), the `asset_type`, `asset_type_override`, and `asset_category` attributes **must** always use a key directly from `ASSET_TYPE_DEFINITIONS`. + * Example: `asset_rule.asset_type = "SURFACE_3D"` or `asset_rule.asset_category = "FABRIC"`. + +Adherence to these definitions ensures that terminology remains consistent throughout the application, from configuration to core logic and output. + Adhering to these conventions will make the codebase more consistent, easier to understand, and more maintainable for all contributors. \ No newline at end of file diff --git a/Documentation/02_Developer_Guide/12_LLM_Predictor_Integration.md b/Documentation/02_Developer_Guide/12_LLM_Predictor_Integration.md new file mode 100644 index 0000000..f23ff6d --- /dev/null +++ b/Documentation/02_Developer_Guide/12_LLM_Predictor_Integration.md @@ -0,0 +1,123 @@ +# LLM Predictor Integration + +## Overview + +The LLM Predictor feature provides an alternative method for classifying asset textures using a Large Language Model (LLM). This allows for more flexible and potentially more accurate classification compared to traditional rule-based methods, especially for diverse or complex asset names. + +## Configuration + +The LLM Predictor is configured via settings in the dedicated `config/llm_settings.json` file. These settings control the behavior of the LLM interaction: + +- `llm_predictor_prompt`: The template for the prompt sent to the LLM. This prompt should guide the LLM to classify the asset based on its name and potentially other context. It can include placeholders that will be replaced with actual data during processing. +- `llm_endpoint_url`: The URL of the LLM API endpoint. +- `llm_api_key`: The API key required for authentication with the LLM endpoint. +- `llm_model_name`: The name of the specific LLM model to be used for prediction. +- `llm_temperature`: Controls the randomness of the LLM's output. A lower temperature results in more deterministic output, while a higher temperature increases creativity. +- `llm_request_timeout`: The maximum time (in seconds) to wait for a response from the LLM API. +- `llm_predictor_examples`: A list of example input/output pairs to include in the prompt for few-shot learning, helping the LLM understand the desired output format and classification logic. + +**Editing:** These settings can be edited directly through the GUI using the **`LLMEditorWidget`** (`gui/llm_editor_widget.py`), which provides a user-friendly interface for modifying the prompt, examples, and API parameters. Changes are saved back to `config/llm_settings.json` via the `configuration.save_llm_config()` function. + +**Loading:** The `LLMInteractionHandler` now loads these settings directly from `config/llm_settings.json` and relevant parts of `config/app_settings.json` when it needs to start an `LLMPredictionHandler` task. It no longer relies on the main `Configuration` class for LLM-specific settings. The prompt structure remains crucial for effective classification. Placeholders within the prompt template (e.g., `{FILE_LIST}`) are dynamically replaced with relevant data before the request is sent. + +## Expected LLM Output Format (Refactored) + +The LLM is now expected to return a JSON object containing two distinct parts. This structure helps the LLM maintain context across multiple files belonging to the same conceptual asset and allows for a more robust grouping mechanism. + +**Rationale:** The previous implicit format made it difficult for the LLM to consistently group related files (e.g., different texture maps for the same material) under a single asset, especially in complex archives. The new two-part structure explicitly separates file-level analysis from asset-level classification, improving accuracy and consistency. + +**Structure:** + +```json +{ + "individual_file_analysis": [ + { + "relative_file_path": "Textures/Wood_Floor_01/Wood_Floor_01_BaseColor.png", + "classified_file_type": "BaseColor", + "proposed_asset_group_name": "Wood_Floor_01" + }, + { + "relative_file_path": "Textures/Wood_Floor_01/Wood_Floor_01_Roughness.png", + "classified_file_type": "Roughness", + "proposed_asset_group_name": "Wood_Floor_01" + }, + { + "relative_file_path": "Textures/Metal_Plate_03/Metal_Plate_03_Metallic.jpg", + "classified_file_type": "Metallic", + "proposed_asset_group_name": "Metal_Plate_03" + } + ], + "asset_group_classifications": { + "Wood_Floor_01": "PBR Material", + "Metal_Plate_03": "PBR Material" + } +} +``` + +- **`individual_file_analysis`**: A list where each object represents a single file within the source. + - `relative_file_path`: The path of the file relative to the source root. + - `classified_file_type`: The LLM's prediction for the *type* of this specific file (e.g., "BaseColor", "Normal", "Model"). This corresponds to the `item_type` in the `FileRule`. + - `proposed_asset_group_name`: A name suggested by the LLM to group this file with others belonging to the same conceptual asset. This is used internally by the parser. +- **`asset_group_classifications`**: A dictionary mapping the `proposed_asset_group_name` values from the list above to a final `asset_type` (e.g., "PBR Material", "HDR Environment"). + +## `LLMInteractionHandler` (Refactored) + +The `gui/llm_interaction_handler.py` module contains the `LLMInteractionHandler` class, which now acts as the central manager for LLM prediction tasks. + +Key Responsibilities & Methods: + +- **Queue Management:** Maintains a queue (`llm_processing_queue`) of pending prediction requests (input path, file list). Handles adding single (`queue_llm_request`) or batch (`queue_llm_requests_batch`) requests. +- **State Management:** Tracks whether an LLM task is currently running (`_is_processing`) and emits `llm_processing_state_changed(bool)` to update the GUI (e.g., disable preset editor). Includes `force_reset_state()` for recovery. +- **Task Orchestration:** Processes the queue sequentially (`_process_next_llm_item`). For each item: + * Loads required settings directly from `config/llm_settings.json` and `config/app_settings.json`. + * Instantiates an `LLMPredictionHandler` in a new `QThread`. + * Passes the loaded settings dictionary to the `LLMPredictionHandler`. + * Connects signals from the handler (`prediction_ready`, `prediction_error`, `status_update`) to internal slots (`_handle_llm_result`, `_handle_llm_error`) or directly re-emits them (`llm_status_update`). + * Starts the thread. +- **Result/Error Handling:** Internal slots (`_handle_llm_result`, `_handle_llm_error`) receive results/errors from the `LLMPredictionHandler`, remove the completed/failed item from the queue, emit the corresponding public signal (`llm_prediction_ready`, `llm_prediction_error`), and trigger processing of the next queue item. +- **Communication:** Emits signals to `MainWindow`: + * `llm_prediction_ready(input_path, source_rule_list)` + * `llm_prediction_error(input_path, error_message)` + * `llm_status_update(status_message)` + * `llm_processing_state_changed(is_processing)` + +## `LLMPredictionHandler` (Refactored) + +The `gui/llm_prediction_handler.py` module contains the `LLMPredictionHandler` class (inheriting from `BasePredictionHandler`), which performs the actual LLM prediction for a *single* input source. It runs in a background thread managed by the `LLMInteractionHandler`. + +Key Responsibilities & Methods: + +- **Initialization**: Takes the source identifier, file list, and a **`settings` dictionary** (passed from `LLMInteractionHandler`) containing all necessary configuration (LLM endpoint, prompt, examples, API details, type definitions, etc.). +- **`_perform_prediction()`**: Implements the core prediction logic: + * **Prompt Preparation (`_prepare_prompt`)**: Uses the passed `settings` dictionary to access the prompt template, type definitions, and examples to build the final prompt string. + * **API Call (`_call_llm`)**: Uses the passed `settings` dictionary to get the endpoint URL, API key, model name, temperature, and timeout to make the API request. + * **Parsing (`_parse_llm_response`)**: Parses the LLM's JSON response (using type definitions from the `settings` dictionary for validation) and constructs the `SourceRule` hierarchy based on the two-part format (`individual_file_analysis`, `asset_group_classifications`). Includes sanitization logic for comments and markdown fences. +- **Signals (Inherited):** Emits `prediction_ready(input_path, source_rule_list)` or `prediction_error(input_path, error_message)` upon completion or failure, which are connected to the `LLMInteractionHandler`. Also emits `status_update(message)`. + +## GUI Integration + +- The LLM predictor mode is selected via the preset dropdown in `PresetEditorWidget`. +- Selecting "LLM Interpretation" triggers `MainWindow._on_preset_selection_changed`, which switches the editor view to the `LLMEditorWidget` and calls `update_preview`. +- `MainWindow.update_preview` (or `add_input_paths`) delegates the LLM prediction request(s) to the `LLMInteractionHandler`'s queue. +- `LLMInteractionHandler` manages the background tasks and signals results/errors/status back to `MainWindow`. +- `MainWindow` slots (`_on_llm_prediction_ready_from_handler`, `_on_prediction_error`, `show_status_message`, `_on_llm_processing_state_changed`) handle these signals to update the `UnifiedViewModel` and the UI state (status bar, progress, button enablement). +- The `LLMEditorWidget` allows users to modify settings, saving them via `configuration.save_llm_config()`. `MainWindow` listens for the `settings_saved` signal to provide user feedback. + +## Model Integration (Refactored) + +The `gui/unified_view_model.py` module's `update_rules_for_sources` method still incorporates the results. + +- When the `prediction_signal` is received from `LLMPredictionHandler`, the accompanying `SourceRule` object (which has already been constructed based on the new two-part JSON parsing logic) is passed to `update_rules_for_sources`. +- This method then merges the new `SourceRule` hierarchy into the existing model data, preserving user overrides where applicable. The internal structure of the received `SourceRule` now directly reflects the groupings and classifications determined by the LLM and the new parser. + +## Error Handling (Updated) + +Error handling is distributed: + +- **Configuration Loading:** `LLMInteractionHandler` handles errors loading `llm_settings.json` or `app_settings.json` before starting a task. +- **LLM API Errors:** Handled within `LLMPredictionHandler._call_llm` (e.g., `requests.exceptions.RequestException`, `HTTPError`) and propagated via the `prediction_error` signal. +- **Sanitization/Parsing Errors:** `LLMPredictionHandler._parse_llm_response` catches errors during comment/markdown removal and `json.loads()`. +- **Structure/Validation Errors:** `LLMPredictionHandler._parse_llm_response` includes explicit checks for the required two-part JSON structure and data consistency. +- **Task Management Errors:** `LLMInteractionHandler` handles errors during thread setup/start. + +All errors ultimately result in the `llm_prediction_error` signal being emitted by `LLMInteractionHandler`, allowing `MainWindow` to inform the user via the status bar and handle the completion state. \ No newline at end of file diff --git a/Documentation/Developer_Guide/LLM_Integration_Progress.md b/Documentation/Developer_Guide/LLM_Integration_Progress.md deleted file mode 100644 index 0242954..0000000 --- a/Documentation/Developer_Guide/LLM_Integration_Progress.md +++ /dev/null @@ -1,55 +0,0 @@ -# Developer Guide: LLM Integration Progress - -This document summarizes the goals, approach, and current progress on integrating Large Language Model (LLM) capabilities into the Asset Processor Tool for handling irregularly named asset inputs. - -## 1. Initial Goal - -The primary goal is to enhance the Asset Processor Tool's ability to process asset sources with irregular or non-standard naming conventions that cannot be reliably handled by the existing regex and keyword-based preset system. This involves leveraging an LLM to interpret lists of filenames and determine asset metadata and file classifications. - -## 2. Agreed Approach - -After initial discussion and exploring several options, the agreed approach for developing this feature is as follows: - -* **Dedicated LLM Preset:** The LLM classification logic will be triggered by selecting a specific preset type (or flag) in the main tool, indicating that standard rule-based processing should be bypassed in favor of the LLM. -* **Standalone Prototype:** The core LLM interaction and classification logic is being developed as a standalone Python prototype within the `llm_prototype/` directory. This allows for focused development, testing, and refinement in isolation before integration into the main application. -* **Configurable LLM Endpoint:** The prototype is designed to allow users to configure the LLM API endpoint, supporting various providers including local LLMs (e.g., via LM Studio) and commercial APIs. API keys are handled via environment variables for security. -* **Multi-Asset Handling:** The prototype is being built to handle input sources that contain multiple distinct assets within a single directory or archive. The LLM is expected to identify these separate assets and return a JSON **list**, where each item in the list represents one asset. -* **Chain of Thought (CoT) Prompting:** To improve the LLM's ability to handle the complex task of identifying multiple assets and classifying files, the prompt includes instructions for the LLM to output its reasoning process within tags before generating the final JSON list. -* **Unified Asset Category:** The asset classification uses a single `asset_category` field with defined valid values: `Model`, `Surface`, `Decal`, `ATLAS`, `Imperfection`. -* **Robust JSON Extraction & Validation:** The prototype includes logic to extract the JSON list from the LLM's response (handling potential extra text) and validate its structure and content against expected schemas and values. - -## 3. Prototype Development Progress - -The initial structure for the standalone prototype has been created in the `llm_prototype/` directory: - -* `llm_prototype/PLAN.md`: This document outlines the detailed plan. -* `llm_prototype/config_llm.py`: Configuration file for LLM settings, expected values, and placeholders. -* `llm_prototype/llm_classifier.py`: Main script containing the core logic (loading config/input/prompt, formatting prompt, calling LLM API, extracting/validating JSON). -* `llm_prototype/requirements_llm.txt`: Lists the `requests` library dependency. -* `llm_prototype/prompt_template.txt`: Contains the Chain of Thought prompt template with placeholders and few-shot examples. -* `llm_prototype/README.md`: Provides setup and running instructions for the prototype. -* `llm_prototype/test_inputs/`: Contains example input JSON files (`dinesen_example.json`, `imperfections_example.json`) representing file lists from asset sources. - -Code has been added to `llm_classifier.py` for loading inputs/config/prompt, formatting the prompt, calling the API, and extracting/validating the JSON response. The JSON extraction logic has been made more robust to handle potential variations in LLM output format. - -## 4. Current Status and Challenges - -Initial testing of the prototype revealed the following: - -* Successful communication with the configured LLM API endpoint. -* The LLM is attempting to follow the Chain of Thought structure and generate the list-based JSON output. -* **Challenge:** The LLM is currently failing to consistently produce complete and valid JSON output, leading to JSON decoding errors in the prototype script. -* **Challenge:** The LLM is not strictly adhering to the specified classification values (e.g., returning "Map" instead of "PBRMap"), despite the prompt explicitly listing the allowed values and including few-shot examples. - -To address these challenges, few-shot examples demonstrating the expected JSON structure and exact classification values were added to the `prompt_template.txt`. The JSON extraction logic in `llm_classifier.py` was also updated to be more resilient. - -## 5. Next Steps - -The immediate next steps are focused on debugging and improving the LLM's output reliability: - -1. Continue testing the prototype with the updated `prompt_template.txt` (including examples) using the example input files. -2. Analyze the terminal output to determine if the few-shot examples and improved extraction logic have resolved the JSON completeness and classification value issues. -3. Based on the results, iterate on the prompt template (e.g., further emphasizing strict adherence to output format and values) and/or the JSON extraction/validation logic in `llm_classifier.py` as needed. -4. Repeat testing and iteration until the prototype reliably produces valid JSON output with correct classifications for the test cases. - -Once the prototype demonstrates reliable classification, we can proceed to evaluate its performance and plan the integration into the main Asset Processor Tool. \ No newline at end of file diff --git a/Presets/Dinesen.json b/Presets/Dinesen.json index 4617016..f91cf36 100644 --- a/Presets/Dinesen.json +++ b/Presets/Dinesen.json @@ -29,7 +29,7 @@ ], "map_type_mapping": [ { - "target_type": "COL", + "target_type": "MAP_COL", "keywords": [ "COLOR*", "COL", @@ -40,7 +40,7 @@ ] }, { - "target_type": "NRM", + "target_type": "MAP_NRM", "keywords": [ "NORMAL*", "NORM*", @@ -49,27 +49,27 @@ ] }, { - "target_type": "ROUGH", + "target_type": "MAP_ROUGH", "keywords": [ "ROUGHNESS", "ROUGH" ] }, { - "target_type": "ROUGH", + "target_type": "MAP_GLOSS", "keywords": [ "GLOSS" ] }, { - "target_type": "AO", + "target_type": "MAP_AO", "keywords": [ "AMBIENTOCCLUSION", "AO" ] }, { - "target_type": "DISP", + "target_type": "MAP_DISP", "keywords": [ "DISPLACEMENT", "DISP", @@ -78,7 +78,7 @@ ] }, { - "target_type": "REFL", + "target_type": "MAP_REFL", "keywords": [ "REFLECTION", "REFL", @@ -87,26 +87,26 @@ ] }, { - "target_type": "SSS", + "target_type": "MAP_SSS", "keywords": [ "SSS", "SUBSURFACE*" ] }, { - "target_type": "FUZZ", + "target_type": "MAP_FUZZ", "keywords": [ "FUZZ" ] }, { - "target_type": "IDMAP", + "target_type": "MAP_IDMAP", "keywords": [ "IDMAP" ] }, { - "target_type": "MASK", + "target_type": "MAP_MASK", "keywords": [ "OPAC*", "TRANSP*", @@ -115,7 +115,7 @@ ] }, { - "target_type": "METAL", + "target_type": "MAP_METAL", "keywords": [ "METAL*", "METALLIC" diff --git a/Presets/Poliigon.json b/Presets/Poliigon.json index 0434a7f..5587475 100644 --- a/Presets/Poliigon.json +++ b/Presets/Poliigon.json @@ -25,102 +25,10 @@ "*.pdf", "*.url", "*.htm*", - "*_Fabric.*" - ], - "map_type_mapping": [ - { - "target_type": "COL", - "keywords": [ - "COLOR*", - "COL", - "DIFFUSE", - "DIF", - "ALBEDO" - ] - }, - { - "target_type": "NRM", - "keywords": [ - "NORMAL*", - "NORM*", - "NRM*", - "N" - ] - }, - { - "target_type": "ROUGH", - "keywords": [ - "ROUGHNESS", - "ROUGH" - ] - }, - { - "target_type": "ROUGH", - "keywords": [ - "GLOSS" - ] - }, - { - "target_type": "AO", - "keywords": [ - "AMBIENTOCCLUSION", - "AO" - ] - }, - { - "target_type": "DISP", - "keywords": [ - "DISPLACEMENT", - "DISP", - "HEIGHT", - "BUMP" - ] - }, - { - "target_type": "REFL", - "keywords": [ - "REFLECTION", - "REFL", - "SPECULAR", - "SPEC" - ] - }, - { - "target_type": "SSS", - "keywords": [ - "SSS", - "SUBSURFACE*" - ] - }, - { - "target_type": "FUZZ", - "keywords": [ - "FUZZ" - ] - }, - { - "target_type": "IDMAP", - "keywords": [ - "IDMAP" - ] - }, - { - "target_type": "MASK", - "keywords": [ - "OPAC*", - "TRANSP*", - "MASK*", - "ALPHA*" - ] - }, - { - "target_type": "METAL", - "keywords": [ - "METAL*", - "METALLIC" - ] - } + "*_Fabric.*", + "*_Albedo*" ], + "map_type_mapping": [], "asset_category_rules": { "model_patterns": [ "*.fbx", diff --git a/Presets/_template.json b/Presets/_template.json index 97a61f7..2c50283 100644 --- a/Presets/_template.json +++ b/Presets/_template.json @@ -29,7 +29,7 @@ ], "map_type_mapping": [ { - "target_type": "COL", + "target_type": "MAP_COL", "keywords": [ "COLOR*", "COL", @@ -39,7 +39,7 @@ ] }, { - "target_type": "NRM", + "target_type": "MAP_NRM", "keywords": [ "NORMAL*", "NORM*", @@ -48,27 +48,27 @@ ] }, { - "target_type": "ROUGH", + "target_type": "MAP_ROUGH", "keywords": [ "ROUGHNESS", "ROUGH" ] }, { - "target_type": "ROUGH", + "target_type": "MAP_ROUGH", "keywords": [ "GLOSS" ] }, { - "target_type": "AO", + "target_type": "MAP_AO", "keywords": [ "AMBIENTOCCLUSION", "AO" ] }, { - "target_type": "DISP", + "target_type": "MAP_DISP", "keywords": [ "DISPLACEMENT", "DISP", @@ -77,7 +77,7 @@ ] }, { - "target_type": "REFL", + "target_type": "MAP_REFL", "keywords": [ "REFLECTION", "REFL", @@ -86,27 +86,27 @@ ] }, { - "target_type": "SSS", + "target_type": "MAP_SSS", "keywords": [ "SSS", "SUBSURFACE*" ] }, { - "target_type": "FUZZ", + "target_type": "MAP_FUZZ", "keywords": [ "FUZZ" ] }, { - "target_type": "IDMAP", + "target_type": "MAP_IDMAP", "keywords": [ "ID*", "IDMAP" ] }, { - "target_type": "MASK", + "target_type": "MAP_MASK", "keywords": [ "OPAC*", "TRANS*", @@ -115,7 +115,7 @@ ] }, { - "target_type": "METAL", + "target_type": "MAP_METAL", "keywords": [ "METALNESS_", "METALLIC" diff --git a/ProjectNotes/ARCHITECTURE_README_DRAFT.md b/ProjectNotes/ARCHITECTURE_README_DRAFT.md deleted file mode 100644 index 247844c..0000000 --- a/ProjectNotes/ARCHITECTURE_README_DRAFT.md +++ /dev/null @@ -1,106 +0,0 @@ -# DRAFT README Enhancements - Architecture Section & Refinements - -**(Note: This is a draft. Integrate the "Architecture" section and the refinements into the main `readme.md` file.)** - ---- - -## Refinements to Existing Sections - -**(Suggest adding these points or similar wording to the relevant existing sections)** - -* **In Features:** - * Add: **Responsive GUI:** Utilizes background threads for processing and file preview generation, ensuring the user interface remains responsive. - * Add: **Optimized Classification:** Pre-compiles regular expressions from presets for faster file identification during classification. -* **In Directory Structure:** - * Update Core Logic bullet: `* **Core Logic:** main.py, monitor.py, asset_processor.py, configuration.py, config.py` (explicitly add `configuration.py`). - ---- - -## Architecture - -**(Suggest adding this new section, perhaps after "Features" or "Directory Structure")** - -This section provides a higher-level overview of the tool's internal structure and design, intended for developers or users interested in the technical implementation. - -### Core Components - -The tool is primarily built around several key Python modules: - -* **`config.py`**: Defines core, global settings (output paths, resolutions, default behaviors, format rules, etc.) that are generally not supplier-specific. -* **`Presets/*.json`**: Supplier-specific JSON files defining rules for interpreting source assets (filename patterns, map type keywords, model identification, etc.). -* **`configuration.py` (`Configuration` class)**: Responsible for loading the core `config.py` settings and merging them with a selected preset JSON file. Crucially, it also **pre-compiles** regular expression patterns defined in the preset (e.g., for map keywords, extra files, 16-bit variants) upon initialization. This pre-compilation significantly speeds up the file classification process. -* **`asset_processor.py` (`AssetProcessor` class)**: Contains the core logic for processing a *single* asset. It orchestrates the pipeline steps: workspace setup, extraction, file classification, metadata determination, map processing, channel merging, metadata file generation, and output organization. -* **`main.py`**: Serves as the entry point for the Command-Line Interface (CLI). It handles argument parsing, sets up logging, manages the parallel processing pool, and calls `AssetProcessor` for each input asset via a wrapper function. -* **`gui/`**: Contains modules related to the Graphical User Interface (GUI), built using PySide6. -* **`monitor.py`**: Implements the directory monitoring functionality for automated processing. - -### Parallel Processing (CLI & GUI) - -To accelerate the processing of multiple assets, the tool utilizes Python's `concurrent.futures.ProcessPoolExecutor`. - -* Both `main.py` (for CLI) and `gui/processing_handler.py` (for GUI background tasks) create a process pool. -* The actual processing for each asset is delegated to the `main.process_single_asset_wrapper` function. This wrapper is executed in a separate worker process within the pool. -* The wrapper function is responsible for instantiating the `Configuration` and `AssetProcessor` classes for the specific asset being processed in that worker. This isolates each asset's processing environment. -* Results (success, skip, failure, error messages) are communicated back from the worker processes to the main coordinating script (either `main.py` or `gui/processing_handler.py`). - -### Asset Processing Pipeline (`AssetProcessor` class) - -The `AssetProcessor` class executes a sequence of steps for each asset: - -1. **`_setup_workspace()`**: Creates a temporary directory for processing. -2. **`_extract_input()`**: Extracts the input ZIP archive or copies the input folder contents into the temporary workspace. -3. **`_inventory_and_classify_files()`**: This is a critical step that scans the workspace and classifies each file based on rules defined in the loaded `Configuration` (which includes the preset). It uses the pre-compiled regex patterns for efficiency. Key logic includes: - * Identifying files explicitly marked for the `Extra/` folder. - * Identifying model files. - * Matching potential texture maps against keyword patterns. - * Identifying and prioritizing 16-bit variants (e.g., `_NRM16.tif`) over their 8-bit counterparts based on `source_naming.bit_depth_variants` patterns. Ignored 8-bit files are tracked. - * Handling map variants (e.g., multiple Color maps) by assigning suffixes (`-1`, `-2`) based on the `RESPECT_VARIANT_MAP_TYPES` setting in `config.py` and the order of keywords defined in the preset's `map_type_mapping`. - * Classifying any remaining files as 'Unrecognised' (which are also moved to the `Extra/` folder). -4. **`_determine_base_metadata()`**: Determines the asset's base name, category (Texture, Asset, Decal), and archetype (e.g., Wood, Metal) based on classified files and preset rules (`source_naming`, `asset_category_rules`, `archetype_rules`). -5. **Skip Check**: If `overwrite` is false, checks if the final output directory and metadata file already exist. If so, processing for this asset stops early. -6. **`_process_maps()`**: Iterates through classified texture maps. For each map: - * Loads the image data (handling potential Gloss->Roughness inversion). - * Resizes the map to each target resolution specified in `config.py`, avoiding upscaling. - * Determines the output bit depth based on `MAP_BIT_DEPTH_RULES` (`respect` source or `force_8bit`). - * Determines the output file format (`.jpg`, `.png`, `.exr`) based on a combination of factors: - * The `RESOLUTION_THRESHOLD_FOR_JPG` (forces JPG for 8-bit maps above the threshold). - * The original input file format (e.g., `.jpg` inputs tend to produce `.jpg` outputs if 8-bit and below threshold). - * The target bit depth (16-bit outputs use configured `OUTPUT_FORMAT_16BIT_PRIMARY` or `_FALLBACK`). - * Configured 8-bit format (`OUTPUT_FORMAT_8BIT`). - * Saves the processed map for each resolution, applying appropriate compression/quality settings. Includes fallback logic if saving in the primary format fails (e.g., EXR -> PNG). - * Calculates basic image statistics (Min/Max/Mean) for a reference resolution (`CALCULATE_STATS_RESOLUTION`). -7. **`_merge_maps()`**: Combines channels from different processed maps into new textures (e.g., NRMRGH) based on `MAP_MERGE_RULES` defined in `config.py`. It determines the output format for merged maps similarly to `_process_maps`, considering the formats of the input maps involved. -8. **`_generate_metadata_file()`**: Collects all gathered information (asset name, maps present, resolutions, stats, etc.) and writes it to the `metadata.json` file. -9. **`_organize_output_files()`**: Moves the processed maps, merged maps, models, metadata file, and any 'Extra'/'Unrecognised'/'Ignored' files from the temporary workspace to the final structured output directory (`///`). -10. **`_cleanup_workspace()`**: Removes the temporary workspace directory. - -### GUI Architecture (`gui/`) - -The GUI provides an interactive way to use the tool and manage presets. - -* **Framework**: Built using `PySide6`, the official Python bindings for the Qt framework. -* **Main Window (`main_window.py`)**: Defines the main application window, which includes: - * An integrated preset editor panel (using `QSplitter`). - * A processing panel with drag-and-drop support, a file preview table, and processing controls. -* **Threading Model**: To prevent the UI from freezing during potentially long operations, background tasks are run in separate `QThread`s: - * **`ProcessingHandler` (`processing_handler.py`)**: Manages the execution of the main processing pipeline (using `ProcessPoolExecutor` and `main.process_single_asset_wrapper`, similar to the CLI) in a background thread. - * **`PredictionHandler` (`prediction_handler.py`)**: Manages the generation of file previews in a background thread. It calls `AssetProcessor.get_detailed_file_predictions()`, which performs the extraction and classification steps without full image processing, making it much faster. -* **Communication**: Qt's **signal and slot mechanism** is used for communication between the background threads (`ProcessingHandler`, `PredictionHandler`) and the main GUI thread (`MainWindow`). For example, signals are emitted to update the progress bar, populate the preview table, and report completion status or errors. -* **Preset Editor**: The editor allows creating, modifying, and saving preset JSON files directly within the GUI. Changes are tracked, and users are prompted to save before closing or loading another preset if changes are pending. - -### Monitor Architecture (`monitor.py`) - -The `monitor.py` script enables automated processing of assets dropped into a designated input directory. - -* **File System Watching**: Uses the `watchdog` library (specifically `PollingObserver` for cross-platform compatibility) to monitor the specified `INPUT_DIR`. -* **Event Handling**: A custom `ZipHandler` detects `on_created` events for `.zip` files. -* **Filename Parsing**: It expects filenames in the format `[preset]_filename.zip` and uses a regular expression (`PRESET_FILENAME_REGEX`) to extract the `preset` name. -* **Preset Validation**: Checks if the extracted preset name corresponds to a valid `.json` file in the `Presets/` directory. -* **Processing Trigger**: If the filename format and preset are valid, it calls the `main.run_processing` function (the same core logic used by the CLI) to process the detected ZIP file using the extracted preset. -* **File Management**: Moves the source ZIP file to either a `PROCESSED_DIR` (on success/skip) or an `ERROR_DIR` (on failure or invalid preset) after the processing attempt. - -### Error Handling - -* Custom exception classes (`ConfigurationError`, `AssetProcessingError`) are defined and used to signal specific types of errors during configuration loading or asset processing. -* Standard Python logging is used throughout the application (CLI, GUI, Monitor, Core Logic) to record information, warnings, and errors. Log levels can be configured. -* Worker processes in the processing pool capture exceptions and report them back to the main process for logging and status updates. \ No newline at end of file diff --git a/ProjectNotes/BLENDER_INTEGRATION_PLAN.md b/ProjectNotes/BLENDER_INTEGRATION_PLAN.md deleted file mode 100644 index fd3a22f..0000000 --- a/ProjectNotes/BLENDER_INTEGRATION_PLAN.md +++ /dev/null @@ -1,124 +0,0 @@ -# Blender Integration Plan: Node Groups from Processed Assets - -**Objective:** Develop a Python script (`blenderscripts/create_nodegroups.py`) to run manually inside Blender. This script will scan the output directory generated by the Asset Processor Tool, read `metadata.json` files, and create/update corresponding PBR node groups in the active Blender file, leveraging the pre-calculated metadata. - -**Key Principles:** - -* **Leverage Existing Tool Output:** Rely entirely on the structured output and `metadata.json` from the Asset Processor Tool. Avoid reprocessing or recalculating data already available. -* **Blender Environment:** The script is designed solely for Blender's Python environment (`bpy`). -* **Manual Execution:** Users will manually run this script from Blender's Text Editor. -* **Target Active File:** All operations modify the currently open `.blend` file. -* **Assume Templates:** The script will assume node group templates (`Template_PBRSET`, `Template_PBRTYPE`) exist in the active file. Error handling will be added if they are missing. -* **Focus on Node Groups:** The script's scope is limited to creating and updating the node groups, not materials. - -**Detailed Plan:** - -1. **Script Setup & Configuration:** - * Create a new Python file named `create_nodegroups.py` (intended location: `blenderscripts/`). - * Import necessary modules (`bpy`, `os`, `json`, `pathlib`). - * Define user-configurable variables at the top: - * `PROCESSED_ASSET_LIBRARY_ROOT`: Path to the root output directory of the Asset Processor Tool. - * `PARENT_TEMPLATE_NAME`: Name of the parent node group template (e.g., `"Template_PBRSET"`). - * `CHILD_TEMPLATE_NAME`: Name of the child node group template (e.g., `"Template_PBRTYPE"`). - * `ASPECT_RATIO_NODE_LABEL`: Label of the Value node in the parent template for aspect ratio correction (e.g., `"AspectRatioCorrection"`). - * `STATS_NODE_PREFIX`: Prefix for Combine XYZ nodes storing stats in the parent template (e.g., `"Histogram-"`). - * `ENABLE_MANIFEST`: Boolean flag to enable/disable the manifest system (default: `True`). - -2. **Manifest Handling:** - * **Location:** Use a separate JSON file named `[ActiveBlendFileName]_manifest.json`, located in the same directory as the active `.blend` file. - * **Loading:** Implement `load_manifest(context)` that finds and reads the manifest JSON file. If not found or invalid, return an empty dictionary. - * **Saving:** Implement `save_manifest(context, manifest_data)` that writes the `manifest_data` dictionary to the manifest JSON file. - * **Checking:** Implement helper functions `is_asset_processed(manifest_data, asset_name)` and `is_map_processed(manifest_data, asset_name, map_type, resolution)` to check against the loaded manifest. - * **Updating:** Update the manifest dictionary in memory as assets/maps are processed. Save the manifest once at the end of the script. - -3. **Core Logic - `process_library()` function:** - * Get Blender context. - * Load manifest data (if enabled). - * Validate that `PROCESSED_ASSET_LIBRARY_ROOT` exists. - * Validate that template node groups exist in `bpy.data.node_groups`. Exit gracefully with an error message if not found. - * Initialize counters (new groups, updated groups, etc.). - * **Scan Directory:** Use `os.walk` or `pathlib.rglob` to find all `metadata.json` files within the `PROCESSED_ASSET_LIBRARY_ROOT`. - * **Iterate Metadata:** For each `metadata.json` found: - * Parse the JSON data. Extract key information: `asset_name`, `supplier_name`, `archetype`, `maps` (dictionary of maps with resolutions, paths, stats), `aspect_ratio_change_string`. - * Check manifest if asset is already processed (if enabled). Skip if true. - * **Parent Group Handling:** - * Determine target parent group name (e.g., `f"PBRSET_{asset_name}"`). - * Find existing group or create a copy from `PARENT_TEMPLATE_NAME`. - * Mark group as asset (`asset_mark()`) if not already. - * **Apply Metadata:** - * Find the aspect ratio node using `ASPECT_RATIO_NODE_LABEL`. Calculate the correction factor based on the `aspect_ratio_change_string` from metadata (using helper function `calculate_factor_from_string`) and set the node's default value. - * For relevant map types (e.g., ROUGH, DISP), find the stats node (`STATS_NODE_PREFIX` + map type). Set the X, Y, Z inputs using the `min`, `max`, `mean` values stored in the map's metadata entry for the reference resolution. - * **Apply Asset Tags:** Use `asset_data.tags.new()` to add the `supplier_name` and `archetype` tags (checking for existence first). - * **Child Group Handling (Iterate through `maps` in metadata):** - * For each `map_type` (e.g., "COL", "NRM") and its data in the metadata: - * Determine target child group name (e.g., `f"PBRTYPE_{asset_name}_{map_type}"`). - * Find existing child group or create a copy from `CHILD_TEMPLATE_NAME`. - * Find the corresponding placeholder node in the *parent* group (by label matching `map_type`). Assign the child node group to this placeholder (`placeholder_node.node_tree = child_group`). - * Link the child group's output to the corresponding parent group's output socket. Ensure the parent output socket type is `NodeSocketColor`. - * **Image Node Handling (Iterate through resolutions for the map type):** - * For each `resolution` (e.g., "4K", "2K") and its `image_path` in the metadata: - * Check manifest if this specific map/resolution is processed (if enabled). Skip if true. - * Find the corresponding Image Texture node in the *child* group (by label matching `resolution`, e.g., "4K"). - * Load the image using `bpy.data.images.load(image_path, check_existing=True)`. Handle potential file-not-found errors. - * Assign the loaded image to the `image_node.image`. - * Set the `image_node.image.colorspace_settings.name` based on the `map_type` (using a helper function `get_color_space`). - * Update manifest dictionary for this map/resolution (if enabled). - * Update manifest dictionary for the processed asset (if enabled). - * Save manifest data (if enabled and changes were made). - * Print summary (duration, groups created/updated, etc.). - -4. **Helper Functions:** - * `find_nodes_by_label(node_tree, label, node_type)`: Reusable function to find nodes. - * `calculate_factor_from_string(aspect_string)`: Parses the `aspect_ratio_change_string` from metadata and returns the appropriate UV X-scaling factor. - * `get_color_space(map_type)`: Returns the appropriate Blender color space name for a given map type string. - * `add_tag_if_new(asset_data, tag_name)`: Adds a tag if it doesn't exist. - * Manifest loading/saving/checking functions. - -5. **Execution Block (`if __name__ == "__main__":`)** - * Add pre-run checks (templates exist, library path valid, blend file saved if manifest enabled). - * Call the main `process_library()` function. - * Include basic timing and print statements for start/end. - -**Mermaid Diagram:** - -```mermaid -graph TD - A[Start Script in Blender] --> B(Load Config: Lib Path, Template Names, Node Labels); - B --> C{Check Templates Exist}; - C -- Templates OK --> D(Load Manifest from adjacent .json file); - C -- Templates Missing --> X(Error & Exit); - D --> E{Scan Processed Library for metadata.json}; - E --> F{For each metadata.json}; - F --> G{Parse Metadata (Asset Name, Supplier, Archetype, Maps, Aspect Str, Stats)}; - G --> H{Is Asset in Manifest?}; - H -- Yes --> F; - H -- No --> I{Find/Create Parent Group (PBRSET_)}; - I --> J(Mark as Asset & Apply Supplier + Archetype Tags); - J --> K(Find Aspect Node & Set Value from Aspect String); - K --> M{For each Map Type in Metadata}; - M --> N(Find Stats Node & Set Values from Stats in Metadata); - N --> O{Find/Create Child Group (PBRTYPE_)}; - O --> P(Assign Child to Parent Placeholder); - P --> Q(Link Child Output to Parent Output); - Q --> R{For each Resolution of Map}; - R --> S{Is Map/Res in Manifest?}; - S -- Yes --> R; - S -- No --> T(Find Image Node in Child); - T --> U(Load Processed Image); - U --> V(Assign Image to Node); - V --> W(Set Image Color Space); - W --> W1(Update Manifest Dict for Map/Res); - W1 --> R; - R -- All Resolutions Done --> M; - M -- All Map Types Done --> X1(Update Manifest Dict for Asset); - X1 --> F; - F -- All metadata.json Processed --> Y(Save Manifest Dict to adjacent .json file); - Y --> Z(Print Summary & Finish); - - subgraph "Manifest Operations (External File)" - D; H; S; W1; X1; Y; - end - - subgraph "Node/Asset Operations" - I; J; K; N; O; P; Q; T; U; V; W; - end \ No newline at end of file diff --git a/ProjectNotes/BLENDER_INTEGRATION_PLAN_v2.md b/ProjectNotes/BLENDER_INTEGRATION_PLAN_v2.md deleted file mode 100644 index a644a2b..0000000 --- a/ProjectNotes/BLENDER_INTEGRATION_PLAN_v2.md +++ /dev/null @@ -1,52 +0,0 @@ -# Blender Integration Plan v2 - -## Goal - -Add an optional step to `main.py` to run `blenderscripts/create_nodegroups.py` and `blenderscripts/create_materials.py` on specified `.blend` files after asset processing is complete. - -## Proposed Plan - -1. **Update `config.py`:** - * Add two new optional configuration variables: `DEFAULT_NODEGROUP_BLEND_PATH` and `DEFAULT_MATERIALS_BLEND_PATH`. These will store the default paths to the Blender files. - -2. **Update `main.py` Argument Parser:** - * Add two new optional command-line arguments: `--nodegroup-blend` and `--materials-blend`. - * These arguments will accept file paths to the respective `.blend` files. - * If provided, these arguments will override the default paths specified in `config.py`. - -3. **Update `blenderscripts/create_nodegroups.py` and `blenderscripts/create_materials.py`:** - * Modify both scripts to accept the processed asset library root path (`PROCESSED_ASSET_LIBRARY_ROOT`) as a command-line argument. This will be passed to the script when executed by Blender using the `--` separator. - * Update the scripts to read this path from `sys.argv` instead of using the hardcoded variable. - -4. **Update `main.py` Execution Flow:** - * After the main asset processing loop (`run_processing`) completes and the summary is reported, check if the `--nodegroup-blend` or `--materials-blend` arguments (or their fallbacks from `config.py`) were provided. - * If a path for the nodegroup `.blend` file is available: - * Construct a command to execute Blender in the background (`-b`), load the specified nodegroup `.blend` file, run the `create_nodegroups.py` script using `--python`, pass the processed asset root directory as an argument after `--`, and save the `.blend` file (`-S`). - * Execute this command using the `execute_command` tool. - * If a path for the materials `.blend` file is available: - * Construct a similar command to execute Blender in the background, load the specified materials `.blend` file, run the `create_materials.py` script using `--python`, pass the processed asset root directory as an argument after `--`, and save the `.blend` file (`-S`). - * Execute this command using the `execute_command` tool. - * Include error handling for the execution of the Blender commands. - -## Execution Flow Diagram - -```mermaid -graph TD - A[Asset Processing Complete] --> B[Report Summary]; - B --> C{Nodegroup Blend Path Specified?}; - C -- Yes --> D[Get Nodegroup Blend Path (Arg or Config)]; - D --> E[Construct Blender Command for Nodegroups]; - E --> F[Execute Command: blender -b nodegroup.blend --python create_nodegroups.py -- -S]; - F --> G{Command Successful?}; - G -- Yes --> H{Materials Blend Path Specified?}; - G -- No --> I[Log Nodegroup Error]; - I --> H; - H -- Yes --> J[Get Materials Blend Path (Arg or Config)]; - J --> K[Construct Blender Command for Materials]; - K --> L[Execute Command: blender -b materials.blend --python create_materials.py -- -S]; - L --> M{Command Successful?}; - M -- Yes --> N[End main.py]; - M -- No --> O[Log Materials Error]; - O --> N; - H -- No --> N; - C -- No --> H; \ No newline at end of file diff --git a/ProjectNotes/BLENDER_MATERIAL_CREATION_PLAN.md b/ProjectNotes/BLENDER_MATERIAL_CREATION_PLAN.md deleted file mode 100644 index f432485..0000000 --- a/ProjectNotes/BLENDER_MATERIAL_CREATION_PLAN.md +++ /dev/null @@ -1,131 +0,0 @@ -# Blender Material Creation Script Plan - -This document outlines the plan for creating a new Blender script (`create_materials.py`) in the `blenderscripts/` directory. This script will scan the processed asset library output by the Asset Processor Tool, read the `metadata.json` files, and create or update Blender materials that link to the corresponding PBRSET node groups found in a specified Blender Asset Library. The script will also set the material's viewport properties using pre-calculated statistics from the metadata. The script will skip processing an asset if the corresponding material already exists in the current Blender file. - -## 1. Script Location and Naming - -* Create a new file: `blenderscripts/create_materials.py`. - -## 2. Script Structure - -The script will follow a similar structure to `blenderscripts/create_nodegroups.py`, including: - -* Import statements (`bpy`, `os`, `json`, `pathlib`, `time`, `base64`). -* A `--- USER CONFIGURATION ---` section at the top. -* Helper functions. -* A main processing function (e.g., `process_library_for_materials`). -* An execution block (`if __name__ == "__main__":`) to run the main function. - -## 3. Configuration Variables - -The script will include the following configuration variables in the `--- USER CONFIGURATION ---` section: - -* `PROCESSED_ASSET_LIBRARY_ROOT`: Path to the root output directory of the Asset Processor Tool (same as in `create_nodegroups.py`). This is used to find the `metadata.json` files and reference images for previews. -* `PBRSET_ASSET_LIBRARY_NAME`: The name of the Blender Asset Library (configured in Blender Preferences) that contains the PBRSET node groups created by `create_nodegroups.py`. -* `TEMPLATE_MATERIAL_NAME`: Name of the required template material in the Blender file (e.g., "Template_PBRMaterial"). -* `PLACEHOLDER_NODE_LABEL`: Label of the placeholder Group node within the template material's node tree where the PBRSET node group will be linked (e.g., "PBRSET_PLACEHOLDER"). -* `MATERIAL_NAME_PREFIX`: Prefix for the created materials (e.g., "Mat_"). -* `PBRSET_GROUP_PREFIX`: Prefix used for the PBRSET node groups created by `create_nodegroups.py` (e.g., "PBRSET_"). -* `REFERENCE_MAP_TYPES`: List of map types to look for to find a reference image for the material preview (e.g., `["COL", "COL-1"]`). -* `REFERENCE_RESOLUTION_ORDER`: Preferred resolution order for the reference image (e.g., `["1K", "512", "2K", "4K"]`). -* `IMAGE_FILENAME_PATTERN`: Assumed filename pattern for processed images (same as in `create_nodegroups.py`). -* `FALLBACK_IMAGE_EXTENSIONS`: Fallback extensions for finding image files (same as in `create_nodegroups.py`). -* `VIEWPORT_COLOR_MAP_TYPES`: List of map types to check in metadata's `image_stats_1k` for viewport diffuse color. -* `VIEWPORT_ROUGHNESS_MAP_TYPES`: List of map types to check in metadata's `image_stats_1k` for viewport roughness. -* `VIEWPORT_METALLIC_MAP_TYPES`: List of map types to check in metadata's `image_stats_1k` for viewport metallic. - -## 4. Helper Functions - -The script will include the following helper functions: - -* `find_nodes_by_label(node_tree, label, node_type=None)`: Reusable from `create_nodegroups.py` to find nodes in a node tree. -* `add_tag_if_new(asset_data, tag_name)`: Reusable from `create_nodegroups.py` to add asset tags. -* `reconstruct_image_path_with_fallback(...)`: Reusable from `create_nodegroups.py` to find image paths (needed for setting the custom preview). -* `get_stat_value(stats_dict, map_type_list, stat_key)`: A helper function to safely retrieve a specific statistic from the `image_stats_1k` dictionary. - -## 5. Main Processing Logic (`process_library_for_materials`) - -The main function will perform the following steps: - -* **Pre-run Checks:** - * Verify `PROCESSED_ASSET_LIBRARY_ROOT` exists and is a directory. - * Verify the `PBRSET_ASSET_LIBRARY_NAME` exists in Blender's user preferences (`bpy.context.preferences.filepaths.asset_libraries`). - * Verify the `TEMPLATE_MATERIAL_NAME` material exists and uses nodes. - * Verify the `PLACEHOLDER_NODE_LABEL` Group node exists in the template material's node tree. -* **Scan for Metadata:** - * Iterate through supplier directories within `PROCESSED_ASSET_LIBRARY_ROOT`. - * Iterate through asset directories within each supplier directory. - * Identify `metadata.json` files. -* **Process Each Metadata File:** - * Load the `metadata.json` file. - * Extract `asset_name`, `supplier_name`, `archetype`, `processed_map_resolutions`, `merged_map_resolutions`, `map_details`, and `image_stats_1k`. - * Determine the expected PBRSET node group name: `f"{PBRSET_GROUP_PREFIX}{asset_name}"`. - * Determine the target material name: `f"{MATERIAL_NAME_PREFIX}{asset_name}"`. - * **Find or Create Material:** - * Check if a material with the `target_material_name` already exists in `bpy.data.materials`. - * If it exists, log a message indicating the asset is being skipped and move to the next metadata file. - * If it doesn't exist, copy the `TEMPLATE_MATERIAL_NAME` material and rename the copy to `target_material_name` (create mode). Handle potential copy failures. - * **Find Placeholder Node:** - * Find the node with `PLACEHOLDER_NODE_LABEL` in the target material's node tree using `find_nodes_by_label`. Handle cases where the node is not found or is not a Group node. - * **Find and Link PBRSET Node Group from Asset Library:** - * Get the path to the `.blend` file associated with the `PBRSET_ASSET_LIBRARY_NAME` from user preferences. - * Use `bpy.data.libraries.load(filepath, link=True)` to link the node group with `target_pbrset_group_name` from the external `.blend` file into the current file. Handle cases where the library or the node group is not found. - * Once linked, get the reference to the newly linked node group in `bpy.data.node_groups`. - * **Link Linked Node Group to Placeholder:** - * If both the placeholder node and the *newly linked* PBRSET node group are found, assign the linked node group to the `node_tree` property of the placeholder node. - * **Mark Material as Asset:** - * If the material is new or not already marked, call `material.asset_mark()`. - * **Copy Asset Tags:** - * If both the material and the *linked* PBRSET node group have asset data, copy tags (supplier, archetype) from the node group to the material using `add_tag_if_new`. - * **Set Custom Preview:** - * Find a suitable reference image path (e.g., lowest resolution COL map) using `reconstruct_image_path_with_fallback` and the `REFERENCE_MAP_TYPES` and `REFERENCE_RESOLUTION_ORDER` configurations. - * If a reference image path is found, use `bpy.ops.ed.lib_id_load_custom_preview` to set the custom preview for the material. This operation requires overriding the context. - * **Set Viewport Properties (using metadata stats):** - * Check if `image_stats_1k` is present and valid in the metadata. - * **Diffuse Color:** Use the `get_stat_value` helper to get the 'mean' stat for map types in `VIEWPORT_COLOR_MAP_TYPES`. If found and is a valid color list `[R, G, B]`, set `material.diffuse_color` to this value. - * **Roughness:** Use the `get_stat_value` helper to get the 'mean' stat for map types in `VIEWPORT_ROUGHNESS_MAP_TYPES`. If found, get the first value (for grayscale). Check if stats for `VIEWPORT_METALLIC_MAP_TYPES` exist. If metallic stats are *not* found, invert the roughness value (`1.0 - value`) before assigning it to `material.roughness`. Clamp the final value between 0.0 and 1.0. - * **Metallic:** Use the `get_stat_value` helper to get the 'mean' stat for map types in `VIEWPORT_METALLIC_MAP_TYPES`. If found, get the first value (for grayscale) and assign it to `material.metallic`. If metallic stats are *not* found, set `material.metallic` to 0.0. - * **Error Handling and Reporting:** - * Include `try...except` blocks to catch errors during file reading, JSON parsing, Blender operations, linking, etc. - * Print informative messages about progress, creation/update status, and errors. -* **Summary Report:** - * Print a summary of how many metadata files were processed, materials created/updated, node groups linked, errors encountered, and assets skipped. - -## 6. Process Flow Diagram (Updated) - -```mermaid -graph TD - A[Start Script] --> B{Pre-run Checks Pass?}; - B -- No --> C[Abort Script]; - B -- Yes --> D[Scan Processed Asset Root]; - D --> E{Found metadata.json?}; - E -- No --> F[Finish Script (No Assets)]; - E -- Yes --> G[Loop through metadata.json files]; - G --> H[Read metadata.json]; - H --> I{Metadata Valid?}; - I -- No --> J[Log Error, Skip Asset]; - I -- Yes --> K[Extract Asset Info & Stats]; - K --> L[Find or Create Material]; - L --> M{Material Exists?}; - M -- Yes --> N[Log Skip, Continue Loop]; - M -- No --> O[Find Placeholder Node in Material]; - O --> P[Find PBRSET NG in Library & Link]; - P --> Q{Linking Successful?}; - Q -- No --> R[Log Error, Skip Asset]; - Q -- Yes --> S[Link Linked NG to Placeholder]; - S --> T[Mark Material as Asset]; - T --> U[Copy Asset Tags]; - U --> V[Find Reference Image Path for Preview]; - V --> W{Reference Image Found?}; - W -- Yes --> X[Set Custom Material Preview]; - W -- No --> Y[Log Warning (No Preview)]; - X --> Z[Set Viewport Properties from Stats]; - Y --> Z; - Z --> AA[Increment Counters]; - AA --> G; - G --> AB[Print Summary Report]; - AB --> AC[End Script]; - - J --> G; - N --> G; - R --> G; \ No newline at end of file diff --git a/ProjectNotes/BLENDER_MATERIAL_MERGE_ADDON_PLAN.md b/ProjectNotes/BLENDER_MATERIAL_MERGE_ADDON_PLAN.md deleted file mode 100644 index ea82ba8..0000000 --- a/ProjectNotes/BLENDER_MATERIAL_MERGE_ADDON_PLAN.md +++ /dev/null @@ -1,103 +0,0 @@ -# Blender Addon Plan: Material Merger - -**Version:** 1.1 (Includes Extensibility Consideration) - -**1. Goal:** -Create a standalone Blender addon that allows users to select two existing materials (generated by the Asset Processor Tool, or previously merged by this addon) and merge them into a new material. The merge should preserve their individual node structures (including custom tweaks) and combine their final outputs using a dedicated `MaterialMerge` node group. - -**2. Core Functionality (Approach 2 - Node Copying):** - -* **Trigger:** User selects two materials in Blender and invokes an operator (e.g., via a button in the Shader Editor's UI panel). -* **New Material Creation:** The addon creates a new Blender material, named appropriately (e.g., `MAT_Merged__`). -* **Node Copying:** - * For *each* selected source material: - * Iterate through its node tree. - * Copy all nodes *except* the `Material Output` node into the *new* material's node tree, attempting to preserve relative layout and offsetting subsequent copies. - * **Identify Final Outputs:** Determine the node providing the final BSDF shader output and the node providing the final Displacement output *before* the original `Material Output` node. - * In a base material (from Asset Processor), these are expected to be the `PBR_BSDF` node group (BSDF output) and the `PBR_Handler` node group (Displacement output). - * In an already-merged material, these will be the outputs of its top-level `MaterialMerge` node group. - * Store references to these final output nodes and their relevant sockets. -* **MaterialMerge Node:** - * **Link/Append** the `MaterialMerge` node group into the new material's node tree. - * **Assumption:** This node group exists in `blender_files/utility_nodegroups.blend` relative to the addon's location. - * **Assumption:** Socket names are `Shader A`, `Shader B`, `Displacement A`, `Displacement B` (inputs) and `BSDF`, `Displacement` (outputs). -* **Connections:** - * Connect the identified final BSDF output of the *first* source material's copied structure to the `MaterialMerge` node's `Shader A` input. - * Connect the identified final Displacement output of the *first* source material's copied structure to the `MaterialMerge` node's `Displacement A` input. - * Connect the identified final BSDF output of the *second* source material's copied structure to the `MaterialMerge` node's `Shader B` input. - * Connect the identified final Displacement output of the *second* source material's copied structure to the `MaterialMerge` node's `Displacement B` input. - * Connect the `MaterialMerge` node's `BSDF` output to the new material's `Material Output` node's `Surface` input. - * Connect the `MaterialMerge` node's `Displacement` output to the new material's `Material Output` node's `Displacement` input. -* **Layout:** Optionally, attempt a basic auto-layout (`node_tree.nodes.update()`) or arrange the key nodes logically. - -**3. User Interface (UI):** - -* A simple panel in the Blender Shader Editor (Properties region - 'N' panel). -* Two dropdowns or search fields allowing the user to select existing materials from the current `.blend` file. -* A button labeled "Merge Selected Materials". -* Status messages/feedback (e.g., "Merged material created: [Name]", "Error: Could not find required nodes in [Material Name]"). - -**4. Addon Structure (Python):** - -* `__init__.py`: Registers the addon, panel, and operator classes. -* `operator.py`: Contains the `OT_MergeMaterials` operator class implementing the core logic. -* `panel.py`: Contains the `PT_MaterialMergePanel` class defining the UI layout. -* (Optional) `utils.py`: Helper functions for node finding, copying, linking, identifying final outputs, etc. - -**5. Error Handling:** - -* Check if two valid materials are selected. -* Verify that the selected materials have node trees. -* Handle cases where the expected final BSDF/Displacement output nodes cannot be reliably identified in one or both source materials. -* Handle potential errors during node copying. -* Handle errors if the `utility_nodegroups.blend` file or the `MaterialMerge` node group within it cannot be found/linked. - -**6. Assumptions to Verify (Based on User Feedback):** - -* **Node Identification:** - * Base Material Handler: Node named `PBR_Handler`. - * Base Material BSDF: Node named `PBR_BSDF`. - * Merged Material Outputs: The `BSDF` and `Displacement` outputs of the top-level `MaterialMerge` node. -* **`MaterialMerge` Node:** - * Location: `blender_files/utility_nodegroups.blend` (relative path). - * Input Sockets: `Shader A`, `Shader B`, `Displacement A`, `Displacement B`. - * Output Sockets: `BSDF`, `Displacement`. - -**7. Future Extensibility - Recursive Merging:** - -* The core merging logic (copying nodes, identifying final outputs, connecting to a new `MaterialMerge` node) is designed to inherently support selecting an already-merged material as an input without requiring separate code paths initially. The identification of final BSDF/Displacement outputs needs to correctly handle both base materials and merged materials (checking for `PBR_BSDF`/`PBR_Handler` or the outputs of an existing `MaterialMerge` node). - -**8. Mermaid Diagram of Node Flow:** - -```mermaid -graph TD - subgraph New Merged Material - subgraph Copied from Source A (Mat_A or Merge_A) - %% Nodes representing the structure of Source A - Structure_A[...] - Final_BSDF_A[Final BSDF Output A] - Final_Disp_A[Final Displacement Output A] - Structure_A --> Final_BSDF_A - Structure_A --> Final_Disp_A - end - - subgraph Copied from Source B (Mat_B or Merge_B) - %% Nodes representing the structure of Source B - Structure_B[...] - Final_BSDF_B[Final BSDF Output B] - Final_Disp_B[Final Displacement Output B] - Structure_B --> Final_BSDF_B - Structure_B --> Final_Disp_B - end - - Merge[MaterialMerge] - Output[Material Output] - - Final_BSDF_A -- BSDF --> Merge -- Shader A --> Merge - Final_Disp_A -- Displacement --> Merge -- Displacement A --> Merge - Final_BSDF_B -- BSDF --> Merge -- Shader B --> Merge - Final_Disp_B -- Displacement --> Merge -- Displacement B --> Merge - - Merge -- BSDF --> Output -- Surface --> Output - Merge -- Displacement --> Output -- Displacement --> Output - end \ No newline at end of file diff --git a/ProjectNotes/ConfigurationRefactoringPlan.md b/ProjectNotes/ConfigurationRefactoringPlan.md new file mode 100644 index 0000000..e762060 --- /dev/null +++ b/ProjectNotes/ConfigurationRefactoringPlan.md @@ -0,0 +1,107 @@ +# Configuration System Refactoring Plan + +This document outlines the plan for refactoring the configuration system of the Asset Processor Tool. + +## Overall Goals + +1. **Decouple Definitions:** Separate `ASSET_TYPE_DEFINITIONS` and `FILE_TYPE_DEFINITIONS` from the main `config/app_settings.json` into dedicated files. +2. **Introduce User Overrides:** Allow users to override base settings via a new `config/user_settings.json` file. +3. **Improve GUI Saving:** (Lower Priority) Make GUI configuration saving more targeted to avoid overwriting unrelated settings when saving changes from `ConfigEditorDialog` or `LLMEditorWidget`. + +## Proposed Plan Phases + +**Phase 1: Decouple Definitions** + +1. **Create New Definition Files:** + * Create `config/asset_type_definitions.json`. + * Create `config/file_type_definitions.json`. +2. **Migrate Content:** + * Move `ASSET_TYPE_DEFINITIONS` object from `config/app_settings.json` to `config/asset_type_definitions.json`. + * Move `FILE_TYPE_DEFINITIONS` object from `config/app_settings.json` to `config/file_type_definitions.json`. +3. **Update `configuration.py`:** + * Add constants for new definition file paths. + * Modify `Configuration` class to load these new files. + * Update property methods (e.g., `get_asset_type_definitions`, `get_file_type_definitions_with_examples`) to use data from the new definition dictionaries. + * Adjust validation (`_validate_configs`) as needed. +4. **Update GUI & `load_base_config()`:** + * Modify `load_base_config()` to load and return a combined dictionary including `app_settings.json` and the two new definition files. + * Update GUI components relying on `load_base_config()` to ensure they receive the necessary definition data. + +**Phase 2: Implement User Overrides** + +1. **Define `user_settings.json`:** + * Establish `config/user_settings.json` for user-specific overrides, mirroring parts of `app_settings.json`. +2. **Update `configuration.py` Loading:** + * In `Configuration.__init__`, load `app_settings.json`, then definition files, then attempt to load and deep merge `user_settings.json` (user settings override base). + * Load presets *after* the base+user merge (presets override combined base+user). + * Modify `load_base_config()` to also load and merge `user_settings.json` after `app_settings.json`. +3. **Update GUI Editors:** + * Modify `ConfigEditorDialog` to load the effective settings (base+user) but save changes *only* to `config/user_settings.json`. + * `LLMEditorWidget` continues targeting `llm_settings.json`. + +**Phase 3: Granular GUI Saving (Lower Priority)** + +1. **Refactor Saving Logic:** + * In `ConfigEditorDialog` and `LLMEditorWidget`: + * Load the current target file (`user_settings.json` or `llm_settings.json`). + * Identify specific setting(s) changed by the user in the GUI session. + * Update only those specific key(s) in the loaded dictionary. + * Write the entire modified dictionary back to the target file, preserving untouched settings. + +## Proposed File Structure & Loading Flow + +```mermaid +graph LR + subgraph Config Files + A[config/asset_type_definitions.json] + B[config/file_type_definitions.json] + C[config/app_settings.json (Base Defaults)] + D[config/user_settings.json (User Overrides)] + E[config/llm_settings.json] + F[config/suppliers.json] + G[Presets/*.json] + end + + subgraph Code + H[configuration.py] + I[GUI] + J[Processing Engine / Pipeline] + K[LLM Handlers] + end + + subgraph Loading Flow (Configuration Class) + L(Load Asset Types) --> H + M(Load File Types) --> H + N(Load Base Settings) --> P(Merge Base + User) + O(Load User Settings) --> P + P --> R(Merge Preset Overrides) + Q(Load LLM Settings) --> H + R --> T(Final Config Object) + G -- Load Preset --> R + H -- Contains --> T + end + + subgraph Loading Flow (GUI - load_base_config) + L2(Load Asset Types) --> U(Return Merged Defaults + Defs) + M2(Load File Types) --> U + N2(Load Base Settings) --> V(Merge Base + User) + O2(Load User Settings) --> V + V --> U + I -- Calls --> U + end + + + T -- Used by --> J + T -- Used by --> K + + I -- Edits --> D + I -- Edits --> E + I -- Manages --> F + + style A fill:#f9f,stroke:#333,stroke-width:2px + style B fill:#f9f,stroke:#333,stroke-width:2px + style C fill:#ccf,stroke:#333,stroke-width:2px + style D fill:#9cf,stroke:#333,stroke-width:2px + style E fill:#ccf,stroke:#333,stroke-width:2px + style F fill:#9cf,stroke:#333,stroke-width:2px + style G fill:#ffc,stroke:#333,stroke-width:2px \ No newline at end of file diff --git a/ProjectNotes/Data_Structures/Preview_Edit_Interface.md b/ProjectNotes/Data_Structures/Preview_Edit_Interface.md deleted file mode 100644 index 89201d9..0000000 --- a/ProjectNotes/Data_Structures/Preview_Edit_Interface.md +++ /dev/null @@ -1,118 +0,0 @@ -# Data Interface: GUI Preview Edits to Processing Handler - -## 1. Purpose - -This document defines the data structures and interface used to pass user edits made in the GUI's file preview table (specifically changes to 'Status' and 'Predicted Asset' output name) to the backend `ProcessingHandler`. It also incorporates a structure for future asset-level properties. - -## 2. Data Structures - -Two primary data structures are used: - -### 2.1. File Data (`file_list`) - -* **Type:** `list[dict]` -* **Description:** A flat list where each dictionary represents a single file identified during the prediction phase. This list is modified by the GUI to reflect user edits to file status and output names. -* **Dictionary Keys (per file):** - * `original_path` (`str`): The full path to the source file. (Read-only by GUI) - * `predicted_asset_name` (`str | None`): The name of the asset group the file belongs to, derived from input. (Read-only by GUI) - * `predicted_output_name` (`str | None`): The backend's predicted final output filename. **EDITABLE** by the user in the GUI ('Predicted Asset' column). - * `status` (`str`): The backend's predicted status (e.g., 'Mapped', 'Ignored'). **EDITABLE** by the user in the GUI ('Status' column). - * `details` (`str | None`): Additional information or error messages. (Read-only by GUI, potentially updated by model validation). - * `source_asset` (`str`): An identifier for the source asset group (e.g., input folder/zip name). (Read-only by GUI) - -### 2.2. Asset Properties (`asset_properties`) - -* **Type:** `dict[str, dict]` -* **Description:** A dictionary mapping the `source_asset` identifier (string key) to a dictionary of asset-level properties determined by the backend prediction/preset. This structure is initially read-only in the GUI but designed for future expansion (e.g., editing asset category). -* **Asset Properties Dictionary Keys (Example):** - * `asset_category` (`str`): The determined category (e.g., 'surface', 'model'). - * `asset_tags` (`list[str]`): Any relevant tags associated with the asset. - * *(Other future asset-level properties can be added here)* - -## 3. Data Flow & Interface - -```mermaid -graph LR - subgraph Backend - A[Prediction Logic] -- Generates --> B(file_list); - A -- Generates --> C(asset_properties); - end - - subgraph GUI Components - D[PredictionHandler] -- prediction_results_ready(file_list, asset_properties) --> E(PreviewTableModel); - E -- Stores & Allows Edits --> F(Internal file_list); - E -- Stores --> G(Internal asset_properties); - H[MainWindow] -- Retrieves --> F; - H -- Retrieves --> G; - H -- Passes (file_list, asset_properties) --> I[ProcessingHandler]; - end - - subgraph Backend - I -- Uses Edited --> F; - I -- Uses Read-Only --> G; - end - - style A fill:#lightblue,stroke:#333,stroke-width:1px - style B fill:#lightgreen,stroke:#333,stroke-width:1px - style C fill:#lightyellow,stroke:#333,stroke-width:1px - style D fill:#f9f,stroke:#333,stroke-width:2px - style E fill:#ccf,stroke:#333,stroke-width:2px - style F fill:#lightgreen,stroke:#333,stroke-width:1px - style G fill:#lightyellow,stroke:#333,stroke-width:1px - style H fill:#f9f,stroke:#333,stroke-width:2px - style I fill:#lightblue,stroke:#333,stroke-width:2px - -``` - -1. **Prediction:** The `PredictionHandler` generates both the `file_list` and `asset_properties`. -2. **Signal:** It emits a signal (e.g., `prediction_results_ready`) containing both structures, likely as a tuple `(file_list, asset_properties)`. -3. **Table Model:** The `PreviewTableModel` receives the tuple. It stores `asset_properties` (read-only for now). It stores `file_list` and allows user edits to the `status` and `predicted_output_name` values within this list. -4. **Processing Trigger:** When the user initiates processing, the `MainWindow` retrieves the (potentially modified) `file_list` and the (unmodified) `asset_properties` from the `PreviewTableModel`. -5. **Processing Execution:** The `MainWindow` passes both structures to the `ProcessingHandler`. -6. **Handler Logic:** The `ProcessingHandler` iterates through the `file_list`. For each file, it uses the potentially edited `status` and `predicted_output_name`. If asset-level information is needed, it uses the file's `source_asset` key to look up the data in the `asset_properties` dictionary. - -## 4. Example Data Passed to `ProcessingHandler` - -* **`file_list` (Example):** - ```python - [ - { - 'original_path': 'C:/Path/To/AssetA/AssetA_Diffuse.png', - 'predicted_asset_name': 'AssetA', - 'predicted_output_name': 'T_AssetA_BC.tga', - 'status': 'Ignored', # <-- User Edit - 'details': 'User override: Ignored', - 'source_asset': 'AssetA' - }, - { - 'original_path': 'C:/Path/To/AssetA/AssetA_Normal.png', - 'predicted_asset_name': 'AssetA', - 'predicted_output_name': 'T_AssetA_Normals_DX.tga', # <-- User Edit - 'status': 'Mapped', - 'details': None, - 'source_asset': 'AssetA' - }, - # ... other files - ] - ``` -* **`asset_properties` (Example):** - ```python - { - 'AssetA': { - 'asset_category': 'surface', - 'asset_tags': ['wood', 'painted'] - }, - 'AssetB': { - 'asset_category': 'model', - 'asset_tags': ['metal', 'sci-fi'] - } - # ... other assets - } - ``` - -## 5. Implications - -* **`PredictionHandler`:** Needs modification to generate and emit both `file_list` and `asset_properties`. Signal signature changes. -* **`PreviewTableModel`:** Needs modification to receive, store, and provide both structures. Must implement editing capabilities (`flags`, `setData`) for the relevant columns using the `file_list`. Needs methods like `get_edited_data()` returning both structures. -* **`MainWindow`:** Needs modification to retrieve both structures from the table model and pass them to the `ProcessingHandler`. -* **`ProcessingHandler`:** Needs modification to accept both structures in its processing method signature. Must update logic to use the edited `status` and `predicted_output_name` from `file_list` and look up data in `asset_properties` using `source_asset` when needed. \ No newline at end of file diff --git a/ProjectNotes/FEAT-003_Implementation_Plan.md b/ProjectNotes/FEAT-003_Implementation_Plan.md deleted file mode 100644 index 5504c49..0000000 --- a/ProjectNotes/FEAT-003_Implementation_Plan.md +++ /dev/null @@ -1,37 +0,0 @@ -# FEAT-003: Selective Nodegroup Generation and Category Tagging - Implementation Plan - -**Objective:** Modify `blenderscripts/create_nodegroups.py` to read the asset category from `metadata.json`, conditionally create nodegroups for "Surface" and "Decal" assets, and add the category as a tag to the Blender asset. - -**Plan:** - -1. **Modify `blenderscripts/create_nodegroups.py`:** - * Locate the main loop in `blenderscripts/create_nodegroups.py` that iterates through the processed assets. - * Inside this loop, for each asset directory, construct the path to the `metadata.json` file. - * Read the `metadata.json` file using Python's `json` module. - * Extract the `category` value from the parsed JSON data. - * Implement a conditional check: If the extracted `category` is *not* "Surface" and *not* "Decal", skip the existing nodegroup creation logic for this asset and proceed to the tagging step. - * If the `category` *is* "Surface" or "Decal", execute the existing nodegroup creation logic. - * After the conditional nodegroup creation (or skipping), use the Blender Python API (`bpy`) to find the corresponding Blender asset (likely the created node group, if applicable, or potentially the asset representation even if the nodegroup was skipped). - * Add the extracted `category` string as a tag to the found Blender asset. - -2. **Testing:** - * Prepare a test set of processed assets that includes examples of "Surface", "Decal", and "Asset" categories, each with a corresponding `metadata.json` file. - * Run the modified `create_nodegroups.py` script within Blender, pointing it to the test asset library root. - * Verify in Blender that: - * Node groups were created only for the "Surface" and "Decal" assets. - * No node groups were created for "Asset" category assets. - * All processed assets (Surface, Decal, and Asset) have a tag corresponding to their category ("Surface", "Decal", or "Asset") in the Blender asset browser. - -```mermaid -graph TD - A[Start Script] --> B{Iterate Assets}; - B --> C[Read metadata.json]; - C --> D[Get Category]; - D --> E{Category in ["Surface", "Decal"]?}; - E -- Yes --> F[Create Nodegroup]; - E -- No --> G[Skip Nodegroup]; - F --> H[Find Blender Asset]; - G --> H[Find Blender Asset]; - H --> I[Add Category Tag]; - I --> B; - B -- No More Assets --> J[End Script]; \ No newline at end of file diff --git a/ProjectNotes/FEAT-rar-7z-support-plan.md b/ProjectNotes/FEAT-rar-7z-support-plan.md deleted file mode 100644 index e931612..0000000 --- a/ProjectNotes/FEAT-rar-7z-support-plan.md +++ /dev/null @@ -1,49 +0,0 @@ -# Plan for Adding .rar and .7z Support - -**Goal:** Extend the Asset Processor Tool to accept `.rar` and `.7z` files as input sources, in addition to the currently supported `.zip` files and folders. - -**Plan:** - -1. **Add Required Libraries:** - * Update the `requirements.txt` file to include `py7zr` and `rarfile` as dependencies. This will ensure these libraries are installed when setting up the project. - -2. **Modify Input Extraction Logic:** - * Locate the `_extract_input` method within the `AssetProcessor` class in `asset_processor.py`. - * Modify this method to check the file extension of the input source. - * If the extension is `.zip`, retain the existing extraction logic using Python's built-in `zipfile` module. - * If the extension is `.rar`, implement extraction using the `rarfile` library. - * If the extension is `.7z`, implement extraction using the `py7zr` library. - * Include error handling for cases where the archive might be corrupted, encrypted (since we are not implementing password support at this stage, these should likely be skipped or logged as errors), or uses an unsupported compression method. Log appropriate warnings or errors in such cases. - * If the input is a directory, retain the existing logic to copy its contents to the temporary workspace. - -3. **Update CLI and Monitor Input Handling:** - * Review `main.py` (CLI entry point) and `monitor.py` (Directory Monitor). - * Ensure that the argument parsing in `main.py` can accept `.rar` and `.7z` file paths as valid inputs. - * In `monitor.py`, modify the `ZipHandler` (or create a new handler) to watch for `.rar` and `.7z` file creation events in the watched directory, in addition to `.zip` files. The logic for triggering processing via `main.run_processing` should then be extended to handle these new file types. - -4. **Update Documentation:** - * Edit `Documentation/00_Overview.md` to explicitly mention `.rar` and `.7z` as supported input formats in the overview section. - * Update `Documentation/01_User_Guide/02_Features.md` to list `.rar` and `.7z` alongside `.zip` and folders in the features list. - * Modify `Documentation/01_User_Guide/03_Installation.md` to include instructions for installing the new `py7zr` and `rarfile` dependencies (likely via `pip install -r requirements.txt`). - * Revise `Documentation/02_Developer_Guide/05_Processing_Pipeline.md` to accurately describe the updated `_extract_input` method, detailing how `.zip`, `.rar`, `.7z`, and directories are handled. - -5. **Testing:** - * Prepare sample `.rar` and `.7z` files (including nested directories and various file types) to test the extraction logic thoroughly. - * Test processing of these new archive types via both the CLI and the Directory Monitor. - * Verify that the subsequent processing steps (classification, map processing, metadata generation, etc.) work correctly with files extracted from `.rar` and `.7z` archives. - -Here is a simplified flow diagram illustrating the updated input handling: - -```mermaid -graph TD - A[Input Source] --> B{Is it a file or directory?}; - B -- Directory --> C[Copy Contents to Workspace]; - B -- File --> D{What is the file extension?}; - D -- .zip --> E[Extract using zipfile]; - D -- .rar --> F[Extract using rarfile]; - D -- .7z --> G[Extract using py7zr]; - E --> H[Temporary Workspace]; - F --> H; - G --> H; - C --> H; - H --> I[Processing Pipeline Starts]; \ No newline at end of file diff --git a/ProjectNotes/FORCE_LOSSLESS_PLAN.md b/ProjectNotes/FORCE_LOSSLESS_PLAN.md deleted file mode 100644 index 9dfca0b..0000000 --- a/ProjectNotes/FORCE_LOSSLESS_PLAN.md +++ /dev/null @@ -1,51 +0,0 @@ -# Plan: Implement "Force Lossless" Format for Specific Map Types - -**Goal:** Modify the asset processor to ensure specific map types ("NRM", "DISP") are always saved in a lossless format (PNG or EXR based on bit depth), overriding the JPG threshold and input format rules. This rule should apply to both individually processed maps and merged maps. - -**Steps:** - -1. **Add Configuration Setting (`config.py`):** - * Introduce a new list named `FORCE_LOSSLESS_MAP_TYPES` in `config.py`. - * Populate this list: `FORCE_LOSSLESS_MAP_TYPES = ["NRM", "DISP"]`. - -2. **Expose Setting in `Configuration` Class (`configuration.py`):** - * Add a default value for `FORCE_LOSSLESS_MAP_TYPES` in the `_load_core_config` method's `default_core_settings` dictionary. - * Add a new property to the `Configuration` class to access this list: - ```python - @property - def force_lossless_map_types(self) -> list: - """Gets the list of map types that must always be saved losslessly.""" - return self._core_settings.get('FORCE_LOSSLESS_MAP_TYPES', []) - ``` - -3. **Modify `_process_maps` Method (`asset_processor.py`):** - * Locate the section determining the output format (around line 805). - * **Before** the `if output_bit_depth == 8 and target_dim >= threshold:` check (line 811), insert the new logic: - * Check if the current `map_type` is in `self.config.force_lossless_map_types`. - * If yes, determine the appropriate lossless format (`png` or configured 16-bit format like `exr`) based on `output_bit_depth`, set `output_format`, `output_ext`, `save_params`, and `needs_float16` accordingly, and skip the subsequent `elif` / `else` blocks for format determination. - * Use an `elif` for the existing JPG threshold check and the final `else` for the rule-based logic, ensuring they only run if `force_lossless` is false. - -4. **Modify `_merge_maps` Method (`asset_processor.py`):** - * Locate the section determining the output format for the merged map (around line 1151). - * **Before** the `if output_bit_depth == 8 and target_dim >= threshold:` check (line 1158), insert similar logic as in step 3: - * Check if the `output_map_type` (the type of the *merged* map) is in `self.config.force_lossless_map_types`. - * If yes, determine the appropriate lossless format based on the merged map's `output_bit_depth`, set `output_format`, `output_ext`, `save_params`, and `needs_float16`, and skip the subsequent `elif` / `else` blocks. - * Use `elif` and `else` for the existing threshold and hierarchy logic. - -**Process Flow Diagram:** - -```mermaid -graph TD - subgraph Format Determination (per resolution, _process_maps & _merge_maps) - A[Start] --> B(Get map_type / output_map_type); - B --> C{Determine output_bit_depth}; - C --> D{Is map_type in FORCE_LOSSLESS_MAP_TYPES?}; - D -- Yes --> E[Set format = Lossless (PNG/EXR based on bit_depth)]; - D -- No --> F{Is output_bit_depth == 8 AND target_dim >= threshold?}; - F -- Yes --> G[Set format = JPG]; - F -- No --> H[Set format based on input/hierarchy/rules]; - G --> I[Set Save Params]; - H --> I; - E --> I; - I --> J[Save Image]; - end \ No newline at end of file diff --git a/ProjectNotes/Failed Attempt at exrsupport.txt b/ProjectNotes/Failed Attempt at exrsupport.txt deleted file mode 100644 index 71bbe36..0000000 --- a/ProjectNotes/Failed Attempt at exrsupport.txt +++ /dev/null @@ -1,17 +0,0 @@ -Here is a summary of our session focused on adding OpenEXR support: - -Goal: -Enable the Asset Processor Tool to reliably write .exr files for 16-bit maps, compatible with Windows executables and Docker. - -Approach: -We decided to use the dedicated openexr Python library for writing EXR files, rather than compiling a custom OpenCV build. The plan involved modifying asset_processor.py to use this library, updating the Dockerfile with necessary system libraries, and outlining steps for PyInstaller on Windows. - -Issues & Fixes: - -Initial Changes: Modified asset_processor.py and Dockerfile. -NameError: name 'log' is not defined: Fixed by moving the logger initialization earlier in asset_processor.py. -AttributeError: module 'Imath' has no attribute 'Header' & NameError: name 'fallback_fmt' is not defined: Attempted fixes by changing Imath.Header to OpenEXR.Header and correcting one instance of fallback_fmt. -TypeError: __init__(): incompatible constructor arguments... Invoked with: HALF & NameError: name 'fallback_fmt' is not defined (again): Corrected the OpenEXR.Channel constructor call and fixed the remaining fallback_fmt typo. -concurrent.futures.process.BrokenProcessPool: The script crashed when attempting the EXR save, likely due to an internal OpenEXR library error or a Windows dependency issue. -Next Steps (when resuming): -Investigate the BrokenProcessPool error, possibly by testing EXR saving with a simple array or verifying Windows dependencies for the OpenEXR library. \ No newline at end of file diff --git a/ProjectNotes/GUI_BLENDER_INTEGRATION_PLAN.md b/ProjectNotes/GUI_BLENDER_INTEGRATION_PLAN.md deleted file mode 100644 index cc4442f..0000000 --- a/ProjectNotes/GUI_BLENDER_INTEGRATION_PLAN.md +++ /dev/null @@ -1,52 +0,0 @@ -# GUI Blender Integration Plan - -## Goal - -Add a checkbox and input fields to the GUI (`gui/main_window.py`) to enable/disable Blender script execution and specify the `.blend` file paths, defaulting to `config.py` values. Integrate this control into the processing logic (`gui/processing_handler.py`). - -## Proposed Plan - -1. **Modify `gui/main_window.py`:** - * Add a `QCheckBox` (e.g., `self.blender_integration_checkbox`) to the processing panel layout. - * Add two pairs of `QLineEdit` widgets and `QPushButton` browse buttons for the nodegroup `.blend` path (`self.nodegroup_blend_path_input`, `self.browse_nodegroup_blend_button`) and the materials `.blend` path (`self.materials_blend_path_input`, `self.browse_materials_blend_button`). - * Initialize the text of the `QLineEdit` widgets by reading the `DEFAULT_NODEGROUP_BLEND_PATH` and `DEFAULT_MATERIALS_BLEND_PATH` values from `config.py` when the GUI starts. - * Connect signals from the browse buttons to new methods that open a `QFileDialog` to select `.blend` files and update the corresponding input fields. - * Modify the slot connected to the "Start Processing" button to: - * Read the checked state of `self.blender_integration_checkbox`. - * Read the text from `self.nodegroup_blend_path_input` and `self.materials_blend_path_input`. - * Pass these three pieces of information (checkbox state, nodegroup path, materials path) to the `ProcessingHandler` when initiating the processing task. - -2. **Modify `gui/processing_handler.py`:** - * Add parameters to the method that starts the processing (likely `start_processing`) to accept the Blender integration flag (boolean), the nodegroup `.blend` path (string), and the materials `.blend` path (string). - * Implement the logic for finding the Blender executable (reading `BLENDER_EXECUTABLE_PATH` from `config.py` or checking PATH) within `processing_handler.py`. - * Implement the logic for executing the Blender scripts using `subprocess.run` within `processing_handler.py`. This logic should be similar to the `run_blender_script` function added to `main.py` in the previous step. - * Ensure this Blender script execution logic is conditional based on the received integration flag and runs *after* the main asset processing (handled by the worker pool) is complete. - -## Execution Flow Diagram (GUI) - -```mermaid -graph TD - A[GUI: User Clicks Start Processing] --> B{Blender Integration Checkbox Checked?}; - B -- Yes --> C[Get Blend File Paths from Input Fields]; - C --> D[Pass Paths and Flag to ProcessingHandler]; - D --> E[ProcessingHandler: Start Asset Processing (Worker Pool)]; - E --> F[ProcessingHandler: Asset Processing Complete]; - F --> G{Blender Integration Flag True?}; - G -- Yes --> H[ProcessingHandler: Find Blender Executable]; - H --> I{Blender Executable Found?}; - I -- Yes --> J{Nodegroup Blend Path Valid?}; - J -- Yes --> K[ProcessingHandler: Run Nodegroup Script in Blender]; - K --> L{Script Successful?}; - L -- Yes --> M{Materials Blend Path Valid?}; - L -- No --> N[ProcessingHandler: Report Nodegroup Error]; - N --> M; - M -- Yes --> O[ProcessingHandler: Run Materials Script in Blender]; - O --> P{Script Successful?}; - P -- Yes --> Q[ProcessingHandler: Report Completion]; - P -- No --> R[ProcessingHandler: Report Materials Error]; - R --> Q; - M -- No --> Q; - J -- No --> M[Skip Nodegroup Script]; - I -- No --> Q[Skip Blender Scripts]; - G -- No --> Q; - B -- No --> E; \ No newline at end of file diff --git a/ProjectNotes/GUI_Enhancement_Plan.md b/ProjectNotes/GUI_Enhancement_Plan.md deleted file mode 100644 index e570749..0000000 --- a/ProjectNotes/GUI_Enhancement_Plan.md +++ /dev/null @@ -1,43 +0,0 @@ -# GUI Enhancement Plan - -## Objective - -Implement two new features in the Graphical User Interface (GUI) of the Asset Processor Tool: -1. Automatically switch the preview to "simple view" when more than 10 input files (ZIPs or folders) are added to the queue. -2. Remove the specific visual area labeled "Drag and drop folders here" while keeping the drag-and-drop functionality active for the main processing panel. - -## Implementation Plan - -The changes will be made in the `gui/main_window.py` file. - -1. **Implement automatic preview switch:** - * Locate the `add_input_paths` method. - * After adding the `newly_added_paths` to `self.current_asset_paths`, check the total number of items in `self.current_asset_paths`. - * If the count is greater than 10, programmatically set the state of the "Disable Detailed Preview" menu action (`self.toggle_preview_action`) to `checked=True`. This will automatically trigger the `update_preview` method, which will then render the simple list view. - -2. **Remove the "Drag and drop folders here" visual area:** - * Locate the `setup_main_panel_ui` method. - * Find the creation of the `self.drag_drop_area` QFrame and its associated QLabel (`drag_drop_label`). - * Add a line after the creation of `self.drag_drop_area` to hide this widget (`self.drag_drop_area.setVisible(False)`). This will remove the visual box and label while keeping the drag-and-drop functionality enabled for the main window. - -## Workflow Diagram - -```mermaid -graph TD - A[User drops files/folders] --> B{Call add_input_paths} - B --> C[Add paths to self.current_asset_paths] - C --> D{Count items in self.current_asset_paths} - D{Count > 10?} -->|Yes| E[Set toggle_preview_action.setChecked(True)] - D{Count > 10?} -->|No| F[Keep current preview state] - E --> G[update_preview triggered] - F --> G[update_preview triggered] - G --> H{Check toggle_preview_action state} - H{Checked (Simple)?} -->|Yes| I[Display simple list in preview_table] - H{Checked (Simple)?} -->|No| J[Run PredictionHandler for detailed preview] - J --> K[Display detailed results in preview_table] - - L[GUI Initialization] --> M[Call setup_main_panel_ui] - M --> N[Create drag_drop_area QFrame] - N --> O[Hide drag_drop_area QFrame] - O --> P[Main window accepts drops] - P --> B \ No newline at end of file diff --git a/ProjectNotes/GUI_FEATURE_PLAN.md b/ProjectNotes/GUI_FEATURE_PLAN.md deleted file mode 100644 index baadff5..0000000 --- a/ProjectNotes/GUI_FEATURE_PLAN.md +++ /dev/null @@ -1,103 +0,0 @@ -# GUI Feature Enhancement Plan - -**Overall Goal:** Modify the GUI (`gui/main_window.py`, `gui/prediction_handler.py`) to make the output path configurable, improve UI responsiveness during preview generation, and add a toggle to switch between detailed file preview and a simple input path list. - -**Detailed Plan:** - -1. **Feature: Configurable Output Path** - * **File:** `gui/main_window.py` - * **Changes:** - * **UI Addition:** - * Below the `preset_combo` layout, add a new `QHBoxLayout`. - * Inside this layout, add: - * A `QLabel` with text "Output Directory:". - * A `QLineEdit` (e.g., `self.output_path_edit`) to display/edit the path. Make it read-only initially if preferred, or editable. - * A `QPushButton` (e.g., `self.browse_output_button`) with text "Browse...". - * **Initialization (`__init__` or `setup_main_panel_ui`):** - * Read the default `OUTPUT_BASE_DIR` from `core_config`. - * Resolve this path relative to the project root (`project_root / output_base_dir_config`). - * Set the initial text of `self.output_path_edit` to this resolved default path. - * **Browse Button Logic:** - * Connect the `clicked` signal of `self.browse_output_button` to a new method (e.g., `_browse_for_output_directory`). - * Implement `_browse_for_output_directory`: - * Use `QFileDialog.getExistingDirectory` to let the user select a folder. - * If a directory is selected, update the text of `self.output_path_edit`. - * **Processing Logic (`start_processing`):** - * Instead of reading/resolving the path from `core_config`, get the path string directly from `self.output_path_edit.text()`. - * Convert this string to a `Path` object. - * **Add Validation:** Before passing the path to the handler, check if the directory exists. If not, attempt to create it using `output_dir.mkdir(parents=True, exist_ok=True)`. Handle potential `OSError` exceptions during creation and show an error message if it fails. Also, consider adding a basic writability check if possible. - * Pass the validated `output_dir_str` to `self.processing_handler.run_processing`. - -2. **Feature: Responsive UI (Address Prediction Bottleneck)** - * **File:** `gui/prediction_handler.py` - * **Changes:** - * **Import:** Add `from concurrent.futures import ThreadPoolExecutor, as_completed`. - * **Modify `run_prediction`:** - * Inside the `try` block (after loading `config`), create a `ThreadPoolExecutor` (e.g., `with ThreadPoolExecutor(max_workers=...) as executor:`). Determine a reasonable `max_workers` count (e.g., `os.cpu_count() // 2` or a fixed number like 4 or 8). - * Instead of iterating through `input_paths` sequentially, submit a task to the executor for each `input_path_str`. - * The task submitted should be a helper method (e.g., `_predict_single_asset`) that takes `input_path_str` and the loaded `config` object as arguments. - * `_predict_single_asset` will contain the logic currently inside the loop: instantiate `AssetProcessor`, call `get_detailed_file_predictions`, handle exceptions, and return the list of prediction dictionaries for that *single* asset (or an error dictionary). - * Store the `Future` objects returned by `executor.submit`. - * Use `as_completed(futures)` to process results as they become available. - * Append the results from each completed future to the `all_file_results` list. - * Emit `prediction_results_ready` once at the very end with the complete `all_file_results` list. - * **File:** `gui/main_window.py` - * **Changes:** - * No changes needed in the `on_prediction_results_ready` slot itself, as the handler will still emit the full list at the end. - -3. **Feature: Preview Toggle** - * **File:** `gui/main_window.py` - * **Changes:** - * **UI Addition:** - * Add a `QCheckBox` (e.g., `self.disable_preview_checkbox`) with text "Disable Detailed Preview". Place it logically, perhaps near the `overwrite_checkbox` or above the `preview_table`. Set its default state to unchecked. - * **Modify `update_preview`:** - * At the beginning of the method, check `self.disable_preview_checkbox.isChecked()`. - * **If Checked (Simple View):** - * Clear the `preview_table`. - * Set simplified table headers (e.g., `self.preview_table.setColumnCount(1); self.preview_table.setHorizontalHeaderLabels(["Input Path"])`). Adjust column resize modes. - * Iterate through `self.current_asset_paths`. For each path, add a row to the table containing just the path string. - * Set status bar message (e.g., "Preview disabled. Showing input list."). - * **Crucially:** `return` from the method here to prevent the `PredictionHandler` from being started. - * **If Unchecked (Detailed View):** - * Ensure the table headers and column count are set back to the detailed view configuration (Status, Original Path, Predicted Name, Details). - * Continue with the rest of the existing `update_preview` logic to start the `PredictionHandler`. - * **Connect Signal:** In `__init__` or `setup_main_panel_ui`, connect the `toggled` signal of `self.disable_preview_checkbox` to the `self.update_preview` slot. - * **Initial State:** Ensure the first call to `update_preview` (if any) respects the initial unchecked state of the checkbox. - -**Mermaid Diagram:** - -```mermaid -graph TD - subgraph MainWindow - A[User Action: Add Asset / Change Preset / Toggle Preview] --> B{Update Preview Triggered}; - B --> C{Is 'Disable Preview' Checked?}; - C -- Yes --> D[Show Simple List View in Table]; - C -- No --> E[Set Detailed Table Headers]; - E --> F[Start PredictionHandler Thread]; - F --> G[PredictionHandler Runs]; - G --> H[Slot: Populate Table with Detailed Results]; - - I[User Clicks Start Processing] --> J{Get Output Path from UI LineEdit}; - J --> K[Validate/Create Output Path]; - K -- Path OK --> L[Start ProcessingHandler Thread]; - K -- Path Error --> M[Show Error Message]; - L --> N[ProcessingHandler Runs]; - N --> O[Update UI (Progress, Status)]; - - P[User Clicks Browse...] --> Q[Show QFileDialog]; - Q --> R[Update Output Path LineEdit]; - end - - subgraph PredictionHandler [Background Thread] - style PredictionHandler fill:#f9f,stroke:#333,stroke-width:2px - F --> S{Use ThreadPoolExecutor}; - S --> T[Run _predict_single_asset Concurrently]; - T --> U[Collect Results]; - U --> V[Emit prediction_results_ready (Full List)]; - V --> H; - end - - subgraph ProcessingHandler [Background Thread] - style ProcessingHandler fill:#ccf,stroke:#333,stroke-width:2px - L --> N; - end \ No newline at end of file diff --git a/ProjectNotes/GUI_LOG_CONSOLE_PLAN.md b/ProjectNotes/GUI_LOG_CONSOLE_PLAN.md deleted file mode 100644 index 2c7020a..0000000 --- a/ProjectNotes/GUI_LOG_CONSOLE_PLAN.md +++ /dev/null @@ -1,78 +0,0 @@ -# GUI Log Console Feature Plan - -**Overall Goal:** Add a log console panel to the GUI's editor panel, controlled by a "View" menu action. Move the "Disable Detailed Preview" control to the same "View" menu. - -**Detailed Plan:** - -1. **Create Custom Log Handler:** - * **New File/Location:** Potentially add this to a new `gui/log_handler.py` or keep it within `gui/main_window.py` if simple enough. - * **Implementation:** - * Define a class `QtLogHandler(logging.Handler, QObject)` that inherits from both `logging.Handler` and `QObject` (for signals). - * Add a Qt signal, e.g., `log_record_received = Signal(str)`. - * Override the `emit(self, record)` method: - * Format the log record using `self.format(record)`. - * Emit the `log_record_received` signal with the formatted string. - -2. **Modify `gui/main_window.py`:** - * **Imports:** Add `QMenuBar`, `QMenu`, `QAction` from `PySide6.QtWidgets`. Import the new `QtLogHandler`. - * **UI Elements (`__init__` / `setup_editor_panel_ui`):** - * **Menu Bar:** - * Create `self.menu_bar = self.menuBar()`. - * Create `view_menu = self.menu_bar.addMenu("&View")`. - * **Log Console:** - * Create `self.log_console_output = QTextEdit()`. Set it to read-only (`self.log_console_output.setReadOnly(True)`). - * Create a container widget, e.g., `self.log_console_widget = QWidget()`. Create a layout for it (e.g., `QVBoxLayout`) and add `self.log_console_output` to this layout. - * In `setup_editor_panel_ui`, insert `self.log_console_widget` into the `editor_layout` *before* adding the `list_layout` (the preset list). - * Initially hide the console: `self.log_console_widget.setVisible(False)`. - * **Menu Actions:** - * Create `self.toggle_log_action = QAction("Show Log Console", self, checkable=True)`. Connect `self.toggle_log_action.toggled.connect(self._toggle_log_console_visibility)`. Add it to `view_menu`. - * Create `self.toggle_preview_action = QAction("Disable Detailed Preview", self, checkable=True)`. Connect `self.toggle_preview_action.toggled.connect(self.update_preview)`. Add it to `view_menu`. - * **Remove Old Checkbox:** Delete the lines creating and adding `self.disable_preview_checkbox`. - * **Logging Setup (`__init__`):** - * Instantiate the custom handler: `self.log_handler = QtLogHandler()`. - * Connect its signal: `self.log_handler.log_record_received.connect(self._append_log_message)`. - * Add the handler to the logger: `log.addHandler(self.log_handler)`. Set an appropriate level if needed (e.g., `self.log_handler.setLevel(logging.INFO)`). - * **New Slots:** - * Implement `_toggle_log_console_visibility(self, checked)`: This slot will simply call `self.log_console_widget.setVisible(checked)`. - * Implement `_append_log_message(self, message)`: - * Append the `message` string to `self.log_console_output`. - * Optional: Add logic to limit the number of lines in the text edit to prevent performance issues. - * Optional: Add basic HTML formatting for colors based on log level. - * **Modify `update_preview`:** - * Replace the check for `self.disable_preview_checkbox.isChecked()` with `self.toggle_preview_action.isChecked()`. - * Update the log messages within this method to reflect checking the action state. - -**Mermaid Diagram:** - -```mermaid -graph TD - subgraph MainWindow - A[Initialization] --> B(Create Menu Bar); - B --> C(Add View Menu); - C --> D(Add 'Show Log Console' Action); - C --> E(Add 'Disable Detailed Preview' Action); - A --> F(Create Log Console QTextEdit); - F --> G(Place Log Console Widget in Layout [Hidden]); - A --> H(Create & Add QtLogHandler); - H --> I(Connect Log Handler Signal to _append_log_message); - - D -- Toggled --> J[_toggle_log_console_visibility]; - J --> K(Show/Hide Log Console Widget); - - E -- Toggled --> L[update_preview]; - - M[update_preview] --> N{Is 'Disable Preview' Action Checked?}; - N -- Yes --> O[Show Simple List View]; - N -- No --> P[Start PredictionHandler]; - - Q[Any Log Message] -- Emitted by Logger --> H; - I --> R[_append_log_message]; - R --> S(Append Message to Log Console QTextEdit); - end - - subgraph QtLogHandler - style QtLogHandler fill:#lightgreen,stroke:#333,stroke-width:2px - T1[emit(record)] --> T2(Format Record); - T2 --> T3(Emit log_record_received Signal); - T3 --> I; - end \ No newline at end of file diff --git a/ProjectNotes/GUI_PLAN.md b/ProjectNotes/GUI_PLAN.md deleted file mode 100644 index fb0ee14..0000000 --- a/ProjectNotes/GUI_PLAN.md +++ /dev/null @@ -1,119 +0,0 @@ -# Asset Processor GUI Development Plan - -This document outlines the plan for developing a Graphical User Interface (GUI) for the Asset Processor Tool. - -**I. Foundation & Framework Choice** - -1. **Choose GUI Framework:** PySide6 (LGPL license, powerful features). -2. **Project Structure:** Create `gui/` directory. Core logic remains separate. -3. **Dependencies:** Add `PySide6` to `requirements.txt`. - -**II. Core GUI Layout & Basic Functionality** - -1. **Main Window:** `gui/main_window.py`. -2. **Layout Elements:** - * Drag & Drop Area (`QWidget` subclass). - * Preset Dropdown (`QComboBox`). - * Preview Area (`QListWidget`). - * Progress Bar (`QProgressBar`). - * Control Buttons (`QPushButton`: Start, Cancel, Manage Presets). - * Status Bar (`QStatusBar`). - -**III. Input Handling & Predictive Preview** - -1. **Connect Drag & Drop:** Validate drops, add valid paths to Preview List. -2. **Refactor for Prediction:** Create/refactor methods in `AssetProcessor`/`Configuration` to predict output names without full processing. -3. **Implement Preview Update:** On preset change or file add, load config, call prediction logic, update Preview List items (e.g., "Input: ... -> Output: ..."). -4. **Responsive Preview:** Utilize PySide6 list widget efficiency (consider model/view for very large lists). - -**IV. Processing Integration & Progress Reporting** - -1. **Adapt Processing Logic:** Refactor `main.py`'s `ProcessPoolExecutor` loop into a callable function/class (`gui/processing_handler.py`). -2. **Background Execution:** Run processing logic in a `QThread` to keep GUI responsive. -3. **Progress Updates (Signals & Slots):** - * Background thread emits signals: `progress_update(current, total)`, `file_status_update(path, status, msg)`, `processing_finished(stats)`. - * Main window connects slots to these signals to update UI elements (`QProgressBar`, `QListWidget` items, status bar). -4. **Completion/Error Handling:** Re-enable controls, display summary stats, report errors on `processing_finished`. - -**V. Preset Management Interface (Sub-Task)** - -1. **New Window/Dialog:** `gui/preset_editor.py`. -2. **Functionality:** List, load, edit (tree view/form), create, save/save as presets (`.json` in `presets/`). Basic validation. - -**VI. Refinements & Additional Features (Ideas)** - -1. **Cancel Button:** Implement cancellation signal to background thread/workers. -2. **Log Viewer:** Add `QTextEdit` to display `logging` output. -3. **Output Directory Selection:** Add browse button/field. -4. **Configuration Options:** Expose key `config.py` settings. -5. **Clearer Error Display:** Tooltips, status bar updates, or error panel. - -**VII. Packaging (Deployment)** - -1. **Tooling:** PyInstaller or cx_Freeze. -2. **Configuration:** Build scripts (`.spec`) to bundle code, dependencies, assets, and `presets/`. - -**High-Level Mermaid Diagram:** - -```mermaid -graph TD - subgraph GUI Application (PySide6) - A[Main Window] --> B(Drag & Drop Area); - A --> C(Preset Dropdown); - A --> D(Preview List Widget); - A --> E(Progress Bar); - A --> F(Start Button); - A -- Contains --> SB(Status Bar); - A --> CB(Cancel Button); - A --> MB(Manage Presets Button); - - B -- fileDroppedSignal --> X(Handle Input Files); - C -- currentIndexChangedSignal --> Y(Update Preview); - X -- Adds paths --> D; - X -- Triggers --> Y; - - Y -- Reads --> C; - Y -- Uses --> J(Prediction Logic); - Y -- Updates --> D; - - F -- clickedSignal --> Z(Start Processing); - CB -- clickedSignal --> AA(Cancel Processing); - MB -- clickedSignal --> L(Preset Editor Dialog); - - Z -- Starts --> BB(Processing Thread: QThread); - AA -- Signals --> BB; - - BB -- progressSignal(curr, total) --> E; - BB -- fileStatusSignal(path, status, msg) --> D; - BB -- finishedSignal(stats) --> AB(Handle Processing Finished); - AB -- Updates --> SB; - AB -- Enables/Disables --> F; - AB -- Enables/Disables --> CB; - AB -- Enables/Disables --> C; - AB -- Enables/Disables --> B; - - L -- Modifies --> H(Presets Dir: presets/*.json); - end - - subgraph Backend Logic (Existing + Refactored) - H -- Loaded by --> C; - H -- Loaded/Saved by --> L; - - J -- Reads --> M(configuration.py / asset_processor.py); - BB -- Runs --> K(Adapted main.py Logic); - K -- Uses --> N(ProcessPoolExecutor); - N -- Runs --> O(process_single_asset_wrapper); - O -- Uses --> M; - O -- Reports Status --> K; - K -- Reports Progress --> BB; - end - - classDef gui fill:#f9f,stroke:#333,stroke-width:2px; - classDef backend fill:#ccf,stroke:#333,stroke-width:2px; - classDef thread fill:#ffc,stroke:#333,stroke-width:1px; - class A,B,C,D,E,F,CB,MB,L,SB,X,Y,Z,AA,AB gui; - class H,J,K,M,N,O backend; - class BB thread; -``` - -This plan provides a roadmap for the GUI development. \ No newline at end of file diff --git a/ProjectNotes/GUI_Preset_Selection_Plan.md b/ProjectNotes/GUI_Preset_Selection_Plan.md deleted file mode 100644 index 55f51ee..0000000 --- a/ProjectNotes/GUI_Preset_Selection_Plan.md +++ /dev/null @@ -1,42 +0,0 @@ -# GUI Preset Selection Plan - -## Objective - -Modify the GUI so that no preset is selected by default, and the preview table is only populated after the user explicitly selects a preset. This aims to prevent accidental processing with an unintended preset and clearly indicate to the user that a preset selection is required for preview. - -## Plan - -1. **Modify `gui/main_window.py`:** - * Remove the logic that selects a default preset during initialization. - * Initialize the preview table to display the text "please select a preset". - * Disable the mechanism that triggers the `PredictionHandler` for preview generation until a preset is selected. - * Update the slot connected to the preset selection signal: - * When a preset is selected, clear the placeholder text and enable the preview generation mechanism. - * Pass the selected preset configuration to the `PredictionHandler`. - * Trigger the `PredictionHandler` to generate and display the preview. - * (Optional but recommended) Add logic to handle the deselection of a preset, which should clear the preview table and display the "please select a preset" text again, and disable preview generation. - -2. **Review `gui/prediction_handler.py`:** - * Verify that the `PredictionHandler`'s methods that generate predictions (`get_detailed_file_predictions`) correctly handle being called only when a valid preset is provided. No major changes are expected here, but it's good practice to confirm. - -3. **Update Preview Table Handling (`gui/preview_table_model.py` and `gui/main_window.py`):** - * Ensure the `PreviewTableModel` can gracefully handle having no data when no preset is selected. - * In `gui/main_window.py`, configure the `QTableView` or its parent widget to display the placeholder text "please select a preset" when the model is empty or no preset is active. - -## Data Flow Change - -The current data flow for preview generation is roughly: -Initialization -> Default Preset Loaded -> Trigger PredictionHandler -> Update Preview Table - -The proposed data flow would be: -Initialization -> No Preset Selected -> Preview Table Empty/Placeholder -> User Selects Preset -> Trigger PredictionHandler with Selected Preset -> Update Preview Table - -```mermaid -graph TD - A[GUI Initialization] --> B{Is a Preset Selected?}; - B -- Yes (Current) --> C[Load Default Preset]; - B -- No (Proposed) --> D[Preview Table Empty/Placeholder]; - C --> E[Trigger PredictionHandler]; - D --> F[User Selects Preset]; - F --> E; - E --> G[Update Preview Table]; \ No newline at end of file diff --git a/ProjectNotes/GUI_PreviewTable_AlternatingColors_Plan.md b/ProjectNotes/GUI_PreviewTable_AlternatingColors_Plan.md deleted file mode 100644 index 168b9b6..0000000 --- a/ProjectNotes/GUI_PreviewTable_AlternatingColors_Plan.md +++ /dev/null @@ -1,59 +0,0 @@ -# Plan: Implement Alternating Row Colors Per Asset Group in GUI Preview Table - -## Objective - -Modify the GUI preview table to display alternating background colors for rows based on the asset group they belong to, rather than alternating colors for each individual row. The visual appearance should be similar to the default alternating row colors (dark greys, no border, no rounded corners). - -## Current State - -The preview table in the GUI uses a `QTableView` with `setAlternatingRowColors(True)` enabled, which applies alternating background colors based on the row index. The `PreviewTableModel` groups file prediction data by `source_asset` in its internal `_table_rows` structure and provides data to the view. - -## Proposed Plan - -To achieve alternating colors per asset group, we will implement custom coloring logic within the `PreviewTableModel`. - -1. **Disable Default Alternating Colors:** - * In `gui/main_window.py`, locate the initialization of the `preview_table_view` (a `QTableView`). - * Change `self.preview_table_view.setAlternatingRowColors(True)` to `self.preview_table_view.setAlternatingRowColors(False)`. - -2. **Modify `PreviewTableModel.data()`:** - * Open `gui/preview_table_model.py`. - * In the `data()` method, add a case to handle the `Qt.ItemDataRole.BackgroundRole`. - * Inside this case, retrieve the `source_asset` for the current row from the `self._table_rows` structure. - * Maintain a sorted list of unique `source_asset` values. This can be done when the data is set in `set_data()`. - * Find the index of the current row's `source_asset` within this sorted list. - * Based on whether the index is even or odd, return a `QColor` object representing one of the two desired grey colors. - * Ensure that the `Qt.ItemDataRole.BackgroundRole` is handled correctly for all columns in the row. - -3. **Define Colors:** - * Define two `QColor` objects within the `PreviewTableModel` class to represent the two grey colors for alternating groups. These should be chosen to be visually similar to the default alternating row colors. - -## Visual Representation of Data Flow with Custom Coloring - -```mermaid -graph TD - A[QTableView] --> B{Requests Data for Row/Column}; - B --> C[PreviewSortFilterProxyModel]; - C --> D[PreviewTableModel]; - D -- data(index, role) --> E{Check Role}; - E -- Qt.ItemDataRole.BackgroundRole --> F{Get source_asset for row}; - F --> G{Determine Asset Group Index}; - G --> H{Assign Color based on Index Parity}; - H --> I[Return QColor]; - E -- Other Roles --> J[Return Display/Tooltip/Foreground Data}; - I --> C; - J --> C; - C --> A{Displays Data with Custom Background Color}; - - style D fill:#f9f,stroke:#333,stroke-width:2px - style H fill:#ccf,stroke:#333,stroke-width:1px - style I fill:#ccf,stroke:#333,stroke-width:1px -``` - -## Implementation Steps (for Code Mode) - -1. Modify `gui/main_window.py` to disable default alternating row colors. -2. Modify `gui/preview_table_model.py` to: - * Define the two grey `QColor` objects. - * Update `set_data()` to create and store a sorted list of unique asset groups. - * Implement the `Qt.ItemDataRole.BackgroundRole` logic in the `data()` method to return alternating colors based on the asset group index. \ No newline at end of file diff --git a/ProjectNotes/GUI_PreviewTable_Coloring_Plan.md b/ProjectNotes/GUI_PreviewTable_Coloring_Plan.md deleted file mode 100644 index fea7a1b..0000000 --- a/ProjectNotes/GUI_PreviewTable_Coloring_Plan.md +++ /dev/null @@ -1,49 +0,0 @@ -# Plan: Enhance GUI Preview Table Coloring - -## Objective - -Modify the GUI preview table to apply status-based text coloring to all relevant cells in a row, providing a more consistent visual indication of a file's status. - -## Current State - -The `PreviewTableModel` in `gui/preview_table_model.py` currently applies status-based text colors only to the "Status" column (based on the main file's status) and the "Additional Files" column (based on the additional file's status). Other cells in the row do not have status-based coloring. - -## Proposed Change - -Extend the status-based text coloring logic in the `PreviewTableModel`'s `data()` method to apply the relevant status color to any cell that corresponds to either the main file or an additional file in that row. - -## Plan - -1. **Modify the `data()` method in `gui/preview_table_model.py`:** - * Locate the section handling the `Qt.ItemDataRole.ForegroundRole`. - * Currently, this section checks the column index (`col`) to decide which file's status to use for coloring (main file for `COL_STATUS`, additional file for `COL_ADDITIONAL_FILES`). - * We will change this logic to determine which file (main or additional) the *current row and column* corresponds to, and then use that file's status to look up the color. - * For columns related to the main file (`COL_STATUS`, `COL_PREDICTED_ASSET`, `COL_ORIGINAL_PATH`, `COL_PREDICTED_OUTPUT`, `COL_DETAILS`), if the row contains a `main_file`, use the `main_file`'s status for coloring. - * For the `COL_ADDITIONAL_FILES` column, if the row contains `additional_file_details`, use the `additional_file_details`' status for coloring. - * If a cell does not correspond to a file (e.g., a main file column in a row that only has an additional file), return `None` for the `ForegroundRole` to use the default text color. - -## Detailed Steps - -1. Open `gui/preview_table_model.py`. -2. Navigate to the `data()` method. -3. Find the `if role == Qt.ItemDataRole.ForegroundRole:` block. -4. Inside this block, modify the logic to determine the `status` variable based on the current `col` and the presence of `main_file` or `additional_file_details` in `row_data`. -5. Use the determined `status` to look up the color in `self.STATUS_COLORS`. -6. Return the color if found, otherwise return `None`. - -## Modified Color Logic Flow - -```mermaid -graph TD - A[data(index, role)] --> B{role == Qt.ItemDataRole.ForegroundRole?}; - B -- Yes --> C{Determine relevant file for cell (row, col)}; - C -- Cell corresponds to Main File --> D{Get main_file status}; - C -- Cell corresponds to Additional File --> E{Get additional_file_details status}; - C -- Cell is empty --> F[status = None]; - D --> G{Lookup color in STATUS_COLORS}; - E --> G; - F --> H[Return None]; - G -- Color found --> I[Return Color]; - G -- No color found --> H; - B -- No --> J[Handle other roles]; - J --> K[Return data based on role]; \ No newline at end of file diff --git a/ProjectNotes/GUI_Preview_Table_Restructure_Plan.md b/ProjectNotes/GUI_Preview_Table_Restructure_Plan.md deleted file mode 100644 index 0096eb2..0000000 --- a/ProjectNotes/GUI_Preview_Table_Restructure_Plan.md +++ /dev/null @@ -1,90 +0,0 @@ -# GUI Preview Table Restructure Plan - -## Objective - -Restructure the Graphical User Interface (GUI) preview table to group files by source asset and display "Ignored" and "Extra" files in a new "Additional Files" column, aligned with the mapped files of the same asset. - -## Analysis - -Based on the review of `gui/prediction_handler.py` and `gui/preview_table_model.py`: - -* The `PredictionHandler` provides a flat list of file prediction dictionaries. -* The `PreviewTableModel` currently stores and displays this flat list directly. -* The `PreviewSortFilterProxyModel` sorts this flat list. -* The data transformation to achieve the desired grouped layout must occur within the `PreviewTableModel`. - -## Proposed Plan - -1. **Modify `gui/preview_table_model.py`:** - * **Add New Column:** - * Define a new constant: `COL_ADDITIONAL_FILES = 5`. - * Add "Additional Files" to the `_headers_detailed` list. - * **Introduce New Internal Data Structure:** - * Create a new internal list, `self._table_rows`, to store dictionaries representing the final rows to be displayed in the table. - * **Update `set_data(self, data: list)`:** - * Process the incoming flat `data` list (received from `PredictionHandler`). - * Group file dictionaries by their `source_asset`. - * Within each asset group, separate files into two lists: - * `main_files`: Files with status "Mapped", "Model", or "Error". - * `additional_files`: Files with status "Ignored", "Extra", "Unrecognised", or "Unmatched Extra". - * Determine the maximum number of rows needed for this asset block: `max(len(main_files), len(additional_files))`. - * Build the row dictionaries for `self._table_rows` for this asset block: - * For `i` from 0 to `max_rows - 1`: - * Get the `i`-th file from `main_files` (or `None` if `i` is out of bounds). - * Get the `i`-th file from `additional_files` (or `None` if `i` is out of bounds). - * Create a row dictionary containing: - * `source_asset`: The asset name. - * `predicted_asset`: From the `main_file` (if exists). - * `details`: From the `main_file` (if exists). - * `original_path`: From the `main_file` (if exists). - * `additional_file_path`: Path from the `additional_file` (if exists). - * `additional_file_details`: The original dictionary of the `additional_file` (if exists, for tooltips). - * `is_main_row`: Boolean flag (True if this row corresponds to a file in `main_files`, False otherwise). - * Append these row dictionaries to `self._table_rows`. - * After processing all assets, call `self.beginResetModel()` and `self.endResetModel()`. - * **Update `rowCount`:** Return `len(self._table_rows)` when in detailed mode. - * **Update `columnCount`:** Return `len(self._headers_detailed)`. - * **Update `data(self, index, role)`:** - * Retrieve the row dictionary: `row_data = self._table_rows[index.row()]`. - * For `Qt.ItemDataRole.DisplayRole`: - * If `index.column()` is `COL_ADDITIONAL_FILES`, return `row_data.get('additional_file_path', '')`. - * For other columns (`COL_STATUS`, `COL_PREDICTED_ASSET`, `COL_ORIGINAL_PATH`, `COL_DETAILS`), return data from the `main_file` part of `row_data` if `row_data['is_main_row']` is True, otherwise return an empty string or appropriate placeholder. - * For `Qt.ItemDataRole.ToolTipRole`: - * If `index.column()` is `COL_ADDITIONAL_FILES` and `row_data.get('additional_file_details')` exists, generate a tooltip using the status and details from `additional_file_details`. - * For other columns, use the existing tooltip logic based on the `main_file` data. - * For `Qt.ItemDataRole.ForegroundRole`: - * Apply existing color-coding based on the status of the `main_file` if `row_data['is_main_row']` is True. - * For the `COL_ADDITIONAL_FILES` cell and for rows where `row_data['is_main_row']` is False, use neutral styling (default text color). - * **Update `headerData`:** Return the correct header for `COL_ADDITIONAL_FILES`. - -2. **Modify `gui/preview_table_model.py` (`PreviewSortFilterProxyModel`):** - * **Update `lessThan(self, left, right)`:** - * Retrieve the row dictionaries for `left` and `right` indices from the source model (`model._table_rows[left.row()]`, etc.). - * **Level 1: Source Asset:** Compare `source_asset` from the row dictionaries. - * **Level 2: Row Type:** If assets are the same, compare `is_main_row` (True sorts before False). - * **Level 3 (Main Rows):** If both are main rows (`is_main_row` is True), compare `original_path`. - * **Level 4 (Additional-Only Rows):** If both are additional-only rows (`is_main_row` is False), compare `additional_file_path`. - -## Clarifications & Decisions - -* **Error Handling:** "Error" files will remain in the main columns, similar to "Mapped" files, with their current "Error" status. -* **Sorting within Asset:** The proposed sorting logic within an asset block is acceptable (mapped rows by original path, additional-only rows by additional file path). -* **Styling of Additional Column:** Use neutral text and background styling for the "Additional Files" column, relying on tooltips for specific file details. - -## Mermaid Diagram (Updated Data Flow) - -```mermaid -graph LR - A[PredictionHandler] -- prediction_results_ready(flat_list) --> B(PreviewTableModel); - subgraph PreviewTableModel - C[set_data] -- Processes flat_list --> D{Internal Grouping & Transformation}; - D -- Creates --> E[_table_rows (Structured List)]; - F[data()] -- Reads from --> E; - end - B -- Provides data via data() --> G(QTableView via Proxy); - - style B fill:#f9f,stroke:#333,stroke-width:2px - style C fill:#ccf,stroke:#333,stroke-width:1px - style D fill:#lightgrey,stroke:#333,stroke-width:1px - style E fill:#ccf,stroke:#333,stroke-width:1px - style F fill:#ccf,stroke:#333,stroke-width:1px \ No newline at end of file diff --git a/ProjectNotes/GUI_REFACTOR_PLAN.md b/ProjectNotes/GUI_REFACTOR_PLAN.md deleted file mode 100644 index edf1a63..0000000 --- a/ProjectNotes/GUI_REFACTOR_PLAN.md +++ /dev/null @@ -1,123 +0,0 @@ -# Asset Processor GUI Refactor Plan - -This document outlines the plan to refactor the Asset Processor GUI based on user requirements. - -## Goals - -1. **Improve File Visibility:** Display all files found within an asset in the preview list, including those that don't match the preset, are moved to 'Extra', or have errors, along with their status. -2. **Integrate Preset Editor:** Move the preset editing functionality from the separate dialog into a collapsible panel within the main window. - -## Goal 1: Improve File Visibility in Preview List - -**Problem:** The current preview (`PredictionHandler` calling `AssetProcessor.predict_output_structure`) only shows files that successfully match a map type rule and get a predicted output name. It doesn't show files that are ignored, moved to 'Extra', or encounter errors during classification. - -**Solution:** Leverage the more comprehensive classification logic already present in `AssetProcessor._inventory_and_classify_files` for the GUI preview. - -**Plan Steps:** - -1. **Modify `asset_processor.py`:** - * Create a new method in `AssetProcessor`, perhaps named `get_detailed_file_predictions()`. - * This new method will perform the core steps of `_setup_workspace()`, `_extract_input()`, and `_inventory_and_classify_files()`. - * It will then iterate through *all* categories in `self.classified_files` ('maps', 'models', 'extra', 'ignored'). - * For each file, it will determine a 'status' (e.g., "Mapped", "Model", "Extra", "Ignored", "Error") and attempt to predict the output name (similar to `predict_output_structure` for maps, maybe just the original name for others). - * It will return a more detailed list of dictionaries, each containing: `{'original_path': str, 'predicted_name': str | None, 'status': str, 'details': str | None}`. - * Crucially, this method will *not* perform the actual processing (`_process_maps`, `_merge_maps`, etc.) or file moving, only the classification and prediction. It should also include cleanup (`_cleanup_workspace`). -2. **Modify `gui/prediction_handler.py`:** - * Update `PredictionHandler.run_prediction` to call the new `AssetProcessor.get_detailed_file_predictions()` method instead of `predict_output_structure()`. - * Adapt the code that processes the results to handle the new dictionary format (including the 'status' and 'details' fields). - * Emit this enhanced list via the `prediction_results_ready` signal. -3. **Modify `gui/main_window.py`:** - * In `setup_ui`, add a new column to `self.preview_table` for "Status". Adjust column count and header labels. - * In `on_prediction_results_ready`, populate the new "Status" column using the data received from `PredictionHandler`. - * Consider adding tooltips to the status column to show the 'details' (e.g., the reason for being ignored or moved to extra). - * Optionally, use background colors or icons in the status column for better visual distinction. - -## Goal 2: Integrate Preset Editor into Main Window - -**Problem:** Preset editing requires opening a separate modal dialog, interrupting the main workflow. - -**Solution:** Embed the preset editing controls directly into the main window within a collapsible panel. - -**Plan Steps:** - -1. **Modify `gui/main_window.py` - UI Changes:** - * Remove the "Manage Presets" button (`self.manage_presets_button`). - * Add a collapsible panel (potentially using a `QFrame` with show/hide logic triggered by a button, or a `QDockWidget` if more appropriate) to the left side of the main layout. - * Inside this panel: - * Add the `QListWidget` for displaying presets (`self.preset_list`). - * Add the "New" and "Delete" buttons below the list. (The "Load" button becomes implicit - selecting a preset in the list loads it into the editor). - * Recreate the `QTabWidget` (`self.preset_editor_tabs`) with the "General & Naming" and "Mapping & Rules" tabs. - * Recreate *all* the widgets currently inside the `PresetEditorDialog` tabs (QLineEdit, QTextEdit, QSpinBox, QListWidget+controls, QTableWidget+controls) within the corresponding tabs in the main window's panel. Give them appropriate instance names (e.g., `self.editor_preset_name`, `self.editor_supplier_name`, etc.). - * Add "Save" and "Save As..." buttons within the collapsible panel, likely at the bottom. -2. **Modify `gui/main_window.py` - Logic Integration:** - * Adapt the `populate_presets` method to populate the new `self.preset_list` in the panel. - * Connect `self.preset_list.currentItemChanged` to a new method `load_selected_preset_for_editing`. This method will handle checking for unsaved changes in the editor panel and then load the selected preset's data into the editor widgets (similar to `PresetEditorDialog.load_preset`). - * Implement `save_preset`, `save_preset_as`, `new_preset`, `delete_preset` methods directly within `MainWindow`, adapting the logic from `PresetEditorDialog`. These will interact with the editor widgets in the panel. - * Implement `check_unsaved_changes` logic for the editor panel, prompting the user if they try to load/create/delete a preset or close the application with unsaved edits in the panel. - * Connect the editor widgets' change signals (`textChanged`, `valueChanged`, `itemChanged`, etc.) to a `mark_editor_unsaved` method in `MainWindow`. - * Ensure the main preset selection `QComboBox` (`self.preset_combo`) is repopulated when presets are saved/deleted via the editor panel. -3. **Cleanup:** - * Delete the `gui/preset_editor_dialog.py` file. - * Remove imports and references to `PresetEditorDialog` from `gui/main_window.py`. - -## Visual Plan - -**Current Layout:** - -```mermaid -graph TD - subgraph "Current Main Window Layout" - A[Preset Combo + Manage Button] --> B(Drag & Drop Area); - B --> C{File Preview Table}; - C --> D[Progress Bar]; - D --> E[Options + Start/Cancel Buttons]; - end - - subgraph "Current Preset Editor (Separate Dialog)" - F[Preset List + Load/New/Delete] --> G{Tab Widget}; - subgraph "Tab Widget" - G1[General & Naming Tab] - G2[Mapping & Rules Tab] - end - G --> H[Save / Save As / Close Buttons]; - end - - A -- Manage Button Click --> F; - - style F fill:#f9f,stroke:#333,stroke-width:2px - style G fill:#f9f,stroke:#333,stroke-width:2px - style H fill:#f9f,stroke:#333,stroke-width:2px -``` - -**Proposed Layout:** - -```mermaid -graph TD - subgraph "Proposed Main Window Layout" - direction LR - subgraph "Collapsible Preset Editor Panel (Left)" - P_List[Preset List] --> P_Buttons[New / Delete Buttons] - P_Buttons --> P_Tabs{Tab Widget} - subgraph "Editor Tabs" - P_Tab1[General & Naming] - P_Tab2[Mapping & Rules] - end - P_Tabs --> P_Save[Save / Save As Buttons] - end - - subgraph "Main Area (Right)" - M_Preset[Preset Combo (for processing)] --> M_DragDrop(Drag & Drop Area) - M_DragDrop --> M_Preview{File Preview Table (with Status Column)} - M_Preview --> M_Progress[Progress Bar] - M_Progress --> M_Controls[Options + Start/Cancel Buttons] - end - - P_List -- Selection Loads --> P_Tabs; - P_Save -- Updates --> P_List; - P_List -- Updates --> M_Preset; - - style M_Preview fill:#ccf,stroke:#333,stroke-width:2px - style P_List fill:#cfc,stroke:#333,stroke-width:2px - style P_Tabs fill:#cfc,stroke:#333,stroke-width:2px - style P_Save fill:#cfc,stroke:#333,stroke-width:2px - end \ No newline at end of file diff --git a/ProjectNotes/GUI_STATUS_UPDATE_PLAN.md b/ProjectNotes/GUI_STATUS_UPDATE_PLAN.md deleted file mode 100644 index a5820e1..0000000 --- a/ProjectNotes/GUI_STATUS_UPDATE_PLAN.md +++ /dev/null @@ -1,63 +0,0 @@ -# Plan: Update GUI Preview Status - -**Objective:** Modify the Asset Processor GUI preview to distinguish between files explicitly marked as "Extra" by preset patterns and those that are simply unclassified. - -**Current Statuses:** -* Mapped -* Ignored -* Extra (Includes both explicitly matched and unclassified files) - -**Proposed Statuses:** -* Mapped -* Ignored -* Extra (Files explicitly matched by `move_to_extra_patterns` in the preset) -* Unrecognised (Files not matching any map, model, or explicit extra pattern) - -**Visual Plan:** - -```mermaid -graph TD - A[Start: User Request] --> B{Analyze Request: Split 'Extra' status}; - B --> C{Info Gathering}; - C --> D[Read gui/prediction_handler.py]; - D --> E[Read asset_processor.py]; - E --> F[Read Presets/Poliigon.json]; - F --> G[Read gui/main_window.py]; - G --> H{Identify Code Locations}; - H --> I[asset_processor.py: get_detailed_file_predictions()]; - H --> J[gui/main_window.py: on_prediction_results_ready()]; - I --> K{Plan Code Changes}; - J --> K; - K --> L[Modify asset_processor.py: Differentiate status based on 'reason']; - K --> M[Modify gui/main_window.py: Add color rule for 'Unrecognised' (#92371f)]; - L --> N{Final Plan}; - M --> N; - N --> O[Present Plan to User]; - O --> P{User Approval + Color Choice}; - P --> Q[Switch to Code Mode for Implementation]; - - subgraph "Code Modification" - L - M - end - - subgraph "Information Gathering" - D - E - F - G - end -``` - -**Implementation Steps:** - -1. **Modify `asset_processor.py` (`get_detailed_file_predictions` method):** - * Locate the loop processing the `self.classified_files["extra"]` list (around line 1448). - * Inside this loop, check the `reason` associated with each file: - * If `reason == 'Unclassified'`, set the output `status` to `"Unrecognised"`. - * Otherwise (if the reason indicates an explicit pattern match), set the output `status` to `"Extra"`. - * Adjust the `details` string provided in the output for clarity (e.g., show pattern match reason for "Extra", maybe just "[Unrecognised]" for the new status). - -2. **Modify `gui/main_window.py` (`on_prediction_results_ready` method):** - * Locate the section where text color is applied based on the `status` (around line 673). - * Add a new `elif` condition to handle `status == "Unrecognised"` and assign it the color `QColor("#92371f")`. \ No newline at end of file diff --git a/ProjectNotes/ISSUE-011-plan.md b/ProjectNotes/ISSUE-011-plan.md deleted file mode 100644 index 1df4131..0000000 --- a/ProjectNotes/ISSUE-011-plan.md +++ /dev/null @@ -1,34 +0,0 @@ -# Plan to Resolve ISSUE-011: Blender nodegroup script creates empty assets for skipped items - -**Issue:** The Blender nodegroup creation script (`blenderscripts/create_nodegroups.py`) creates empty asset entries in the target .blend file for assets belonging to categories that the script is designed to skip, even though it correctly identifies them as skippable. - -**Root Cause:** The script creates the parent node group, marks it as an asset, and applies tags *before* checking if the asset category is one that should be skipped for full nodegroup generation. - -**Plan:** - -1. **Analyze `blenderscripts/create_nodegroups.py` (Completed):** Confirmed that parent group creation and asset marking occur before the asset category skip check. -2. **Modify `blenderscripts/create_nodegroups.py`:** - * Relocate the code block responsible for creating/updating the parent node group, marking it as an asset, and applying tags (currently lines 605-645) to *after* the conditional check `if asset_category not in CATEGORIES_FOR_NODEGROUP_GENERATION:` (line 646). - * This ensures that if an asset's category is in the list of categories to be skipped, the `continue` statement will be hit before any actions are taken to create the parent asset entry in the Blender file. - -3. **Testing:** - * Use test assets that represent both categories that *should* and *should not* result in full nodegroup generation based on the `CATEGORIES_FOR_NODEGROUP_GENERATION` list. - * Run the asset processor with these test assets, ensuring the Blender script is executed. - * Inspect the resulting `.blend` file to confirm: - * No `PBRSET_` node groups are created for assets belonging to skipped categories. - * `PBRSET_` node groups are correctly created and populated for assets belonging to categories in `CATEGORIES_FOR_NODEGROUP_GENERATION`. - -4. **Update Ticket Status:** - * Once the fix is implemented and verified, update the `Status` field in `Tickets/ISSUE-011-blender-nodegroup-empty-assets.md` to `Resolved`. - -**Logic Flow:** - -```mermaid -graph TD - A[create_nodegroups.py] --> B{Load Asset Metadata}; - B --> C{Determine Asset Category}; - C --> D{Is Category Skipped?}; - D -- Yes --> E[Exit Processing for Asset]; - D -- No --> F{Create/Update Parent Group}; - F --> G{Mark as Asset & Add Tags}; - G --> H{Proceed with Child Group Creation etc.}; \ No newline at end of file diff --git a/ProjectNotes/MAP_VARIANT_HANDLING_PLAN.md b/ProjectNotes/MAP_VARIANT_HANDLING_PLAN.md deleted file mode 100644 index af9be60..0000000 --- a/ProjectNotes/MAP_VARIANT_HANDLING_PLAN.md +++ /dev/null @@ -1,68 +0,0 @@ -# Map Variant Handling Plan (Revised) - -**Goal:** - -1. Ensure map types listed in a new `RESPECT_VARIANT_MAP_TYPES` config setting (initially just "COL") always receive a numeric suffix (`-1`, `-2`, etc.), based on their order determined by preset keywords and alphabetical sorting within keywords. -2. Ensure all other map types *never* receive a numeric suffix. -3. Correctly prioritize 16-bit map variants (identified by `bit_depth_variants` in presets) over their 8-bit counterparts, ensuring the 8-bit version is ignored/marked as extra and the 16-bit version is correctly classified ("Mapped") in the GUI preview. - -**Affected Files:** - -* `config.py`: To define the `RESPECT_VARIANT_MAP_TYPES` list. -* `asset_processor.py`: To modify the classification and suffix assignment logic according to the new rule. -* `Presets/Poliigon.json`: To remove the conflicting pattern. - -**Plan Details:** - -```mermaid -graph TD - A[Start] --> B(Modify config.py); - B --> C(Modify asset_processor.py); - C --> D(Modify Presets/Poliigon.json); - D --> E{Review Revised Plan}; - E -- Approve --> F(Optional: Write Plan to MD); - F --> G(Switch to Code Mode); - E -- Request Changes --> B; - G --> H[End Plan]; - - subgraph Modifications - B[1. Add RESPECT_VARIANT_MAP_TYPES list to config.py] - C[2. Update suffix logic in asset_processor.py (_inventory_and_classify_files)] - D[3. Remove "*_16BIT*" pattern from move_to_extra_patterns in Presets/Poliigon.json] - end -``` - -1. **Modify `config.py`:** - * **Action:** Introduce a new configuration list named `RESPECT_VARIANT_MAP_TYPES`. - * **Value:** Initialize it as `RESPECT_VARIANT_MAP_TYPES = ["COL"]`. - * **Location:** Add this near other map-related settings like `STANDARD_MAP_TYPES`. - * **Purpose:** To explicitly define which map types should always respect variant numbering via suffixes. - -2. **Modify `asset_processor.py`:** - * **File:** `asset_processor.py` - * **Method:** `_inventory_and_classify_files` - * **Location:** Within Step 5, replacing the suffix assignment logic (currently lines ~470-474). - * **Action:** Implement the new conditional logic for assigning the `final_map_type`. - * **New Logic:** Inside the loop iterating through `final_ordered_candidates` (for each `base_map_type`): - ```python - # Determine final map type based on the new rule - if base_map_type in self.config.respect_variant_map_types: # Check the new config list - # Always assign suffix for types in the list - final_map_type = f"{base_map_type}-{i + 1}" - else: - # Never assign suffix for types NOT in the list - final_map_type = base_map_type - - # Assign to the final map list entry - final_map_list.append({ - "map_type": final_map_type, - # ... rest of the dictionary assignment ... - }) - ``` - * **Purpose:** To implement the strict rule: only types in `RESPECT_VARIANT_MAP_TYPES` get a suffix; all others do not. - -3. **Modify `Presets/Poliigon.json`:** - * **File:** `Presets/Poliigon.json` - * **Location:** Within the `move_to_extra_patterns` list (currently line ~28). - * **Action:** Remove the string `"*_16BIT*"`. - * **Purpose:** To prevent premature classification of 16-bit variants as "Extra", allowing the specific 16-bit prioritization logic to function correctly. \ No newline at end of file diff --git a/ProjectNotes/MEMORY_OPTIMIZATION_PLAN.md b/ProjectNotes/MEMORY_OPTIMIZATION_PLAN.md deleted file mode 100644 index 4c914e7..0000000 --- a/ProjectNotes/MEMORY_OPTIMIZATION_PLAN.md +++ /dev/null @@ -1,90 +0,0 @@ -# Memory Optimization Plan: Strategy 2 - Load Grayscale Directly - -This plan outlines the steps to implement memory optimization strategy #2, which involves loading known grayscale map types directly as grayscale images using OpenCV's `IMREAD_GRAYSCALE` flag. This reduces the memory footprint compared to loading them with `IMREAD_UNCHANGED` and then potentially converting later. - -## 1. Identify Target Grayscale Map Types - -Define a list of map type names (case-insensitive check recommended during implementation) that should always be treated as single-channel grayscale data. - -**Initial List:** -```python -GRAYSCALE_MAP_TYPES = ['HEIGHT', 'ROUGH', 'METAL', 'AO', 'OPC', 'MASK'] -``` -*(Note: This list might need adjustment based on specific preset configurations or workflow requirements.)* - -## 2. Modify `_process_maps` Loading Logic - -Locate the primary image loading section within the `_process_maps` method in `asset_processor.py` (around line 608). - -**Change:** Before calling `cv2.imread`, determine the correct flag based on the `map_type`: - -```python -# (Define GRAYSCALE_MAP_TYPES list earlier in the scope or class) - -# ... inside the loop ... -full_source_path = self.temp_dir / source_path_rel - -# Determine the read flag -read_flag = cv2.IMREAD_GRAYSCALE if map_type.upper() in GRAYSCALE_MAP_TYPES else cv2.IMREAD_UNCHANGED -log.debug(f"Loading source {source_path_rel.name} with flag: {'GRAYSCALE' if read_flag == cv2.IMREAD_GRAYSCALE else 'UNCHANGED'}") - -# Load the image using the determined flag -img_loaded = cv2.imread(str(full_source_path), read_flag) - -if img_loaded is None: - raise AssetProcessingError(f"Failed to load image file: {full_source_path.name} with flag {read_flag}") - -# ... rest of the processing logic ... -``` - -## 3. Modify `_merge_maps` Loading Logic - -Locate the image loading section within the resolution loop in the `_merge_maps` method (around line 881). - -**Change:** Apply the same conditional logic to determine the `imread` flag when loading input maps for merging: - -```python -# ... inside the loop ... -input_file_path = self.temp_dir / res_details['path'] - -# Determine the read flag (reuse GRAYSCALE_MAP_TYPES list) -read_flag = cv2.IMREAD_GRAYSCALE if map_type.upper() in GRAYSCALE_MAP_TYPES else cv2.IMREAD_UNCHANGED -log.debug(f"Loading merge input {input_file_path.name} ({map_type}) with flag: {'GRAYSCALE' if read_flag == cv2.IMREAD_GRAYSCALE else 'UNCHANGED'}") - -# Load the image using the determined flag -img = cv2.imread(str(input_file_path), read_flag) - -if img is None: - raise AssetProcessingError(f"Failed to load merge input {input_file_path.name} with flag {read_flag}") - -# ... rest of the merging logic ... -``` - -## 4. Verification - -During implementation in `code` mode: -* Ensure the `GRAYSCALE_MAP_TYPES` list is defined appropriately (e.g., as a class constant or module-level constant). -* Confirm that downstream code (e.g., stats calculation, channel extraction, data type conversions) correctly handles numpy arrays that might be 2D (grayscale) instead of 3D (BGR/BGRA). The existing code appears to handle this, but it's important to verify. - -## Mermaid Diagram of Change - -```mermaid -graph TD - subgraph _process_maps - A[Loop through map_info] --> B{Is map_type Grayscale?}; - B -- Yes --> C[imread(path, GRAYSCALE)]; - B -- No --> D[imread(path, UNCHANGED)]; - C --> E[Process Image]; - D --> E; - end - subgraph _merge_maps - F[Loop through resolutions] --> G[Loop through required_input_types]; - G --> H{Is map_type Grayscale?}; - H -- Yes --> I[imread(path, GRAYSCALE)]; - H -- No --> J[imread(path, UNCHANGED)]; - I --> K[Use Image in Merge]; - J --> K; - end - - style B fill:#f9f,stroke:#333,stroke-width:2px - style H fill:#f9f,stroke:#333,stroke-width:2px \ No newline at end of file diff --git a/ProjectNotes/OUTPUT_FORMAT_PLAN.md b/ProjectNotes/OUTPUT_FORMAT_PLAN.md deleted file mode 100644 index ad00b61..0000000 --- a/ProjectNotes/OUTPUT_FORMAT_PLAN.md +++ /dev/null @@ -1,103 +0,0 @@ -# Plan: Implement Input-Based Output Format Logic - -This plan outlines the steps to modify the Asset Processor Tool to determine the output format of texture maps based on the input file format and specific rules. - -## Requirements Summary - -Based on user clarifications: - -1. **JPG Input -> JPG Output:** If the original source map is a JPG, the output for that map (at all processed resolutions) will also be JPG (8-bit). -2. **TIF Input -> PNG/EXR Output:** If the original source map is a TIF, the output will be PNG (if the target bit depth is 8-bit, or if 16-bit PNG is the configured preference) or EXR (if the target bit depth is 16-bit and EXR is the configured preference). -3. **Other Inputs (PNG, etc.) -> Configured Output:** For other input formats, the output will follow the existing logic based on target bit depth (using configured 16-bit or 8-bit formats, typically EXR/PNG). -4. **`force_8bit` Rule:** If a map type has a `force_8bit` rule, it overrides the input format. Even if the input was 16-bit TIF, the output will be 8-bit PNG. -5. **Merged Maps:** The output format is determined by the highest format in the hierarchy (EXR > TIF > PNG > JPG) based on the *original* formats of the input files used in the merge. However, if the highest format is TIF, the actual output will be PNG/EXR based on the target bit depth. The target bit depth itself is determined separately by the merge rule's `output_bit_depth` setting. -6. **JPG Resizing:** Resized JPGs will be saved as JPG. - -## Implementation Plan - -**Phase 1: Data Gathering Enhancement** - -1. **Modify `_inventory_and_classify_files` in `asset_processor.py`:** - * When classifying map files, extract and store the original file extension (e.g., `.jpg`, `.tif`, `.png`) along with the `source_path`, `map_type`, etc., within the `self.classified_files["maps"]` list. - -**Phase 2: Implement New Logic for Individual Maps (`_process_maps`)** - -1. **Modify `_process_maps` in `asset_processor.py`:** - * Inside the loop processing each `map_info`: - * Retrieve the stored original file extension. - * Determine the target output bit depth (8 or 16) using the existing logic (`config.get_bit_depth_rule`, source data type). - * **Implement New Format Determination:** - * Initialize `output_format` and `output_ext`. - * Check the `force_8bit` rule first: If the rule is `force_8bit`, set `output_format = 'png'` and `output_ext = '.png'`, regardless of input format. - * If not `force_8bit`: - * If `original_extension == '.jpg'` and `target_bit_depth == 8`: Set `output_format = 'jpg'`, `output_ext = '.jpg'`. - * If `original_extension == '.tif'`: - * If `target_bit_depth == 16`: Determine format (EXR/PNG) and extension based on `config.get_16bit_output_formats()`. - * If `target_bit_depth == 8`: Set `output_format = 'png'`, `output_ext = '.png'`. - * If `original_extension` is neither `.jpg` nor `.tif` (e.g., `.png`): - * If `target_bit_depth == 16`: Determine format (EXR/PNG) and extension based on `config.get_16bit_output_formats()`. - * If `target_bit_depth == 8`: Set `output_format = config.get_8bit_output_format()` (likely 'png'), `output_ext = f".{output_format}"`. - * **Remove Old Logic:** Delete the code block that checks `self.config.resolution_threshold_for_jpg`. - * Set `save_params` based on the *newly determined* `output_format` (e.g., `cv2.IMWRITE_JPEG_QUALITY` for JPG, `cv2.IMWRITE_PNG_COMPRESSION` for PNG). - * Proceed with data type conversion (if needed based on target bit depth and format requirements like EXR needing float16) and saving using `cv2.imwrite` with the determined `output_path_temp` (using the new `output_ext`) and `save_params`. Ensure fallback logic (e.g., EXR -> PNG) still functions correctly if needed. - -**Phase 3: Implement New Logic for Merged Maps (`_merge_maps`)** - -1. **Modify `_merge_maps` in `asset_processor.py`:** - * Inside the loop for each `current_res_key`: - * When loading input maps (`loaded_inputs`), also retrieve and store their *original* file extensions (obtained during classification and now available via `self.classified_files["maps"]`, potentially needing a lookup based on `res_details['path']` or storing it earlier). - * **Determine Highest Input Format:** Iterate through the original extensions of the loaded inputs for this resolution. Use the hierarchy (EXR > TIF > PNG > JPG) to find the highest format present. - * **Determine Final Output Format:** - * Start with the `highest_input_format`. - * If `highest_input_format == 'tif'`: - * Check the target bit depth determined by the merge rule (`output_bit_depth`). - * If `target_bit_depth == 16`: Set final format based on `config.get_16bit_output_formats()` (EXR/PNG). - * If `target_bit_depth == 8`: Set final format to `png`. - * Otherwise (JPG, PNG, EXR), the final format is the `highest_input_format`. - * Set `output_ext` based on the `final_output_format`. - * Set `save_params` based on the `final_output_format`. - * Proceed with merging channels, converting the merged data to the target bit depth specified by the *merge rule*, and saving using `cv2.imwrite` with the determined `merged_output_path_temp` (using the new `output_ext`) and `save_params`. - -**Phase 4: Configuration and Documentation** - -1. **Modify `config.py`:** - * Comment out or remove the `RESOLUTION_THRESHOLD_FOR_JPG` variable as it's no longer used. Add a comment explaining why it was removed. -2. **Update `readme.md`:** - * Modify the "Features" section (around line 21) and the "Configuration" section (around lines 36-40, 86) to accurately describe the new output format logic: - * Explain that JPG inputs result in JPG outputs. - * Explain that TIF inputs result in PNG/EXR outputs based on target bit depth and config. - * Explain the merged map format determination based on input hierarchy (with the TIF->PNG/EXR adjustment). - * Mention the removal of the JPG resolution threshold. - -## Visual Plan (Mermaid) - -```mermaid -graph TD - A[Start] --> B(Phase 1: Enhance Classification); - B --> B1(Store original extension in classified_files['maps']); - B1 --> C(Phase 2: Modify _process_maps); - C --> C1(Get original extension); - C --> C2(Determine target bit depth); - C --> C3(Apply New Format Logic); - C3 -- force_8bit --> C3a[Format=PNG]; - C3 -- input=.jpg, 8bit --> C3b[Format=JPG]; - C3 -- input=.tif, 16bit --> C3c[Format=EXR/PNG (Config)]; - C3 -- input=.tif, 8bit --> C3d[Format=PNG]; - C3 -- input=other, 16bit --> C3c; - C3 -- input=other, 8bit --> C3e[Format=PNG (Config)]; - C3a --> C4(Remove JPG Threshold Check); - C3b --> C4; - C3c --> C4; - C3d --> C4; - C3e --> C4; - C4 --> C5(Set Save Params & Save); - C5 --> D(Phase 3: Modify _merge_maps); - D --> D1(Get original extensions of inputs); - D --> D2(Find highest format via hierarchy); - D2 --> D3(Adjust TIF -> PNG/EXR based on target bit depth); - D3 --> D4(Determine target bit depth from rule); - D4 --> D5(Set Save Params & Save Merged); - D5 --> E(Phase 4: Config & Docs); - E --> E1(Update config.py - Remove threshold); - E --> E2(Update readme.md); - E2 --> F[End]; \ No newline at end of file diff --git a/ProjectNotes/OUTPUT_FORMAT_PLAN_v2.md b/ProjectNotes/OUTPUT_FORMAT_PLAN_v2.md deleted file mode 100644 index 9fc10ae..0000000 --- a/ProjectNotes/OUTPUT_FORMAT_PLAN_v2.md +++ /dev/null @@ -1,127 +0,0 @@ -# Revised Plan: Implement Input-Based Output Format Logic with JPG Threshold Override - -This plan outlines the steps to modify the Asset Processor Tool to determine the output format of texture maps based on the input file format, specific rules, and a JPG resolution threshold override. - -## Requirements Summary (Revised) - -Based on user clarifications: - -1. **JPG Threshold Override:** If the target output bit depth is 8-bit AND the image resolution is greater than or equal to `RESOLUTION_THRESHOLD_FOR_JPG` (defined in `config.py`), the output format **must** be JPG. -2. **Input-Based Logic (if threshold not met):** - * **JPG Input -> JPG Output:** If the original source map is JPG and the target is 8-bit (and below threshold), output JPG. - * **TIF Input -> PNG/EXR Output:** If the original source map is TIF: - * If target is 16-bit, output EXR or PNG based on `OUTPUT_FORMAT_16BIT_PRIMARY` config. - * If target is 8-bit (and below threshold), output PNG. - * **Other Inputs (PNG, etc.) -> Configured Output:** For other input formats (and below threshold if 8-bit): - * If target is 16-bit, output EXR or PNG based on `OUTPUT_FORMAT_16BIT_PRIMARY` config. - * If target is 8-bit, output PNG (or format specified by `OUTPUT_FORMAT_8BIT`). -3. **`force_8bit` Rule:** If a map type has a `force_8bit` rule, the target bit depth is 8-bit. The output format will then be determined by the JPG threshold override or the input-based logic (resulting in JPG or PNG). -4. **Merged Maps:** - * Determine target `output_bit_depth` from the merge rule (`respect_inputs`, `force_8bit`, etc.). - * **Check JPG Threshold Override:** If target `output_bit_depth` is 8-bit AND resolution >= threshold, the final output format is JPG. - * **Else (Hierarchy Logic):** Determine the highest format among original inputs (EXR > TIF > PNG > JPG). - * If highest was TIF, adjust based on target bit depth (16-bit -> EXR/PNG config; 8-bit -> PNG). - * Otherwise, use the highest format found (EXR, PNG, JPG). - * **JPG 8-bit Check:** If the final format is JPG but the target bit depth was 16, force the merged data to 8-bit before saving. -5. **JPG Resizing:** Resized JPGs will be saved as JPG if the logic determines JPG as the output format. - -## Implementation Plan (Revised) - -**Phase 1: Data Gathering Enhancement** (Already Done & Correct) - -* `_inventory_and_classify_files` stores the original file extension in `self.classified_files["maps"]`. - -**Phase 2: Modify `_process_maps`** - -1. Retrieve the `original_extension` from `map_info`. -2. Determine the target `output_bit_depth` (8 or 16). -3. Get the `threshold = self.config.resolution_threshold_for_jpg`. -4. Get the `target_dim` for the current resolution loop iteration. -5. **New Format Logic (Revised):** - * Initialize `output_format`, `output_ext`, `save_params`, `needs_float16`. - * **Check JPG Threshold Override:** - * If `output_bit_depth == 8` AND `target_dim >= threshold`: Set format to JPG, set JPG params. - * **Else (Apply Input/Rule-Based Logic):** - * If `bit_depth_rule == 'force_8bit'`: Set format to PNG (8-bit), set PNG params. - * Else if `original_extension == '.jpg'` and `output_bit_depth == 8`: Set format to JPG, set JPG params. - * Else if `original_extension == '.tif'`: - * If `output_bit_depth == 16`: Set format to EXR/PNG (16-bit config), set params, set `needs_float16` if EXR. - * If `output_bit_depth == 8`: Set format to PNG, set PNG params. - * Else (other inputs like `.png`): - * If `output_bit_depth == 16`: Set format to EXR/PNG (16-bit config), set params, set `needs_float16` if EXR. - * If `output_bit_depth == 8`: Set format to PNG (8-bit config), set PNG params. -6. Proceed with data type conversion and saving. - -**Phase 3: Modify `_merge_maps`** - -1. Retrieve original extensions of inputs (`input_original_extensions`). -2. Determine target `output_bit_depth` from the merge rule. -3. Get the `threshold = self.config.resolution_threshold_for_jpg`. -4. Get the `target_dim` for the current resolution loop iteration. -5. **New Format Logic (Revised):** - * Initialize `final_output_format`, `output_ext`, `save_params`, `needs_float16`. - * **Check JPG Threshold Override:** - * If `output_bit_depth == 8` AND `target_dim >= threshold`: Set `final_output_format = 'jpg'`. - * **Else (Apply Hierarchy/Rule-Based Logic):** - * Determine `highest_input_format` (EXR > TIF > PNG > JPG). - * Start with `final_output_format = highest_input_format`. - * If `highest_input_format == 'tif'`: Adjust based on target bit depth (16->EXR/PNG config; 8->PNG). - * Set `output_format = final_output_format`. - * Set `output_ext`, `save_params`, `needs_float16` based on `output_format`. - * **JPG 8-bit Check:** If `output_format == 'jpg'` and `output_bit_depth == 16`, force final merged data to 8-bit before saving and update `output_bit_depth` variable. -6. Proceed with merging, data type conversion, and saving. - -**Phase 4: Configuration and Documentation** - -1. **Modify `config.py`:** Ensure `RESOLUTION_THRESHOLD_FOR_JPG` is uncommented and set correctly (revert previous change). -2. **Update `readme.md`:** Clarify the precedence: 8-bit maps >= threshold become JPG, otherwise the input-based logic applies. - -## Visual Plan (Mermaid - Revised) - -```mermaid -graph TD - A[Start] --> B(Phase 1: Enhance Classification - Done); - B --> C(Phase 2: Modify _process_maps); - C --> C1(Get original extension); - C --> C2(Determine target bit depth); - C --> C3(Get target_dim & threshold); - C --> C4{8bit AND >= threshold?}; - C4 -- Yes --> C4a[Format=JPG]; - C4 -- No --> C5(Apply Input/Rule Logic); - C5 -- force_8bit --> C5a[Format=PNG]; - C5 -- input=.jpg, 8bit --> C5b[Format=JPG]; - C5 -- input=.tif, 16bit --> C5c[Format=EXR/PNG (Config)]; - C5 -- input=.tif, 8bit --> C5d[Format=PNG]; - C5 -- input=other, 16bit --> C5c; - C5 -- input=other, 8bit --> C5e[Format=PNG (Config)]; - C4a --> C6(Set Save Params & Save); - C5a --> C6; - C5b --> C6; - C5c --> C6; - C5d --> C6; - C5e --> C6; - C6 --> D(Phase 3: Modify _merge_maps); - D --> D1(Get original extensions of inputs); - D --> D2(Determine target bit depth from rule); - D --> D3(Get target_dim & threshold); - D --> D4{8bit AND >= threshold?}; - D4 -- Yes --> D4a[FinalFormat=JPG]; - D4 -- No --> D5(Apply Hierarchy Logic); - D5 --> D5a(Find highest input format); - D5a --> D5b{Highest = TIF?}; - D5b -- Yes --> D5c{Target 16bit?}; - D5c -- Yes --> D5d[FinalFormat=EXR/PNG (Config)]; - D5c -- No --> D5e[FinalFormat=PNG]; - D5b -- No --> D5f[FinalFormat=HighestInput]; - D4a --> D6(Set Save Params); - D5d --> D6; - D5e --> D6; - D5f --> D6; - D6 --> D7{Format=JPG AND Target=16bit?}; - D7 -- Yes --> D7a(Force data to 8bit); - D7 -- No --> D8(Save Merged); - D7a --> D8; - D8 --> E(Phase 4: Config & Docs); - E --> E1(Uncomment threshold in config.py); - E --> E2(Update readme.md); - E2 --> F[End]; \ No newline at end of file diff --git a/ProjectNotes/issue_definitions_editor_list_selection.md b/ProjectNotes/issue_definitions_editor_list_selection.md new file mode 100644 index 0000000..73082d4 --- /dev/null +++ b/ProjectNotes/issue_definitions_editor_list_selection.md @@ -0,0 +1,62 @@ +# Issue: List item selection not working in Definitions Editor + +**Date:** 2025-05-13 + +**Affected File:** [`gui/definitions_editor_dialog.py`](gui/definitions_editor_dialog.py) + +**Problem Description:** +User mouse clicks on items within the `QListWidget` instances (for Asset Types, File Types, and Suppliers) in the Definitions Editor dialog do not trigger item selection or the `currentItemChanged` signal. The first item is selected by default and its details are displayed correctly. Programmatic selection of items (e.g., via a diagnostic button) *does* correctly trigger the `currentItemChanged` signal and updates the UI detail views. The issue is specific to user-initiated mouse clicks for selection after the initial load. + +**Debugging Steps Taken & Findings:** + +1. **Initial Analysis:** + * Reviewed GUI internals documentation ([`Documentation/02_Developer_Guide/06_GUI_Internals.md`](Documentation/02_Developer_Guide/06_GUI_Internals.md)) and [`gui/definitions_editor_dialog.py`](gui/definitions_editor_dialog.py) source code. + * Confirmed signal connections (`currentItemChanged` to display slots) are made. + +2. **Logging in Display Slots (`_display_*_details`):** + * Added logging to display slots. Confirmed they are called for the initial (default) item selection. + * No further calls to these slots occur on user clicks, indicating `currentItemChanged` is not firing. + +3. **Color Swatch Palette Role:** + * Investigated and corrected `QPalette.ColorRole` for color swatches (reverted from `Background` to `Window`). This fixed an `AttributeError` but did not resolve the selection issue. + +4. **Robust Error Handling in Display Slots:** + * Wrapped display slot logic in `try...finally` blocks with detailed logging. Confirmed slots complete without error for initial selection and signals for detail widgets are reconnected. + +5. **Diagnostic Lambda for `currentItemChanged`:** + * Added a lambda logger to `currentItemChanged` alongside the main display slot. + * Confirmed both lambda and display slot fire for initial programmatic selection. + * Neither fires for subsequent user clicks. This proved the `QListWidget` itself was not emitting the signal. + +6. **Explicit `setEnabled` and `setSelectionMode` on `QListWidget`:** + * Explicitly set these properties. No change in behavior. + +7. **Explicit `setEnabled` and `setFocusPolicy(Qt.ClickFocus)` on `tab_page` (parent of `QListWidget` layout):** + * This change **allowed programmatic selection via a diagnostic button to correctly fire `currentItemChanged` and update the UI**. + * However, user mouse clicks still did not work and did not fire the signal. + +8. **Event Filter Investigation:** + * **Filter on `QListWidget`:** Did NOT receive mouse press/release events from user clicks. + * **Filter on `tab_page` (parent of `QListWidget`'s layout):** Did NOT receive mouse press/release events. + * **Filter on `self.tab_widget` (QTabWidget):** DID receive mouse press/release events. + * Modified `self.tab_widget`'s event filter to return `False` for events over the current page, attempting to ensure propagation. + * **Result:** With the modified `tab_widget` filter, an event filter re-added to `asset_type_list_widget` *did* start receiving mouse press/release events. **However, `asset_type_list_widget` still did not emit `currentItemChanged` from these user clicks.** + +9. **`DebugListWidget` (Subclassing `QListWidget`):** + * Created `DebugListWidget` overriding `mousePressEvent` with logging. + * Used `DebugListWidget` for `asset_type_list_widget`. + * **Initial user report indicated that `DebugListWidget.mousePressEvent` logs were NOT appearing for user clicks.** This means that even with the `QTabWidget` event filter attempting to propagate events, and the `asset_type_list_widget`'s filter (from step 8) confirming it received them, the `mousePressEvent` of the `QListWidget` itself was not being triggered by those propagated events. This is the current mystery. + +**Current Status:** +- Programmatic selection works and fires signals. +- User clicks are received by an event filter on `asset_type_list_widget` (after `QTabWidget` filter modification) but do not result in `mousePressEvent` being called on the `QListWidget` (or `DebugListWidget`) itself, and thus no `currentItemChanged` signal is emitted. +- The issue seems to be a very low-level event processing problem specifically for user mouse clicks within the `QListWidget` instances when they are children of the `QTabWidget` pages, even when events appear to reach the list widget via an event filter. + +**Next Steps (When Resuming):** +1. Re-verify the logs from the `DebugListWidget.mousePressEvent` test. If it's truly not being called despite its event filter seeing events, this is extremely unusual. +2. Simplify the `_create_tab_pane` method drastically for one tab: + * Remove the right-hand pane. + * Add the `DebugListWidget` directly to the `tab_page`'s layout without the intermediate `left_pane_layout`. +3. Consider if any styles applied to `QListWidget` or its parents via stylesheets could be interfering with hit testing or event processing (unlikely for this specific symptom, but possible). +4. Explore alternative ways to populate/manage the `QListWidget` or its items if a subtle corruption is occurring. +5. If all else fails, consider replacing the `QListWidget` with a `QListView` and a `QStringListModel` as a more fundamental change to see if the issue is specific to `QListWidget` in this context. \ No newline at end of file diff --git a/PythonCheatsheats/args-and-kwargs.md b/PythonCheatsheats/args-and-kwargs.md deleted file mode 100644 index a94a3e6..0000000 --- a/PythonCheatsheats/args-and-kwargs.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -title: Python Args and Kwargs - Python Cheatsheet -description: args and kwargs may seem scary, but the truth is that they are not that difficult to grasp and have the power to grant your functions with flexibility and readability ---- - - -Python Args and Kwargs - - - - - Python args and kwargs Made Easy - - - *args and **kwargs may seem scary, but the truth is that they are not that difficult to grasp and have the power to grant your functions with lots of flexibility. - - - -Read the article Python \*args and \*\*kwargs Made Easy for a more in deep introduction. - -## Args and Kwargs - -`*args` and `**kwargs` allow you to pass an undefined number of arguments and keywords when calling a function. - -```python ->>> def some_function(*args, **kwargs): -... pass -... ->>> # call some_function with any number of arguments ->>> some_function(arg1, arg2, arg3) - ->>> # call some_function with any number of keywords ->>> some_function(key1=arg1, key2=arg2, key3=arg3) - ->>> # call both, arguments and keywords ->>> some_function(arg, key1=arg1) - ->>> # or none ->>> some_function() -``` - - - - Python conventions - - - The words *args and **kwargs are conventions. They are not imposed by the interpreter, but considered good practice by the Python community. - - - -## args - -You can access the _arguments_ through the `args` variable: - -```python ->>> def some_function(*args): -... print(f'Arguments passed: {args} as {type(args)}') -... ->>> some_function('arg1', 'arg2', 'arg3') -# Arguments passed: ('arg1', 'arg2', 'arg3') as -``` - -## kwargs - -Keywords are accessed through the `kwargs` variable: - -```python ->>> def some_function(**kwargs): -... print(f'keywords: {kwargs} as {type(kwargs)}') -... ->>> some_function(key1='arg1', key2='arg2') -# keywords: {'key1': 'arg1', 'key2': 'arg2'} as -``` diff --git a/PythonCheatsheats/basics.md b/PythonCheatsheats/basics.md deleted file mode 100644 index ed27463..0000000 --- a/PythonCheatsheats/basics.md +++ /dev/null @@ -1,340 +0,0 @@ ---- -title: Python Basics - Python Cheatsheet -description: The basics of python. We all need to start somewhere, so how about doing it here. ---- - - -Python Basics - - -We all need to start somewhere, so how about doing it here. - - - - From the Python 3 tutorial - - - Python is an easy to learn, powerful programming language [...] Python’s elegant syntax and dynamic typing, together with its interpreted nature, make it an ideal language for scripting and rapid application development. - - - -## Math Operators - -From **highest** to **lowest** precedence: - -| Operators | Operation | Example | -| --------- | ----------------- | --------------- | -| \*\* | Exponent | `2 ** 3 = 8` | -| % | Modulus/Remainder | `22 % 8 = 6` | -| // | Integer division | `22 // 8 = 2` | -| / | Division | `22 / 8 = 2.75` | -| \* | Multiplication | `3 * 3 = 9` | -| - | Subtraction | `5 - 2 = 3` | -| + | Addition | `2 + 2 = 4` | - -Examples of expressions: - -```python ->>> 2 + 3 * 6 -# 20 - ->>> (2 + 3) * 6 -# 30 - ->>> 2 ** 8 -#256 - ->>> 23 // 7 -# 3 - ->>> 23 % 7 -# 2 - ->>> (5 - 1) * ((7 + 1) / (3 - 1)) -# 16.0 -``` - -## Augmented Assignment Operators - -| Operator | Equivalent | -| ----------- | ---------------- | -| `var += 1` | `var = var + 1` | -| `var -= 1` | `var = var - 1` | -| `var *= 1` | `var = var * 1` | -| `var /= 1` | `var = var / 1` | -| `var //= 1` | `var = var // 1` | -| `var %= 1` | `var = var % 1` | -| `var **= 1` | `var = var ** 1` | - -Examples: - -```python ->>> greeting = 'Hello' ->>> greeting += ' world!' ->>> greeting -# 'Hello world!' - ->>> number = 1 ->>> number += 1 ->>> number -# 2 - ->>> my_list = ['item'] ->>> my_list *= 3 ->>> my_list -# ['item', 'item', 'item'] -``` - -## Walrus Operator - -The Walrus Operator allows assignment of variables within an expression while returning the value of the variable - -Example: - -```python ->>> print(my_var:="Hello World!") -# 'Hello world!' - ->>> my_var="Yes" ->>> print(my_var) -# 'Yes' - ->>> print(my_var:="Hello") -# 'Hello' -``` - -The _Walrus Operator_, or **Assignment Expression Operator** was firstly introduced in 2018 via [PEP 572](https://peps.python.org/pep-0572/), and then officially released with **Python 3.8** in October 2019. - - - - Syntax Semantics & Examples - - - The PEP 572 provides the syntax, semantics and examples for the Walrus Operator. - - - -## Data Types - -| Data Type | Examples | -| ---------------------- | ----------------------------------------- | -| Integers | `-2, -1, 0, 1, 2, 3, 4, 5` | -| Floating-point numbers | `-1.25, -1.0, --0.5, 0.0, 0.5, 1.0, 1.25` | -| Strings | `'a', 'aa', 'aaa', 'Hello!', '11 cats'` | - -## Concatenation and Replication - -String concatenation: - -```python ->>> 'Alice' 'Bob' -# 'AliceBob' -``` - -String replication: - -```python ->>> 'Alice' * 5 -# 'AliceAliceAliceAliceAlice' -``` - -## Variables - -You can name a variable anything as long as it obeys the following rules: - -1. It can be only one word. - -```python ->>> # bad ->>> my variable = 'Hello' - ->>> # good ->>> var = 'Hello' -``` - -2. It can use only letters, numbers, and the underscore (`_`) character. - -```python ->>> # bad ->>> %$@variable = 'Hello' - ->>> # good ->>> my_var = 'Hello' - ->>> # good ->>> my_var_2 = 'Hello' -``` - -3. It can’t begin with a number. - -```python ->>> # this wont work ->>> 23_var = 'hello' -``` - -4. Variable name starting with an underscore (`_`) are considered as "unuseful". - -```python ->>> # _spam should not be used again in the code ->>> _spam = 'Hello' -``` - -## Comments - -Inline comment: - -```python -# This is a comment -``` - -Multiline comment: - -```python -# This is a -# multiline comment -``` - -Code with a comment: - -```python -a = 1 # initialization -``` - -Please note the two spaces in front of the comment. - -Function docstring: - -```python -def foo(): - """ - This is a function docstring - You can also use: - ''' Function Docstring ''' - """ -``` - -## The print() Function - -The `print()` function writes the value of the argument(s) it is given. [...] it handles multiple arguments, floating point-quantities, and strings. Strings are printed without quotes, and a space is inserted between items, so you can format things nicely: - -```python ->>> print('Hello world!') -# Hello world! - ->>> a = 1 ->>> print('Hello world!', a) -# Hello world! 1 -``` - -### The end keyword - -The keyword argument `end` can be used to avoid the newline after the output, or end the output with a different string: - -```python -phrase = ['printed', 'with', 'a', 'dash', 'in', 'between'] ->>> for word in phrase: -... print(word, end='-') -... -# printed-with-a-dash-in-between- -``` - -### The sep keyword - -The keyword `sep` specify how to separate the objects, if there is more than one: - -```python -print('cats', 'dogs', 'mice', sep=',') -# cats,dogs,mice -``` - -## The input() Function - -This function takes the input from the user and converts it into a string: - -```python ->>> print('What is your name?') # ask for their name ->>> my_name = input() ->>> print('Hi, {}'.format(my_name)) -# What is your name? -# Martha -# Hi, Martha -``` - -`input()` can also set a default message without using `print()`: - -```python ->>> my_name = input('What is your name? ') # default message ->>> print('Hi, {}'.format(my_name)) -# What is your name? Martha -# Hi, Martha -``` - -It is also possible to use formatted strings to avoid using .format: - -```python ->>> my_name = input('What is your name? ') # default message ->>> print(f'Hi, {my_name}') -# What is your name? Martha -# Hi, Martha -``` - - -## The len() Function - -Evaluates to the integer value of the number of characters in a string, list, dictionary, etc.: - -```python ->>> len('hello') -# 5 - ->>> len(['cat', 3, 'dog']) -# 3 -``` - - - Test of emptiness - - Test of emptiness of strings, lists, dictionaries, etc., should not use - len, but prefer direct boolean evaluation. - - - -Test of emptiness example: - -```python ->>> a = [1, 2, 3] - -# bad ->>> if len(a) > 0: # evaluates to True -... print("the list is not empty!") -... -# the list is not empty! - -# good ->>> if a: # evaluates to True -... print("the list is not empty!") -... -# the list is not empty! -``` - -## The str(), int(), and float() Functions - -These functions allow you to change the type of variable. For example, you can transform from an `integer` or `float` to a `string`: - -```python ->>> str(29) -# '29' - ->>> str(-3.14) -# '-3.14' -``` - -Or from a `string` to an `integer` or `float`: - -```python ->>> int('11') -# 11 - ->>> float('3.14') -# 3.14 -``` diff --git a/PythonCheatsheats/built-in-functions.md b/PythonCheatsheats/built-in-functions.md deleted file mode 100644 index c4ceda1..0000000 --- a/PythonCheatsheats/built-in-functions.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Python built-in functions - Python Cheatsheet -description: The Python interpreter has a number of functions and types built into it that are always available. ---- - - -Python Built-in Functions - - -The Python interpreter has a number of functions and types built into it that are always available. - -## Python built-in Functions - -| Function | Description | -| -------------------------------------------------------------------- | ------------------------------------------------------------------------- | -| abs() | Return the absolute value of a number. | -| aiter() | Return an asynchronous iterator for an asynchronous iterable. | -| all() | Return True if all elements of the iterable are true. | -| any() | Return True if any element of the iterable is true. | -| ascii() | Return a string with a printable representation of an object. | -| bin() | Convert an integer number to a binary string. | -| bool() | Return a Boolean value. | -| breakpoint() | Drops you into the debugger at the call site. | -| bytearray() | Return a new array of bytes. | -| bytes() | Return a new “bytes” object. | -| callable() | Return True if the object argument is callable, False if not. | -| chr() | Return the string representing a character. | -| classmethod() | Transform a method into a class method. | -| compile() | Compile the source into a code or AST object. | -| complex() | Return a complex number with the value real + imag\*1j. | -| delattr() | Deletes the named attribute, provided the object allows it. | -| dict() | Create a new dictionary. | -| dir() | Return the list of names in the current local scope. | -| divmod() | Return a pair of numbers consisting of their quotient and remainder. | -| enumerate() | Return an enumerate object. | -| eval() | Evaluates and executes an expression. | -| exec() | This function supports dynamic execution of Python code. | -| filter() | Construct an iterator from an iterable and returns true. | -| float() | Return a floating point number from a number or string. | -| format() | Convert a value to a “formatted” representation. | -| frozenset() | Return a new frozenset object. | -| getattr() | Return the value of the named attribute of object. | -| globals() | Return the dictionary implementing the current module namespace. | -| hasattr() | True if the string is the name of one of the object’s attributes. | -| hash() | Return the hash value of the object. | -| help() | Invoke the built-in help system. | -| hex() | Convert an integer number to a lowercase hexadecimal string. | -| id() | Return the “identity” of an object. | -| input() | This function takes an input and converts it into a string. | -| int() | Return an integer object constructed from a number or string. | -| isinstance() | Return True if the object argument is an instance of an object. | -| issubclass() | Return True if class is a subclass of classinfo. | -| iter() | Return an iterator object. | -| len() | Return the length (the number of items) of an object. | -| list() | Rather than being a function, list is a mutable sequence type. | -| locals() | Update and return a dictionary with the current local symbol table. | -| map() | Return an iterator that applies function to every item of iterable. | -| max() | Return the largest item in an iterable. | -| min() | Return the smallest item in an iterable. | -| next() | Retrieve the next item from the iterator. | -| object() | Return a new featureless object. | -| oct() | Convert an integer number to an octal string. | -| open() | Open file and return a corresponding file object. | -| ord() | Return an integer representing the Unicode code point of a character. | -| pow() | Return base to the power exp. | -| print() | Print objects to the text stream file. | -| property() | Return a property attribute. | -| repr() | Return a string containing a printable representation of an object. | -| reversed() | Return a reverse iterator. | -| round() | Return number rounded to ndigits precision after the decimal point. | -| set() | Return a new set object. | -| setattr() | This is the counterpart of getattr(). | -| slice() | Return a sliced object representing a set of indices. | -| sorted() | Return a new sorted list from the items in iterable. | -| staticmethod() | Transform a method into a static method. | -| str() | Return a str version of object. | -| sum() | Sums start and the items of an iterable. | -| super() | Return a proxy object that delegates method calls to a parent or sibling. | -| tuple() | Rather than being a function, is actually an immutable sequence type. | -| type() | Return the type of an object. | -| vars() | Return the dict attribute for any other object with a dict attribute. | -| zip() | Iterate over several iterables in parallel. | -| **import**() | This function is invoked by the import statement. | diff --git a/PythonCheatsheats/comprehensions.md b/PythonCheatsheats/comprehensions.md deleted file mode 100644 index e8dead8..0000000 --- a/PythonCheatsheats/comprehensions.md +++ /dev/null @@ -1,120 +0,0 @@ ---- -title: Python Comprehensions - Python Cheatsheet -description: List comprehensions provide a concise way to create lists ---- - - -Python Comprehensions - - -List Comprehensions are a special kind of syntax that let us create lists out of other lists, and are incredibly useful when dealing with numbers and with one or two levels of nested for loops. - - - - From the Python 3 tutorial - - - List comprehensions provide a concise way to create lists. [...] or to create a subsequence of those elements that satisfy a certain condition. - - - -Read Python Comprehensions: A step by step Introduction for a more in-depth introduction. - -## List comprehension - -This is how we create a new list from an existing collection with a For Loop: - -```python ->>> names = ['Charles', 'Susan', 'Patrick', 'George'] - ->>> new_list = [] ->>> for n in names: -... new_list.append(n) -... ->>> new_list -# ['Charles', 'Susan', 'Patrick', 'George'] -``` - -And this is how we do the same with a List Comprehension: - -```python ->>> names = ['Charles', 'Susan', 'Patrick', 'George'] - ->>> new_list = [n for n in names] ->>> new_list -# ['Charles', 'Susan', 'Patrick', 'George'] -``` - -We can do the same with numbers: - -```python ->>> n = [(a, b) for a in range(1, 3) for b in range(1, 3)] ->>> n -# [(1, 1), (1, 2), (2, 1), (2, 2)] -``` - -## Adding conditionals - -If we want `new_list` to have only the names that start with C, with a for loop, we would do it like this: - -```python ->>> names = ['Charles', 'Susan', 'Patrick', 'George', 'Carol'] - ->>> new_list = [] ->>> for n in names: -... if n.startswith('C'): -... new_list.append(n) -... ->>> print(new_list) -# ['Charles', 'Carol'] -``` - -In a List Comprehension, we add the `if` statement at the end: - -```python ->>> new_list = [n for n in names if n.startswith('C')] ->>> print(new_list) -# ['Charles', 'Carol'] -``` - -To use an `if-else` statement in a List Comprehension: - -```python ->>> nums = [1, 2, 3, 4, 5, 6] ->>> new_list = [num*2 if num % 2 == 0 else num for num in nums] ->>> print(new_list) -# [1, 4, 3, 8, 5, 12] -``` - - - - Set and Dict comprehensions - - - The basics of `list` comprehensions also apply to sets and dictionaries. - - - -## Set comprehension - -```python ->>> b = {"abc", "def"} ->>> {s.upper() for s in b} -{"ABC", "DEF"} -``` - -## Dict comprehension - -```python ->>> c = {'name': 'Pooka', 'age': 5} ->>> {v: k for k, v in c.items()} -{'Pooka': 'name', 5: 'age'} -``` - -A List comprehension can be generated from a dictionary: - -```python ->>> c = {'name': 'Pooka', 'age': 5} ->>> ["{}:{}".format(k.upper(), v) for k, v in c.items()] -['NAME:Pooka', 'AGE:5'] -``` diff --git a/PythonCheatsheats/context-manager.md b/PythonCheatsheats/context-manager.md deleted file mode 100644 index 475d0f4..0000000 --- a/PythonCheatsheats/context-manager.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: Python Context Manager - Python Cheatsheet -description: While Python's context managers are widely used, few understand the purpose behind their use. These statements, commonly used with reading and writing files, assist the application in conserving system memory and improve resource management by ensuring specific resources are only in use for certain processes. ---- - - -Python Context Manager - - -While Python's context managers are widely used, few understand the purpose behind their use. These statements, commonly used with reading and writing files, assist the application in conserving system memory and improve resource management by ensuring specific resources are only in use for certain processes. - -## The with statement - -A context manager is an object that is notified when a context (a block of code) starts and ends. You commonly use one with the `with` statement. It takes care of the notifying. - -For example, file objects are context managers. When a context ends, the file object is closed automatically: - -```python ->>> with open(filename) as f: -... file_contents = f.read() -... ->>> # the open_file object has automatically been closed. -``` - -Anything that ends execution of the block causes the context manager's exit method to be called. This includes exceptions, and can be useful when an error causes you to prematurely exit an open file or connection. Exiting a script without properly closing files/connections is a bad idea, that may cause data loss or other problems. By using a context manager, you can ensure that precautions are always taken to prevent damage or loss in this way. - -## Writing your own context manager - -It is also possible to write a context manager using generator syntax thanks to the `contextlib.contextmanager` decorator: - -```python ->>> import contextlib ->>> @contextlib.contextmanager -... def context_manager(num): -... print('Enter') -... yield num + 1 -... print('Exit') -... ->>> with context_manager(2) as cm: -... # the following instructions are run when -... # the 'yield' point of the context manager is -... # reached. 'cm' will have the value that was yielded -... print('Right in the middle with cm = {}'.format(cm)) -... -# Enter -# Right in the middle with cm = 3 -# Exit -``` - - -## Class based context manager - -You can define class based context manager. The key methods are `__enter__` and `__exit__` -```python -class ContextManager: - def __enter__(self, *args, **kwargs): - print("--enter--") - - def __exit__(self, *args): - print("--exit--") - - -with ContextManager(): - print("test") -#--enter-- -#test -#--exit-- -``` diff --git a/PythonCheatsheats/control-flow.md b/PythonCheatsheats/control-flow.md deleted file mode 100644 index 13695eb..0000000 --- a/PythonCheatsheats/control-flow.md +++ /dev/null @@ -1,490 +0,0 @@ ---- -title: Python Control Flow - Python Cheatsheet -description: Control flow is the order in which individual statements, instructions or function calls are executed or evaluated. The control flow of a Python program is regulated by conditional statements, loops, and function calls. ---- - - -Python Control Flow - - - - - Python control flow - - - Control flow is the order in which individual statements, instructions, or function calls are executed or evaluated. The control flow of a Python program is regulated by conditional statements, loops, and function calls. - - - -## Comparison Operators - -| Operator | Meaning | -| -------- | ------------------------ | -| `==` | Equal to | -| `!=` | Not equal to | -| `<` | Less than | -| `>` | Greater Than | -| `<=` | Less than or Equal to | -| `>=` | Greater than or Equal to | - -These operators evaluate to True or False depending on the values you give them. - -Examples: - -```python ->>> 42 == 42 -True - ->>> 40 == 42 -False - ->>> 'hello' == 'hello' -True - ->>> 'hello' == 'Hello' -False - ->>> 'dog' != 'cat' -True - ->>> 42 == 42.0 -True - ->>> 42 == '42' -False -``` - -## Boolean Operators - -There are three Boolean operators: `and`, `or`, and `not`. -In the order of precedence, highest to lowest they are `not`, `and` and `or`. - -The `and` Operator’s _Truth_ Table: - -| Expression | Evaluates to | -| ----------------- | ------------ | -| `True and True` | `True` | -| `True and False` | `False` | -| `False and True` | `False` | -| `False and False` | `False` | - -The `or` Operator’s _Truth_ Table: - -| Expression | Evaluates to | -| ---------------- | ------------ | -| `True or True` | `True` | -| `True or False` | `True` | -| `False or True` | `True` | -| `False or False` | `False` | - -The `not` Operator’s _Truth_ Table: - -| Expression | Evaluates to | -| ----------- | ------------ | -| `not True` | `False` | -| `not False` | `True` | - -## Mixing Operators - -You can mix boolean and comparison operators: - -```python ->>> (4 < 5) and (5 < 6) -True - ->>> (4 < 5) and (9 < 6) -False - ->>> (1 == 2) or (2 == 2) -True -``` - -Also, you can mix use multiple Boolean operators in an expression, along with the comparison operators: - -```python ->>> 2 + 2 == 4 and not 2 + 2 == 5 and 2 * 2 == 2 + 2 -True ->>> # In the statement below 3 < 4 and 5 > 5 gets executed first evaluating to False ->>> # Then 5 > 4 returns True so the results after True or False is True ->>> 5 > 4 or 3 < 4 and 5 > 5 -True ->>> # Now the statement within parentheses gets executed first so True and False returns False. ->>> (5 > 4 or 3 < 4) and 5 > 5 -False -``` - -## if Statements - -The `if` statement evaluates an expression, and if that expression is `True`, it then executes the following indented code: - -```python ->>> name = 'Debora' - ->>> if name == 'Debora': -... print('Hi, Debora') -... -# Hi, Debora - ->>> if name != 'George': -... print('You are not George') -... -# You are not George -``` - -The `else` statement executes only if the evaluation of the `if` and all the `elif` expressions are `False`: - -```python ->>> name = 'Debora' - ->>> if name == 'George': -... print('Hi, George.') -... else: -... print('You are not George') -... -# You are not George -``` - -Only after the `if` statement expression is `False`, the `elif` statement is evaluated and executed: - -```python ->>> name = 'George' - ->>> if name == 'Debora': -... print('Hi Debora!') -... elif name == 'George': -... print('Hi George!') -... -# Hi George! -``` - -the `elif` and `else` parts are optional. - -```python ->>> name = 'Antony' - ->>> if name == 'Debora': -... print('Hi Debora!') -... elif name == 'George': -... print('Hi George!') -... else: -... print('Who are you?') -... -# Who are you? -``` - -## Ternary Conditional Operator - -Many programming languages have a ternary operator, which define a conditional expression. The most common usage is to make a terse, simple conditional assignment statement. In other words, it offers one-line code to evaluate the first expression if the condition is true, and otherwise it evaluates the second expression. - -``` - if else -``` - -Example: - -```python ->>> age = 15 - ->>> # this if statement: ->>> if age < 18: -... print('kid') -... else: -... print('adult') -... -# output: kid - ->>> # is equivalent to this ternary operator: ->>> print('kid' if age < 18 else 'adult') -# output: kid -``` - -Ternary operators can be chained: - -```python ->>> age = 15 - ->>> # this ternary operator: ->>> print('kid' if age < 13 else 'teen' if age < 18 else 'adult') - ->>> # is equivalent to this if statement: ->>> if age < 18: -... if age < 13: -... print('kid') -... else: -... print('teen') -... else: -... print('adult') -... -# output: teen -``` - -## Switch-Case Statement - - - - Switch-Case statements - - - In computer programming languages, a switch statement is a type of selection control mechanism used to allow the value of a variable or expression to change the control flow of program execution via search and map. - - - -The _Switch-Case statements_, or **Structural Pattern Matching**, was firstly introduced in 2020 via [PEP 622](https://peps.python.org/pep-0622/), and then officially released with **Python 3.10** in September 2022. - - - - Official Tutorial - - - The PEP 636 provides an official tutorial for the Python Pattern matching or Switch-Case statements. - - - -### Matching single values - -```python ->>> response_code = 201 ->>> match response_code: -... case 200: -... print("OK") -... case 201: -... print("Created") -... case 300: -... print("Multiple Choices") -... case 307: -... print("Temporary Redirect") -... case 404: -... print("404 Not Found") -... case 500: -... print("Internal Server Error") -... case 502: -... print("502 Bad Gateway") -... -# Created -``` - -### Matching with the or Pattern - -In this example, the pipe character (`|` or `or`) allows python to return the same response for two or more cases. - -```python ->>> response_code = 502 ->>> match response_code: -... case 200 | 201: -... print("OK") -... case 300 | 307: -... print("Redirect") -... case 400 | 401: -... print("Bad Request") -... case 500 | 502: -... print("Internal Server Error") -... -# Internal Server Error -``` - -### Matching by the length of an Iterable - -```python ->>> today_responses = [200, 300, 404, 500] ->>> match today_responses: -... case [a]: -... print(f"One response today: {a}") -... case [a, b]: -... print(f"Two responses today: {a} and {b}") -... case [a, b, *rest]: -... print(f"All responses: {a}, {b}, {rest}") -... -# All responses: 200, 300, [404, 500] -``` - -### Default value - -The underscore symbol (`_`) is used to define a default case: - -```python ->>> response_code = 800 ->>> match response_code: -... case 200 | 201: -... print("OK") -... case 300 | 307: -... print("Redirect") -... case 400 | 401: -... print("Bad Request") -... case 500 | 502: -... print("Internal Server Error") -... case _: -... print("Invalid Code") -... -# Invalid Code -``` - -### Matching Builtin Classes - -```python ->>> response_code = "300" ->>> match response_code: -... case int(): -... print('Code is a number') -... case str(): -... print('Code is a string') -... case _: -... print('Code is neither a string nor a number') -... -# Code is a string -``` - -### Guarding Match-Case Statements - -```python ->>> response_code = 300 ->>> match response_code: -... case int(): -... if response_code > 99 and response_code < 500: -... print('Code is a valid number') -... case _: -... print('Code is an invalid number') -... -# Code is a valid number -``` - -## while Loop Statements - -The while statement is used for repeated execution as long as an expression is `True`: - -```python ->>> spam = 0 ->>> while spam < 5: -... print('Hello, world.') -... spam = spam + 1 -... -# Hello, world. -# Hello, world. -# Hello, world. -# Hello, world. -# Hello, world. -``` - -## break Statements - -If the execution reaches a `break` statement, it immediately exits the `while` loop’s clause: - -```python ->>> while True: -... name = input('Please type your name: ') -... if name == 'your name': -... break -... ->>> print('Thank you!') -# Please type your name: your name -# Thank you! -``` - -## continue Statements - -When the program execution reaches a `continue` statement, the program execution immediately jumps back to the start of the loop. - -```python ->>> while True: -... name = input('Who are you? ') -... if name != 'Joe': -... continue -... password = input('Password? (It is a fish.): ') -... if password == 'swordfish': -... break -... ->>> print('Access granted.') -# Who are you? Charles -# Who are you? Debora -# Who are you? Joe -# Password? (It is a fish.): swordfish -# Access granted. -``` - -## For loop - -The `for` loop iterates over a `list`, `tuple`, `dictionary`, `set` or `string`: - -```python ->>> pets = ['Bella', 'Milo', 'Loki'] ->>> for pet in pets: -... print(pet) -... -# Bella -# Milo -# Loki -``` - -## The range() function - -The `range()` function returns a sequence of numbers. It starts from 0, increments by 1, and stops before a specified number: - -```python ->>> for i in range(5): -... print(f'Will stop at 5! or 4? ({i})') -... -# Will stop at 5! or 4? (0) -# Will stop at 5! or 4? (1) -# Will stop at 5! or 4? (2) -# Will stop at 5! or 4? (3) -# Will stop at 5! or 4? (4) -``` - -The `range()` function can also modify its 3 defaults arguments. The first two will be the `start` and `stop` values, and the third will be the `step` argument. The step is the amount that the variable is increased by after each iteration. - -```python -# range(start, stop, step) ->>> for i in range(0, 10, 2): -... print(i) -... -# 0 -# 2 -# 4 -# 6 -# 8 -``` - -You can even use a negative number for the step argument to make the for loop count down instead of up. - -```python ->>> for i in range(5, -1, -1): -... print(i) -... -# 5 -# 4 -# 3 -# 2 -# 1 -# 0 -``` - -## For else statement - -This allows to specify a statement to execute in case of the full loop has been executed. Only -useful when a `break` condition can occur in the loop: - -```python ->>> for i in [1, 2, 3, 4, 5]: -... if i == 3: -... break -... else: -... print("only executed when no item is equal to 3") -``` - -## Ending a Program with sys.exit() - -`exit()` function allows exiting Python. - -```python ->>> import sys - ->>> while True: -... feedback = input('Type exit to exit: ') -... if feedback == 'exit': -... print(f'You typed {feedback}.') -... sys.exit() -... -# Type exit to exit: open -# Type exit to exit: close -# Type exit to exit: exit -# You typed exit -``` diff --git a/PythonCheatsheats/dataclasses.md b/PythonCheatsheats/dataclasses.md deleted file mode 100644 index a41b3c4..0000000 --- a/PythonCheatsheats/dataclasses.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: Python Dataclasses - Python Cheatsheet -description: Dataclasses are python classes, but are suited for storing data objects. This module provides a decorator and functions for automatically adding generated special methods such as __init__() and __repr__() to user-defined classes. ---- - - -Python Dataclasses - - -`Dataclasses` are python classes, but are suited for storing data objects. -This module provides a decorator and functions for automatically adding generated special methods such as `__init__()` and `__repr__()` to user-defined classes. - -## Features - -1. They store data and represent a certain data type. Ex: A number. For people familiar with ORMs, a model instance is a data object. It represents a specific kind of entity. It holds attributes that define or represent the entity. - -2. They can be compared to other objects of the same type. Ex: A number can be greater than, less than, or equal to another number. - -Python 3.7 provides a decorator dataclass that is used to convert a class into a dataclass. - -```python ->>> class Number: -... def __init__(self, val): -... self.val = val -... ->>> obj = Number(2) ->>> obj.val -# 2 -``` - -with dataclass - -```python ->>> @dataclass -... class Number: -... val: int -... ->>> obj = Number(2) ->>> obj.val -# 2 -``` - -## Default values - -It is easy to add default values to the fields of your data class. - -```python ->>> @dataclass -... class Product: -... name: str -... count: int = 0 -... price: float = 0.0 -... ->>> obj = Product("Python") ->>> obj.name -# Python - ->>> obj.count -# 0 - ->>> obj.price -# 0.0 -``` - -## Type hints - -It is mandatory to define the data type in dataclass. However, If you would rather not specify the datatype then, use `typing.Any`. - -```python ->>> from dataclasses import dataclass ->>> from typing import Any - ->>> @dataclass -... class WithoutExplicitTypes: -... name: Any -... value: Any = 42 -``` diff --git a/PythonCheatsheats/debugging.md b/PythonCheatsheats/debugging.md deleted file mode 100644 index b4edf43..0000000 --- a/PythonCheatsheats/debugging.md +++ /dev/null @@ -1,197 +0,0 @@ ---- -title: Python Debugging - Python Cheatsheet -description: In computer programming and software development, debugging is the process of finding and resolving bugs (defects or problems that prevent correct operation) within computer programs, software, or systems. ---- - - -Python Debugging - - - - - Finding and resolving bugs - - - In computer programming and software development, debugging is the process of finding and resolving bugs (defects or problems that prevent correct operation) within computer programs, software, or systems. - - - -## Raising Exceptions - -Exceptions are raised with a raise statement. In code, a raise statement consists of the following: - -- The `raise` keyword -- A call to the `Exception()` function -- A string with a helpful error message passed to the `Exception()` function - -```python ->>> raise Exception('This is the error message.') -# Traceback (most recent call last): -# File "", line 1, in -# raise Exception('This is the error message.') -# Exception: This is the error message. -``` - -Typically, it’s the code that calls the function, not the function itself, that knows how to handle an exception. So, you will commonly see a raise statement inside a function and the `try` and `except` statements in the code calling the function. - -```python ->>> def box_print(symbol, width, height): -... if len(symbol) != 1: -... raise Exception('Symbol must be a single character string.') -... if width <= 2: -... raise Exception('Width must be greater than 2.') -... if height <= 2: -... raise Exception('Height must be greater than 2.') -... print(symbol * width) -... for i in range(height - 2): -... print(symbol + (' ' * (width - 2)) + symbol) -... print(symbol * width) -... ->>> for sym, w, h in (('*', 4, 4), ('O', 20, 5), ('x', 1, 3), ('ZZ', 3, 3)): -... try: -... box_print(sym, w, h) -... except Exception as err: -... print('An exception happened: ' + str(err)) -... -# **** -# * * -# * * -# **** -# OOOOOOOOOOOOOOOOOOOO -# O O -# O O -# O O -# OOOOOOOOOOOOOOOOOOOO -# An exception happened: Width must be greater than 2. -# An exception happened: Symbol must be a single character string. -``` - -Read more about [Exception Handling](/cheatsheet/exception-handling). - -## Getting the Traceback as a string - -The `traceback` is displayed by Python whenever a raised exception goes unhandled. But can also obtain it as a string by calling traceback.format_exc(). This function is useful if you want the information from an exception’s traceback but also want an except statement to gracefully handle the exception. You will need to import Python’s traceback module before calling this function. - -```python ->>> import traceback - ->>> try: -... raise Exception('This is the error message.') ->>> except: -... with open('errorInfo.txt', 'w') as error_file: -... error_file.write(traceback.format_exc()) -... print('The traceback info was written to errorInfo.txt.') -... -# 116 -# The traceback info was written to errorInfo.txt. -``` - -The 116 is the return value from the `write()` method, since 116 characters were written to the file. The `traceback` text was written to errorInfo.txt. - - Traceback (most recent call last): - File "", line 2, in - Exception: This is the error message. - -## Assertions - -An assertion is a sanity check to make sure your code isn’t doing something obviously wrong. These sanity checks are performed by `assert` statements. If the sanity check fails, then an `AssertionError` exception is raised. In code, an `assert` statement consists of the following: - -- The `assert` keyword -- A condition (that is, an expression that evaluates to `True` or `False`) -- A comma -- A `string` to display when the condition is `False` - -```python ->>> pod_bay_door_status = 'open' ->>> assert pod_bay_door_status == 'open', 'The pod bay doors need to be "open".' - ->>> pod_bay_door_status = 'I\'m sorry, Dave. I\'m afraid I can\'t do that.' ->>> assert pod_bay_door_status == 'open', 'The pod bay doors need to be "open".' -# Traceback (most recent call last): -# File "", line 1, in -# assert pod_bay_door_status == 'open', 'The pod bay doors need to be "open".' -# AssertionError: The pod bay doors need to be "open". -``` - -In plain English, an assert statement says, “I assert that this condition holds true, and if not, there is a bug somewhere in the program.” Unlike exceptions, your code should not handle assert statements with try and except; if an assert fails, your program should crash. By failing fast like this, you shorten the time between the original cause of the bug and when you first notice the bug. This will reduce the amount of code you will have to check before finding the code that’s causing the bug. - -### Disabling Assertions - -Assertions can be disabled by passing the `-O` option when running Python. - -## Logging - -To enable the `logging` module to display log messages on your screen as your program runs, copy the following to the top of your program: - -```python ->>> import logging ->>> logging.basicConfig(level=logging.DEBUG, format=' %(asctime)s - %(levelname)s- %(message)s') -``` - -Say you wrote a function to calculate the factorial of a number. In mathematics, factorial 4 is 1 × 2 × 3 × 4, or 24. Factorial 7 is 1 × 2 × 3 × 4 × 5 × 6 × 7, or 5,040. Open a new file editor window and enter the following code. It has a bug in it, but you will also enter several log messages to help yourself figure out what is going wrong. Save the program as factorialLog.py. - -```python ->>> import logging ->>> logging.basicConfig(level=logging.DEBUG, format=' %(asctime)s - %(levelname)s- %(message)s') ->>> logging.debug('Start of program') - ->>> def factorial(n): -... logging.debug('Start of factorial(%s)' % (n)) -... total = 1 -... for i in range(1, n + 1): -... total *= i -... logging.debug('i is ' + str(i) + ', total is ' + str(total)) -... logging.debug('End of factorial(%s)' % (n)) -... return total -... ->>> print(factorial(5)) ->>> logging.debug('End of program') -# 2015-05-23 16:20:12,664 - DEBUG - Start of program -# 2015-05-23 16:20:12,664 - DEBUG - Start of factorial(5) -# 2015-05-23 16:20:12,665 - DEBUG - i is 0, total is 0 -# 2015-05-23 16:20:12,668 - DEBUG - i is 1, total is 0 -# 2015-05-23 16:20:12,670 - DEBUG - i is 2, total is 0 -# 2015-05-23 16:20:12,673 - DEBUG - i is 3, total is 0 -# 2015-05-23 16:20:12,675 - DEBUG - i is 4, total is 0 -# 2015-05-23 16:20:12,678 - DEBUG - i is 5, total is 0 -# 2015-05-23 16:20:12,680 - DEBUG - End of factorial(5) -# 0 -# 2015-05-23 16:20:12,684 - DEBUG - End of program -``` - -## Logging Levels - -Logging levels provide a way to categorize your log messages by importance. There are five logging levels, described in Table 10-1 from least to most important. Messages can be logged at each level using a different logging function. - -| Level | Logging Function | Description | -| ---------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `DEBUG` | `logging.debug()` | The lowest level. Used for small details. Usually you care about these messages only when diagnosing problems. | -| `INFO` | `logging.info()` | Used to record information on general events in your program or confirm that things are working at their point in the program. | -| `WARNING` | `logging.warning()` | Used to indicate a potential problem that doesn’t prevent the program from working but might do so in the future. | -| `ERROR` | `logging.error()` | Used to record an error that caused the program to fail to do something. | -| `CRITICAL` | `logging.critical()` | The highest level. Used to indicate a fatal error that has caused or is about to cause the program to stop running entirely. | - -## Disabling Logging - -After you’ve debugged your program, you probably don’t want all these log messages cluttering the screen. The logging.disable() function disables these so that you don’t have to go into your program and remove all the logging calls by hand. - -```python ->>> import logging - ->>> logging.basicConfig(level=logging.INFO, format=' %(asctime)s -%(levelname)s - %(message)s') ->>> logging.critical('Critical error! Critical error!') -# 2015-05-22 11:10:48,054 - CRITICAL - Critical error! Critical error! - ->>> logging.disable(logging.CRITICAL) ->>> logging.critical('Critical error! Critical error!') ->>> logging.error('Error! Error!') -``` - -## Logging to a File - -Instead of displaying the log messages to the screen, you can write them to a text file. The `logging.basicConfig()` function takes a filename keyword argument, like so: - -```python ->>> import logging ->>> logging.basicConfig(filename='myProgramLog.txt', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') -``` diff --git a/PythonCheatsheats/decorators.md b/PythonCheatsheats/decorators.md deleted file mode 100644 index f385f6c..0000000 --- a/PythonCheatsheats/decorators.md +++ /dev/null @@ -1,188 +0,0 @@ ---- -title: Python Decorators - Python Cheatsheet -description: A Python Decorator is a syntax that provide a concise and reusable way for extending a function or a class. ---- - - -Python Decorators - - -A Python Decorator provides a concise and reusable way for extending -a function or a class. - -## Bare bone decorator - -A decorator in its simplest form is a function that takes another -function as an argument and returns a wrapper. The following example -shows the creation of a decorator and its usage. - -```python -def your_decorator(func): - def wrapper(): - # Do stuff before func... - print("Before func!") - func() - # Do stuff after func... - print("After func!") - return wrapper - -@your_decorator -def foo(): - print("Hello World!") - -foo() -# Before func! -# Hello World! -# After func! -``` - -## Decorator for a function with parameters - -```python -def your_decorator(func): - def wrapper(*args,**kwargs): - # Do stuff before func... - print("Before func!") - func(*args,**kwargs) - # Do stuff after func... - print("After func!") - return wrapper - -@your_decorator -def foo(bar): - print("My name is " + bar) - -foo("Jack") - -# Before func! -# My name is Jack -# After func! -``` - -## Template for a basic decorator - -This template is useful for most decorator use-cases. It is valid for functions -with or without parameters, and with or without a return value. - -```python -import functools - -def your_decorator(func): - @functools.wraps(func) # For preserving the metadata of func. - def wrapper(*args,**kwargs): - # Do stuff before func... - result = func(*args,**kwargs) - # Do stuff after func.. - return result - return wrapper -``` - -## Decorator with parameters - -You can also define parameters for the decorator to use. - -```python -import functools - -def your_decorator(arg): - def decorator(func): - @functools.wraps(func) # For preserving the metadata of func. - def wrapper(*args,**kwargs): - # Do stuff before func possibly using arg... - result = func(*args,**kwargs) - # Do stuff after func possibly using arg... - return result - return wrapper - return decorator -``` - -To use this decorator: - -```python -@your_decorator(arg = 'x') -def foo(bar): - return bar -``` - -## Class based decorators - -To decorate a class method, you must define the decorator within the class. When -only the implicit argument `self` is passed to the method, without any explicit -additional arguments, you must make a separate decorator for only those methods -without any additional arguments. An example of this, shown below, is when you -want to catch and print exceptions in a certain way. - -```python -class DecorateMyMethod: - - def decorator_for_class_method_with_no_args(method): - def wrapper_for_class_method(self) - try: - return method(self) - except Exception as e: - print("\nWARNING: Please make note of the following:\n") - print(e) - return wrapper_for_class_method - - def __init__(self,succeed:bool): - self.succeed = succeed - - @decorator_for_class_method_with_no_args - def class_action(self): - if self.succeed: - print("You succeeded by choice.") - else: - raise Exception("Epic fail of your own creation.") - -test_succeed = DecorateMyMethods(True) -test_succeed.class_action() -# You succeeded by choice. - -test_fail = DecorateMyMethod(False) -test_fail.class_action() -# Exception: Epic fail of your own creation. -``` - -A decorator can also be defined as a class instead of a method. This is useful -for maintaining and updating a state, such as in the following example, where we -count the number of calls made to a method: - -```python -class CountCallNumber: - - def __init__(self, func): - self.func = func - self.call_number = 0 - - def __call__(self, *args, **kwargs): - self.call_number += 1 - print("This is execution number " + str(self.call_number)) - return self.func(*args, **kwargs) - -@CountCallNumber -def say_hi(name): - print("Hi! My name is " + name) - -say_hi("Jack") -# This is execution number 1 -# Hi! My name is Jack - -say_hi("James") -# This is execution number 2 -# Hi! My name is James -``` - - - Count Example - - - This count example is inspired by Patrick Loeber's YouTube tutorial. - - - - - - - - - diff --git a/PythonCheatsheats/dictionaries.md b/PythonCheatsheats/dictionaries.md deleted file mode 100644 index a3e2c19..0000000 --- a/PythonCheatsheats/dictionaries.md +++ /dev/null @@ -1,269 +0,0 @@ ---- -title: Python Dictionaries - Python Cheatsheet -description: In Python, a dictionary is an insertion-ordered (from Python > 3.7) collection of key, value pairs. ---- - - -Python Dictionaries - - -In Python, a dictionary is an _ordered_ (from Python > 3.7) collection of `key`: `value` pairs. - - - - From the Python 3 documentation - - - The main operations on a dictionary are storing a value with some key and extracting the value given the key. It is also possible to delete a key:value pair with del. - - - -Example Dictionary: - -```python -my_cat = { - 'size': 'fat', - 'color': 'gray', - 'disposition': 'loud' -} -``` - -## Set key, value using subscript operator `[]` -```python ->>> my_cat = { -... 'size': 'fat', -... 'color': 'gray', -... 'disposition': 'loud', -... } ->>> my_cat['age_years'] = 2 ->>> print(my_cat) -... -# {'size': 'fat', 'color': 'gray', 'disposition': 'loud', 'age_years': 2} -``` - -## Get value using subscript operator `[]` - -In case the key is not present in dictionary `KeyError` is raised. - -```python ->>> my_cat = { -... 'size': 'fat', -... 'color': 'gray', -... 'disposition': 'loud', -... } ->>> print(my_cat['size']) -... -# fat ->>> print(my_cat['eye_color']) -# Traceback (most recent call last): -# File "", line 1, in -# KeyError: 'eye_color' -``` - -## values() - -The `values()` method gets the **values** of the dictionary: - -```python ->>> pet = {'color': 'red', 'age': 42} ->>> for value in pet.values(): -... print(value) -... -# red -# 42 -``` - -## keys() - -The `keys()` method gets the **keys** of the dictionary: - -```python ->>> pet = {'color': 'red', 'age': 42} ->>> for key in pet.keys(): -... print(key) -... -# color -# age -``` - -There is no need to use **.keys()** since by default you will loop through keys: - -```python ->>> pet = {'color': 'red', 'age': 42} ->>> for key in pet: -... print(key) -... -# color -# age -``` - -## items() - -The `items()` method gets the **items** of a dictionary and returns them as a Tuple: - -```python ->>> pet = {'color': 'red', 'age': 42} ->>> for item in pet.items(): -... print(item) -... -# ('color', 'red') -# ('age', 42) -``` - -Using the `keys()`, `values()`, and `items()` methods, a for loop can iterate over the keys, values, or key-value pairs in a dictionary, respectively. - -```python ->>> pet = {'color': 'red', 'age': 42} ->>> for key, value in pet.items(): -... print(f'Key: {key} Value: {value}') -... -# Key: color Value: red -# Key: age Value: 42 -``` - -## get() - -The `get()` method returns the value of an item with the given key. If the key doesn't exist, it returns `None`: - -```python ->>> wife = {'name': 'Rose', 'age': 33} - ->>> f'My wife name is {wife.get("name")}' -# 'My wife name is Rose' - ->>> f'She is {wife.get("age")} years old.' -# 'She is 33 years old.' - ->>> f'She is deeply in love with {wife.get("husband")}' -# 'She is deeply in love with None' -``` - -You can also change the default `None` value to one of your choice: - -```python ->>> wife = {'name': 'Rose', 'age': 33} - ->>> f'She is deeply in love with {wife.get("husband", "lover")}' -# 'She is deeply in love with lover' -``` - -## Adding items with setdefault() - -It's possible to add an item to a dictionary in this way: - -```python ->>> wife = {'name': 'Rose', 'age': 33} ->>> if 'has_hair' not in wife: -... wife['has_hair'] = True -``` - -Using the `setdefault` method, we can make the same code more short: - -```python ->>> wife = {'name': 'Rose', 'age': 33} ->>> wife.setdefault('has_hair', True) ->>> wife -# {'name': 'Rose', 'age': 33, 'has_hair': True} -``` - -## Removing Items - -### pop() - -The `pop()` method removes and returns an item based on a given key. - -```python ->>> wife = {'name': 'Rose', 'age': 33, 'hair': 'brown'} ->>> wife.pop('age') -# 33 ->>> wife -# {'name': 'Rose', 'hair': 'brown'} -``` - -### popitem() - -The `popitem()` method removes the last item in a dictionary and returns it. - -```python ->>> wife = {'name': 'Rose', 'age': 33, 'hair': 'brown'} ->>> wife.popitem() -# ('hair', 'brown') ->>> wife -# {'name': 'Rose', 'age': 33} -``` - -### del() - -The `del()` method removes an item based on a given key. - -```python ->>> wife = {'name': 'Rose', 'age': 33, 'hair': 'brown'} ->>> del wife['age'] ->>> wife -# {'name': 'Rose', 'hair': 'brown'} -``` - -### clear() - -The`clear()` method removes all the items in a dictionary. - -```python ->>> wife = {'name': 'Rose', 'age': 33, 'hair': 'brown'} ->>> wife.clear() ->>> wife -# {} -``` - -## Checking keys in a Dictionary - -```python ->>> person = {'name': 'Rose', 'age': 33} - ->>> 'name' in person.keys() -# True - ->>> 'height' in person.keys() -# False - ->>> 'skin' in person # You can omit keys() -# False -``` - -## Checking values in a Dictionary - -```python ->>> person = {'name': 'Rose', 'age': 33} - ->>> 'Rose' in person.values() -# True - ->>> 33 in person.values() -# True -``` - -## Pretty Printing - -```python ->>> import pprint - ->>> wife = {'name': 'Rose', 'age': 33, 'has_hair': True, 'hair_color': 'brown', 'height': 1.6, 'eye_color': 'brown'} ->>> pprint.pprint(wife) -# {'age': 33, -# 'eye_color': 'brown', -# 'hair_color': 'brown', -# 'has_hair': True, -# 'height': 1.6, -# 'name': 'Rose'} -``` - -## Merge two dictionaries - -For Python 3.5+: - -```python ->>> dict_a = {'a': 1, 'b': 2} ->>> dict_b = {'b': 3, 'c': 4} ->>> dict_c = {**dict_a, **dict_b} ->>> dict_c -# {'a': 1, 'b': 3, 'c': 4} -``` diff --git a/PythonCheatsheats/exception-handling.md b/PythonCheatsheats/exception-handling.md deleted file mode 100644 index ae3fcd9..0000000 --- a/PythonCheatsheats/exception-handling.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -title: Python Exception Handling - Python Cheatsheet -description: In Python, exception handling is the process of responding to the occurrence of exceptions. ---- - - -Python Exception Handling - - - - - Exception handling - - - In computing and computer programming, exception handling is the process of responding to the occurrence of exceptions – anomalous or exceptional conditions requiring special processing. - - - -Python has many [built-in exceptions](https://docs.python.org/3/library/exceptions.html) that are raised when a program encounters an error, and most external libraries, like the popular [Requests](https://requests.readthedocs.io/en/latest), include his own [custom exceptions](https://requests.readthedocs.io/en/latest/user/quickstart/#errors-and-exceptions) that we will need to deal to. - -## Basic exception handling - -You can't divide by zero, that is a mathematical true, and if you try to do it in Python, the interpreter will raise the built-in exception [ZeroDivisionError](https://docs.python.org/3/library/exceptions.html#ZeroDivisionError): - -```python ->>> def divide(dividend , divisor): -... print(dividend / divisor) -... ->>> divide(dividend=10, divisor=5) -# 2 - ->>> divide(dividend=10, divisor=0) -# Traceback (most recent call last): -# File "", line 1, in -# ZeroDivisionError: division by zero -``` - -Let's say we don't want our program to stop its execution or show the user an output he will not understand. Say we want to print a useful and clear message, then we need to **_handle_** the exception with the `try` and `except` keywords: - -```python ->>> def divide(dividend , divisor): -... try: -... print(dividend / divisor) -... except ZeroDivisionError: -... print('You can not divide by 0') -... ->>> divide(dividend=10, divisor=5) -# 2 - ->>> divide(dividend=10, divisor=0) -# You can not divide by 0 -``` - -## Handling Multiple exceptions using one exception block - -You can also handle multiple exceptions in one line like the following without the need to create multiple exception blocks. - -```python ->>> def divide(dividend , divisor): -... try: -... if (dividend == 10): -... var = 'str' + 1 -... else: -... print(dividend / divisor) -... except (ZeroDivisionError, TypeError) as error: -... print(error) -... - ->>> divide(dividend=20, divisor=5) -# 4 - ->>> divide(dividend=10, divisor=5) -# `can only concatenate str (not "int") to str` Error message - ->>> divide(dividend=10, divisor=0) -# `division by zero` Error message -``` - -## Finally code in exception handling - -The code inside the `finally` section is always executed, no matter if an exception has been raised or not: - -```python ->>> def divide(dividend , divisor): -... try: -... print(dividend / divisor) -... except ZeroDivisionError: -... print('You can not divide by 0') -... finally: -... print('Execution finished') -... ->>> divide(dividend=10, divisor=5) -# 5 -# Execution finished - ->>> divide(dividend=10, divisor=0) -# You can not divide by 0 -# Execution finished -``` - -## Custom Exceptions - -Custom exceptions initialize by creating a `class` that inherits from the base `Exception` class of Python, and are raised using the `raise` keyword: - -```python ->>> class MyCustomException(Exception): -... pass -... ->>> raise MyCustomException -# Traceback (most recent call last): -# File "", line 1, in -# __main__.MyCustomException -``` - -To declare a custom exception message, you can pass it as a parameter: - -```python ->>> class MyCustomException(Exception): -... pass -... ->>> raise MyCustomException('A custom message for my custom exception') -# Traceback (most recent call last): -# File "", line 1, in -# __main__.MyCustomException: A custom message for my custom exception -``` - -Handling a custom exception is the same as any other: - -```python ->>> try: -... raise MyCustomException('A custom message for my custom exception') ->>> except MyCustomException: -... print('My custom exception was raised') -... -# My custom exception was raised -``` diff --git a/PythonCheatsheats/file-directory-path.md b/PythonCheatsheats/file-directory-path.md deleted file mode 100644 index ef42610..0000000 --- a/PythonCheatsheats/file-directory-path.md +++ /dev/null @@ -1,529 +0,0 @@ ---- -title: File and directory Paths - Python Cheatsheet -description: There are two main modules in Python that deals with path manipulation. One is the os.path module and the other is the pathlib module. ---- - - -Handling file and directory Paths - - -There are two main modules in Python that deal with path manipulation. -One is the `os.path` module and the other is the `pathlib` module. - - - - os.path VS pathlib - - - The `pathlib` module was added in Python 3.4, offering an object-oriented way to handle file system paths. - - - -## Linux and Windows Paths - -On Windows, paths are written using backslashes (`\`) as the separator between -folder names. On Unix based operating system such as macOS, Linux, and BSDs, -the forward slash (`/`) is used as the path separator. Joining paths can be -a headache if your code needs to work on different platforms. - -Fortunately, Python provides easy ways to handle this. We will showcase -how to deal with both, `os.path.join` and `pathlib.Path.joinpath` - -Using `os.path.join` on Windows: - -```python ->>> import os - ->>> os.path.join('usr', 'bin', 'spam') -# 'usr\\bin\\spam' -``` - -And using `pathlib` on \*nix: - -```python ->>> from pathlib import Path - ->>> print(Path('usr').joinpath('bin').joinpath('spam')) -# usr/bin/spam -``` - -`pathlib` also provides a shortcut to joinpath using the `/` operator: - -```python ->>> from pathlib import Path - ->>> print(Path('usr') / 'bin' / 'spam') -# usr/bin/spam -``` - -Notice the path separator is different between Windows and Unix based operating -system, that's why you want to use one of the above methods instead of -adding strings together to join paths together. - -Joining paths is helpful if you need to create different file paths under -the same directory. - -Using `os.path.join` on Windows: - -```python ->>> my_files = ['accounts.txt', 'details.csv', 'invite.docx'] - ->>> for filename in my_files: -... print(os.path.join('C:\\Users\\asweigart', filename)) -... -# C:\Users\asweigart\accounts.txt -# C:\Users\asweigart\details.csv -# C:\Users\asweigart\invite.docx -``` - -Using `pathlib` on \*nix: - -```python ->>> my_files = ['accounts.txt', 'details.csv', 'invite.docx'] ->>> home = Path.home() ->>> for filename in my_files: -... print(home / filename) -... -# /home/asweigart/accounts.txt -# /home/asweigart/details.csv -# /home/asweigart/invite.docx -``` - -## The current working directory - -Using `os` on Windows: - -```python ->>> import os - ->>> os.getcwd() -# 'C:\\Python34' ->>> os.chdir('C:\\Windows\\System32') - ->>> os.getcwd() -# 'C:\\Windows\\System32' -``` - -Using `pathlib` on \*nix: - -```python ->>> from pathlib import Path ->>> from os import chdir - ->>> print(Path.cwd()) -# /home/asweigart - ->>> chdir('/usr/lib/python3.6') ->>> print(Path.cwd()) -# /usr/lib/python3.6 -``` - -## Creating new folders - -Using `os` on Windows: - -```python ->>> import os ->>> os.makedirs('C:\\delicious\\walnut\\waffles') -``` - -Using `pathlib` on \*nix: - -```python ->>> from pathlib import Path ->>> cwd = Path.cwd() ->>> (cwd / 'delicious' / 'walnut' / 'waffles').mkdir() -# Traceback (most recent call last): -# File "", line 1, in -# File "/usr/lib/python3.6/pathlib.py", line 1226, in mkdir -# self._accessor.mkdir(self, mode) -# File "/usr/lib/python3.6/pathlib.py", line 387, in wrapped -# return strfunc(str(pathobj), *args) -# FileNotFoundError: [Errno 2] No such file or directory: '/home/asweigart/delicious/walnut/waffles' -``` - -Oh no, we got a nasty error! The reason is that the 'delicious' directory does -not exist, so we cannot make the 'walnut' and the 'waffles' directories under -it. To fix this, do: - -```python ->>> from pathlib import Path ->>> cwd = Path.cwd() ->>> (cwd / 'delicious' / 'walnut' / 'waffles').mkdir(parents=True) -``` - -And all is good :) - -## Absolute vs. Relative paths - -There are two ways to specify a file path. - -- An **absolute path**, which always begins with the root folder -- A **relative path**, which is relative to the program’s current working directory - -There are also the dot (`.`) and dot-dot (`..`) folders. These are not real folders, but special names that can be used in a path. A single period (“dot”) for a folder name is shorthand for “this directory.” Two periods (“dot-dot”) means “the parent folder.” - -### Handling Absolute paths - -To see if a path is an absolute path: - -Using `os.path` on \*nix: - -```python ->>> import os ->>> os.path.isabs('/') -# True - ->>> os.path.isabs('..') -# False -``` - -Using `pathlib` on \*nix: - -```python ->>> from pathlib import Path ->>> Path('/').is_absolute() -# True - ->>> Path('..').is_absolute() -# False -``` - -You can extract an absolute path with both `os.path` and `pathlib` - -Using `os.path` on \*nix: - -```python ->>> import os ->>> os.getcwd() -'/home/asweigart' - ->>> os.path.abspath('..') -'/home' -``` - -Using `pathlib` on \*nix: - -```python -from pathlib import Path -print(Path.cwd()) -# /home/asweigart - -print(Path('..').resolve()) -# /home -``` - -### Handling Relative paths - -You can get a relative path from a starting path to another path. - -Using `os.path` on \*nix: - -```python ->>> import os ->>> os.path.relpath('/etc/passwd', '/') -# 'etc/passwd' -``` - -Using `pathlib` on \*nix: - -```python ->>> from pathlib import Path ->>> print(Path('/etc/passwd').relative_to('/')) -# etc/passwd -``` - -## Path and File validity - -### Checking if a file/directory exists - -Using `os.path` on \*nix: - -```python ->>> import os - ->>> os.path.exists('.') -# True - ->>> os.path.exists('setup.py') -# True - ->>> os.path.exists('/etc') -# True - ->>> os.path.exists('nonexistentfile') -# False -``` - -Using `pathlib` on \*nix: - -```python -from pathlib import Path - ->>> Path('.').exists() -# True - ->>> Path('setup.py').exists() -# True - ->>> Path('/etc').exists() -# True - ->>> Path('nonexistentfile').exists() -# False -``` - -### Checking if a path is a file - -Using `os.path` on \*nix: - -```python ->>> import os - ->>> os.path.isfile('setup.py') -# True - ->>> os.path.isfile('/home') -# False - ->>> os.path.isfile('nonexistentfile') -# False -``` - -Using `pathlib` on \*nix: - -```python ->>> from pathlib import Path - ->>> Path('setup.py').is_file() -# True - ->>> Path('/home').is_file() -# False - ->>> Path('nonexistentfile').is_file() -# False -``` - -### Checking if a path is a directory - -Using `os.path` on \*nix: - -```python ->>> import os - ->>> os.path.isdir('/') -# True - ->>> os.path.isdir('setup.py') -# False - ->>> os.path.isdir('/spam') -# False -``` - -Using `pathlib` on \*nix: - -```python ->>> from pathlib import Path - ->>> Path('/').is_dir() -# True - ->>> Path('setup.py').is_dir() -# False - ->>> Path('/spam').is_dir() -# False -``` - -## Getting a file's size in bytes - -Using `os.path` on Windows: - -```python ->>> import os - ->>> os.path.getsize('C:\\Windows\\System32\\calc.exe') -# 776192 -``` - -Using `pathlib` on \*nix: - -```python ->>> from pathlib import Path - ->>> stat = Path('/bin/python3.6').stat() ->>> print(stat) # stat contains some other information about the file as well -# os.stat_result(st_mode=33261, st_ino=141087, st_dev=2051, st_nlink=2, st_uid=0, -# --snip-- -# st_gid=0, st_size=10024, st_atime=1517725562, st_mtime=1515119809, st_ctime=1517261276) - ->>> print(stat.st_size) # size in bytes -# 10024 -``` - -## Listing directories - -Listing directory contents using `os.listdir` on Windows: - -```python ->>> import os - ->>> os.listdir('C:\\Windows\\System32') -# ['0409', '12520437.cpx', '12520850.cpx', '5U877.ax', 'aaclient.dll', -# --snip-- -# 'xwtpdui.dll', 'xwtpw32.dll', 'zh-CN', 'zh-HK', 'zh-TW', 'zipfldr.dll'] -``` - -Listing directory contents using `pathlib` on \*nix: - -```python ->>> from pathlib import Path - ->>> for f in Path('/usr/bin').iterdir(): -... print(f) -... -# ... -# /usr/bin/tiff2rgba -# /usr/bin/iconv -# /usr/bin/ldd -# /usr/bin/cache_restore -# /usr/bin/udiskie -# /usr/bin/unix2dos -# /usr/bin/t1reencode -# /usr/bin/epstopdf -# /usr/bin/idle3 -# ... -``` - -## Directory file sizes - - - - WARNING - - - Directories themselves also have a size! So, you might want to check for whether a path is a file or directory using the methods in the methods discussed in the above section. - - - -Using `os.path.getsize()` and `os.listdir()` together on Windows: - -```python ->>> import os ->>> total_size = 0 - ->>> for filename in os.listdir('C:\\Windows\\System32'): -... total_size = total_size + os.path.getsize(os.path.join('C:\\Windows\\System32', filename)) -... ->>> print(total_size) -# 1117846456 -``` - -Using `pathlib` on \*nix: - -```python ->>> from pathlib import Path - ->>> total_size = 0 ->>> for sub_path in Path('/usr/bin').iterdir(): -... total_size += sub_path.stat().st_size -... ->>> print(total_size) -# 1903178911 -``` - -## Copying files and folders - -The `shutil` module provides functions for copying files, as well as entire folders. - -```python ->>> import shutil, os - ->>> os.chdir('C:\\') ->>> shutil.copy('C:\\spam.txt', 'C:\\delicious') -# C:\\delicious\\spam.txt' - ->>> shutil.copy('eggs.txt', 'C:\\delicious\\eggs2.txt') -# 'C:\\delicious\\eggs2.txt' -``` - -While `shutil.copy()` will copy a single file, `shutil.copytree()` will copy an entire folder and every folder and file contained in it: - -```python ->>> import shutil, os - ->>> os.chdir('C:\\') ->>> shutil.copytree('C:\\bacon', 'C:\\bacon_backup') -# 'C:\\bacon_backup' -``` - -## Moving and Renaming - -```python ->>> import shutil - ->>> shutil.move('C:\\bacon.txt', 'C:\\eggs') -# 'C:\\eggs\\bacon.txt' -``` - -The destination path can also specify a filename. In the following example, the source file is moved and renamed: - -```python ->>> shutil.move('C:\\bacon.txt', 'C:\\eggs\\new_bacon.txt') -# 'C:\\eggs\\new_bacon.txt' -``` - -If there is no eggs folder, then `move()` will rename bacon.txt to a file named eggs: - -```python ->>> shutil.move('C:\\bacon.txt', 'C:\\eggs') -# 'C:\\eggs' -``` - -## Deleting files and folders - -- Calling `os.unlink(path)` or `Path.unlink()` will delete the file at path. - -- Calling `os.rmdir(path)` or `Path.rmdir()` will delete the folder at path. This folder must be empty of any files or folders. - -- Calling `shutil.rmtree(path)` will remove the folder at path, and all files and folders it contains will also be deleted. - -## Walking a Directory Tree - -```python ->>> import os ->>> ->>> for folder_name, subfolders, filenames in os.walk('C:\\delicious'): -... print(f'The current folder is {folder_name}') -... for subfolder in subfolders: -... print(f'SUBFOLDER OF {folder_name}: {subfolder}') -... for filename in filenames: -... print(f'FILE INSIDE {folder_name}: {filename}') -... print('') -... -# The current folder is C:\delicious -# SUBFOLDER OF C:\delicious: cats -# SUBFOLDER OF C:\delicious: walnut -# FILE INSIDE C:\delicious: spam.txt - -# The current folder is C:\delicious\cats -# FILE INSIDE C:\delicious\cats: catnames.txt -# FILE INSIDE C:\delicious\cats: zophie.jpg - -# The current folder is C:\delicious\walnut -# SUBFOLDER OF C:\delicious\walnut: waffles - -# The current folder is C:\delicious\walnut\waffles -# FILE INSIDE C:\delicious\walnut\waffles: butter.txt -``` - - - - Pathlib vs Os Module - - - `pathlib` provides a lot more functionality than the ones listed above, like getting file name, getting file extension, reading/writing a file without manually opening it, etc. See the official documentation if you intend to know more. - - diff --git a/PythonCheatsheats/functions.md b/PythonCheatsheats/functions.md deleted file mode 100644 index 5781482..0000000 --- a/PythonCheatsheats/functions.md +++ /dev/null @@ -1,176 +0,0 @@ ---- -title: Python Functions - Python Cheatsheet -description: In Python, A function is a block of organized code that is used to perform a single task. ---- - - -Python Functions - - - - - Programming Functions - - - A function is a block of organized code that is used to perform a single task. They provide better modularity for your application and reuse-ability. - - - -## Function Arguments - -A function can take `arguments` and `return values`: - -In the following example, the function **say_hello** receives the argument "name" and prints a greeting: - -```python ->>> def say_hello(name): -... print(f'Hello {name}') -... ->>> say_hello('Carlos') -# Hello Carlos - ->>> say_hello('Wanda') -# Hello Wanda - ->>> say_hello('Rose') -# Hello Rose -``` - -## Keyword Arguments - -To improve code readability, we should be as explicit as possible. We can achieve this in our functions by using `Keyword Arguments`: - -```python ->>> def say_hi(name, greeting): -... print(f"{greeting} {name}") -... ->>> # without keyword arguments ->>> say_hi('John', 'Hello') -# Hello John - ->>> # with keyword arguments ->>> say_hi(name='Anna', greeting='Hi') -# Hi Anna -``` - -## Return Values - -When creating a function using the `def` statement, you can specify what the return value should be with a `return` statement. A return statement consists of the following: - -- The `return` keyword. - -- The value or expression that the function should return. - -```python ->>> def sum_two_numbers(number_1, number_2): -... return number_1 + number_2 -... ->>> result = sum_two_numbers(7, 8) ->>> print(result) -# 15 -``` - -## Local and Global Scope - -- Code in the global scope cannot use any local variables. - -- However, a local scope can access global variables. - -- Code in a function’s local scope cannot use variables in any other local scope. - -- You can use the same name for different variables if they are in different scopes. That is, there can be a local variable named spam and a global variable also named spam. - -```python -global_variable = 'I am available everywhere' - ->>> def some_function(): -... print(global_variable) # because is global -... local_variable = "only available within this function" -... print(local_variable) -... ->>> # the following code will throw error because ->>> # 'local_variable' only exists inside 'some_function' ->>> print(local_variable) -Traceback (most recent call last): - File "", line 10, in -NameError: name 'local_variable' is not defined -``` - -## The global Statement - -If you need to modify a global variable from within a function, use the global statement: - -```python ->>> def spam(): -... global eggs -... eggs = 'spam' -... ->>> eggs = 'global' ->>> spam() ->>> print(eggs) -``` - -There are four rules to tell whether a variable is in a local scope or global scope: - -1. If a variable is being used in the global scope (that is, outside all functions), then it is always a global variable. - -1. If there is a global statement for that variable in a function, it is a global variable. - -1. Otherwise, if the variable is used in an assignment statement in the function, it is a local variable. - -1. But if the variable is not used in an assignment statement, it is a global variable. - -## Lambda Functions - -In Python, a lambda function is a single-line, anonymous function, which can have any number of arguments, but it can only have one expression. - - - - From the Python 3 Tutorial - - - lambda is a minimal function definition that can be used inside an expression. Unlike FunctionDef, body holds a single node. - - - - - - Single line expression - - - Lambda functions can only evaluate an expression, like a single line of code. - - - -This function: - -```python ->>> def add(x, y): -... return x + y -... ->>> add(5, 3) -# 8 -``` - -Is equivalent to the _lambda_ function: - -```python ->>> add = lambda x, y: x + y ->>> add(5, 3) -# 8 -``` - -Like regular nested functions, lambdas also work as lexical closures: - -```python ->>> def make_adder(n): -... return lambda x: x + n -... ->>> plus_3 = make_adder(3) ->>> plus_5 = make_adder(5) - ->>> plus_3(4) -# 7 ->>> plus_5(4) -# 9 -``` diff --git a/PythonCheatsheats/json-yaml.md b/PythonCheatsheats/json-yaml.md deleted file mode 100644 index dbdbbfe..0000000 --- a/PythonCheatsheats/json-yaml.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Python Json and YAML - Python Cheatsheet -description: JSON stands for JavaScript Object Notation and is a lightweight format for storing and transporting data. Json is often used when data is sent from a server to a web page. ---- - - -JSON and YAML - - -## JSON - -JSON stands for JavaScript Object Notation and is a lightweight format for storing and transporting data. Json is often used when data is sent from a server to a web page. - -```python ->>> import json ->>> with open("filename.json", "r") as f: -... content = json.load(f) -``` - -Write a JSON file with: - -```python ->>> import json - ->>> content = {"name": "Joe", "age": 20} ->>> with open("filename.json", "w") as f: -... json.dump(content, f, indent=2) -``` - -## YAML - -Compared to JSON, YAML allows a much better human maintainability and gives ability to add comments. It is a convenient choice for configuration files where a human will have to edit. - -There are two main libraries allowing access to YAML files: - -- [PyYaml](https://pypi.python.org/pypi/PyYAML) -- [Ruamel.yaml](https://pypi.python.org/pypi/ruamel.yaml) - -Install them using `pip install` in your virtual environment. - -The first one is easier to use but the second one, Ruamel, implements much better the YAML -specification, and allow for example to modify a YAML content without altering comments. - -Open a YAML file with: - -```python ->>> from ruamel.yaml import YAML - ->>> with open("filename.yaml") as f: -... yaml=YAML() -... yaml.load(f) -``` - -## Anyconfig - -[Anyconfig](https://pypi.python.org/pypi/anyconfig) is a very handy package, allowing to abstract completely the underlying configuration file format. It allows to load a Python dictionary from JSON, YAML, TOML, and so on. - -Install it with: - -```bash -pip install anyconfig -``` - -Usage: - -```python ->>> import anyconfig ->>> conf1 = anyconfig.load("/path/to/foo/conf.d/a.yml") -``` diff --git a/PythonCheatsheats/lists-and-tuples.md b/PythonCheatsheats/lists-and-tuples.md deleted file mode 100644 index f3586e7..0000000 --- a/PythonCheatsheats/lists-and-tuples.md +++ /dev/null @@ -1,394 +0,0 @@ ---- -title: Python Lists and Tuples - Python Cheatsheet -description: In python, Lists are are one of the 4 data types in Python used to store collections of data. ---- - - -Python Lists - - -Lists are one of the 4 data types in Python used to store collections of data. - -```python -['John', 'Peter', 'Debora', 'Charles'] -``` - -## Getting values with indexes - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] - ->>> furniture[0] -# 'table' - ->>> furniture[1] -# 'chair' - ->>> furniture[2] -# 'rack' - ->>> furniture[3] -# 'shelf' -``` - -## Negative indexes - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] - ->>> furniture[-1] -# 'shelf' - ->>> furniture[-3] -# 'chair' - ->>> f'The {furniture[-1]} is bigger than the {furniture[-3]}' -# 'The shelf is bigger than the chair' -``` - -## Getting sublists with Slices - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] - ->>> furniture[0:4] -# ['table', 'chair', 'rack', 'shelf'] - ->>> furniture[1:3] -# ['chair', 'rack'] - ->>> furniture[0:-1] -# ['table', 'chair', 'rack'] - ->>> furniture[:2] -# ['table', 'chair'] - ->>> furniture[1:] -# ['chair', 'rack', 'shelf'] - ->>> furniture[:] -# ['table', 'chair', 'rack', 'shelf'] -``` - -Slicing the complete list will perform a copy: - -```python ->>> spam2 = spam[:] -# ['cat', 'bat', 'rat', 'elephant'] - ->>> spam.append('dog') ->>> spam -# ['cat', 'bat', 'rat', 'elephant', 'dog'] - ->>> spam2 -# ['cat', 'bat', 'rat', 'elephant'] -``` - -## Getting a list length with len() - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] ->>> len(furniture) -# 4 -``` - -## Changing values with indexes - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] - ->>> furniture[0] = 'desk' ->>> furniture -# ['desk', 'chair', 'rack', 'shelf'] - ->>> furniture[2] = furniture[1] ->>> furniture -# ['desk', 'chair', 'chair', 'shelf'] - ->>> furniture[-1] = 'bed' ->>> furniture -# ['desk', 'chair', 'chair', 'bed'] -``` - -## Concatenation and Replication - -```python ->>> [1, 2, 3] + ['A', 'B', 'C'] -# [1, 2, 3, 'A', 'B', 'C'] - ->>> ['X', 'Y', 'Z'] * 3 -# ['X', 'Y', 'Z', 'X', 'Y', 'Z', 'X', 'Y', 'Z'] - ->>> my_list = [1, 2, 3] ->>> my_list = my_list + ['A', 'B', 'C'] ->>> my_list -# [1, 2, 3, 'A', 'B', 'C'] -``` - -## Using for loops with Lists - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] - ->>> for item in furniture: -... print(item) -# table -# chair -# rack -# shelf -``` - -## Getting the index in a loop with enumerate() - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] - ->>> for index, item in enumerate(furniture): -... print(f'index: {index} - item: {item}') -# index: 0 - item: table -# index: 1 - item: chair -# index: 2 - item: rack -# index: 3 - item: shelf -``` - -## Loop in Multiple Lists with zip() - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] ->>> price = [100, 50, 80, 40] - ->>> for item, amount in zip(furniture, price): -... print(f'The {item} costs ${amount}') -# The table costs $100 -# The chair costs $50 -# The rack costs $80 -# The shelf costs $40 -``` - -## The in and not in operators - -```python ->>> 'rack' in ['table', 'chair', 'rack', 'shelf'] -# True - ->>> 'bed' in ['table', 'chair', 'rack', 'shelf'] -# False - ->>> 'bed' not in furniture -# True - ->>> 'rack' not in furniture -# False -``` - -## The Multiple Assignment Trick - -The multiple assignment trick is a shortcut that lets you assign multiple variables with the values in a list in one line of code. So instead of doing this: - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] ->>> table = furniture[0] ->>> chair = furniture[1] ->>> rack = furniture[2] ->>> shelf = furniture[3] -``` - -You could type this line of code: - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] ->>> table, chair, rack, shelf = furniture - ->>> table -# 'table' - ->>> chair -# 'chair' - ->>> rack -# 'rack' - ->>> shelf -# 'shelf' -``` - -The multiple assignment trick can also be used to swap the values in two variables: - -```python ->>> a, b = 'table', 'chair' ->>> a, b = b, a ->>> print(a) -# chair - ->>> print(b) -# table -``` - -## The index Method - -The `index` method allows you to find the index of a value by passing its name: - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] ->>> furniture.index('chair') -# 1 -``` - -## Adding Values - -### append() - -`append` adds an element to the end of a `list`: - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] ->>> furniture.append('bed') ->>> furniture -# ['table', 'chair', 'rack', 'shelf', 'bed'] -``` - -### insert() - -`insert` adds an element to a `list` at a given position: - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] ->>> furniture.insert(1, 'bed') ->>> furniture -# ['table', 'bed', 'chair', 'rack', 'shelf'] -``` - -## Removing Values - -### del() - -`del` removes an item using the index: - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] ->>> del furniture[2] ->>> furniture -# ['table', 'chair', 'shelf'] - ->>> del furniture[2] ->>> furniture -# ['table', 'chair'] -``` - -### remove() - -`remove` removes an item with using actual value of it: - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] ->>> furniture.remove('chair') ->>> furniture -# ['table', 'rack', 'shelf'] -``` - - - - Removing repeated items - - - If the value appears multiple times in the list, only the first instance of the value will be removed. - - - -### pop() - -By default, `pop` will remove and return the last item of the list. You can also pass the index of the element as an optional parameter: - -```python ->>> animals = ['cat', 'bat', 'rat', 'elephant'] - ->>> animals.pop() -'elephant' - ->>> animals -['cat', 'bat', 'rat'] - ->>> animals.pop(0) -'cat' - ->>> animals -['bat', 'rat'] -``` - -## Sorting values with sort() - -```python ->>> numbers = [2, 5, 3.14, 1, -7] ->>> numbers.sort() ->>> numbers -# [-7, 1, 2, 3.14, 5] - -furniture = ['table', 'chair', 'rack', 'shelf'] -furniture.sort() -furniture -# ['chair', 'rack', 'shelf', 'table'] -``` - -You can also pass `True` for the `reverse` keyword argument to have `sort()` sort the values in reverse order: - -```python ->>> furniture.sort(reverse=True) ->>> furniture -# ['table', 'shelf', 'rack', 'chair'] -``` - -If you need to sort the values in regular alphabetical order, pass `str.lower` for the key keyword argument in the sort() method call: - -```python ->>> letters = ['a', 'z', 'A', 'Z'] ->>> letters.sort(key=str.lower) ->>> letters -# ['a', 'A', 'z', 'Z'] -``` - -You can use the built-in function `sorted` to return a new list: - -```python ->>> furniture = ['table', 'chair', 'rack', 'shelf'] ->>> sorted(furniture) -# ['chair', 'rack', 'shelf', 'table'] -``` - -## The Tuple data type - - - - Tuples vs Lists - - - The key difference between tuples and lists is that, while tuples are immutable objects, lists are mutable. This means that tuples cannot be changed while the lists can be modified. Tuples are more memory efficient than the lists. - - - -```python ->>> furniture = ('table', 'chair', 'rack', 'shelf') - ->>> furniture[0] -# 'table' - ->>> furniture[1:3] -# ('chair', 'rack') - ->>> len(furniture) -# 4 -``` - -The main way that tuples are different from lists is that tuples, like strings, are immutable. - -## Converting between list() and tuple() - -```python ->>> tuple(['cat', 'dog', 5]) -# ('cat', 'dog', 5) - ->>> list(('cat', 'dog', 5)) -# ['cat', 'dog', 5] - ->>> list('hello') -# ['h', 'e', 'l', 'l', 'o'] -``` diff --git a/PythonCheatsheats/main.md b/PythonCheatsheats/main.md deleted file mode 100644 index 4ce35ad..0000000 --- a/PythonCheatsheats/main.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: Python Main function - Python Cheatsheet -description: is the name of the scope in which top-level code executes. A module’s name is set equal to main when read from standard input, a script, or from an interactive prompt. ---- - - -Main top-level script environment - - -## What is it - -`__main__` is the name of the scope in which top-level code executes. -A module’s **name** is set equal to `__main__` when read from standard input, a script, or from an interactive prompt. - -A module can discover whether it is running in the main scope by checking its own `__name__`, which allows a common idiom for conditionally executing code in a module. When it is run as a script or with `python -m` but not when it is imported: - -```python ->>> if __name__ == "__main__": -... # execute only if run as a script -... main() -``` - -For a package, the same effect can be achieved by including a **main**.py module, the contents of which will be executed when the module is run with -m. - -For example, we are developing a script designed to be used as a module, we should do: - -```python ->>> def add(a, b): -... return a+b -... ->>> if __name__ == "__main__": -... add(3, 5) -``` - -## Advantages - -1. Every Python module has it’s `__name__` defined and if this is `__main__`, it implies that the module is run standalone by the user, and we can do corresponding appropriate actions. -2. If you import this script as a module in another script, the **name** is set to the name of the script/module. -3. Python files can act as either reusable modules, or as standalone programs. -4. `if __name__ == "__main__":` is used to execute some code only if the file is run directly, and is not being imported. diff --git a/PythonCheatsheats/manipulating-strings.md b/PythonCheatsheats/manipulating-strings.md deleted file mode 100644 index 03c13fb..0000000 --- a/PythonCheatsheats/manipulating-strings.md +++ /dev/null @@ -1,332 +0,0 @@ ---- -title: Manipulating strings - Python Cheatsheet -description: An escape character is created by typing a backslash \ followed by the character you want to insert. ---- - - -Manipulating Strings - - -## Escape characters - -An escape character is created by typing a backslash `\` followed by the character you want to insert. - -| Escape character | Prints as | -| ---------------- | -------------------- | -| `\'` | Single quote | -| `\"` | Double quote | -| `\t` | Tab | -| `\n` | Newline (line break) | -| `\\` | Backslash | -| `\b` | Backspace | -| `\ooo` | Octal value | -| `\r` | Carriage Return | - -```python ->>> print("Hello there!\nHow are you?\nI\'m doing fine.") -# Hello there! -# How are you? -# I'm doing fine. -``` - -## Raw strings - -A raw string entirely ignores all escape characters and prints any backslash that appears in the string. - -```python ->>> print(r"Hello there!\nHow are you?\nI\'m doing fine.") -# Hello there!\nHow are you?\nI\'m doing fine. -``` - -Raw strings are mostly used for regular expression definition. - -## Multiline Strings - -```python ->>> print( -... """Dear Alice, -... -... Eve's cat has been arrested for catnapping, -... cat burglary, and extortion. -... -... Sincerely, -... Bob""" -... ) - -# Dear Alice, - -# Eve's cat has been arrested for catnapping, -# cat burglary, and extortion. - -# Sincerely, -# Bob -``` - -## Indexing and Slicing strings - - H e l l o w o r l d ! - 0 1 2 3 4 5 6 7 8 9 10 11 - -### Indexing - -```python ->>> spam = 'Hello world!' - ->>> spam[0] -# 'H' - ->>> spam[4] -# 'o' - ->>> spam[-1] -# '!' -``` - -### Slicing - -```python ->>> spam = 'Hello world!' - ->>> spam[0:5] -# 'Hello' - ->>> spam[:5] -# 'Hello' - ->>> spam[6:] -# 'world!' - ->>> spam[6:-1] -# 'world' - ->>> spam[:-1] -# 'Hello world' - ->>> spam[::-1] -# '!dlrow olleH' - ->>> fizz = spam[0:5] ->>> fizz -# 'Hello' -``` - -## The in and not in operators - -```python ->>> 'Hello' in 'Hello World' -# True - ->>> 'Hello' in 'Hello' -# True - ->>> 'HELLO' in 'Hello World' -# False - ->>> '' in 'spam' -# True - ->>> 'cats' not in 'cats and dogs' -# False -``` - -## upper(), lower() and title() - -Transforms a string to upper, lower and title case: - -```python ->>> greet = 'Hello world!' ->>> greet.upper() -# 'HELLO WORLD!' - ->>> greet.lower() -# 'hello world!' - ->>> greet.title() -# 'Hello World!' -``` - -## isupper() and islower() methods - -Returns `True` or `False` after evaluating if a string is in upper or lower case: - -```python ->>> spam = 'Hello world!' ->>> spam.islower() -# False - ->>> spam.isupper() -# False - ->>> 'HELLO'.isupper() -# True - ->>> 'abc12345'.islower() -# True - ->>> '12345'.islower() -# False - ->>> '12345'.isupper() -# False -``` - -## The isX string methods - -| Method | Description | -| ----------- | ------------------------------------------------------------------------------------------------------------------------------ | -| isalpha() | returns `True` if the string consists only of letters. | -| isalnum() | returns `True` if the string consists only of letters and numbers. | -| isdecimal() | returns `True` if the string consists only of numbers. | -| isspace() | returns `True` if the string consists only of spaces, tabs, and new-lines. | -| istitle() | returns `True` if the string consists only of words that begin with an uppercase letter followed by only lowercase characters. | - -## startswith() and endswith() - -```python ->>> 'Hello world!'.startswith('Hello') -# True - ->>> 'Hello world!'.endswith('world!') -# True - ->>> 'abc123'.startswith('abcdef') -# False - ->>> 'abc123'.endswith('12') -# False - ->>> 'Hello world!'.startswith('Hello world!') -# True - ->>> 'Hello world!'.endswith('Hello world!') -# True -``` - -## join() and split() - -### join() - -The `join()` method takes all the items in an iterable, like a list, dictionary, tuple or set, and joins them into a string. You can also specify a separator. - -```python ->>> ''.join(['My', 'name', 'is', 'Simon']) -'MynameisSimon' - ->>> ', '.join(['cats', 'rats', 'bats']) -# 'cats, rats, bats' - ->>> ' '.join(['My', 'name', 'is', 'Simon']) -# 'My name is Simon' - ->>> 'ABC'.join(['My', 'name', 'is', 'Simon']) -# 'MyABCnameABCisABCSimon' -``` - -### split() - -The `split()` method splits a `string` into a `list`. By default, it will use whitespace to separate the items, but you can also set another character of choice: - -```python ->>> 'My name is Simon'.split() -# ['My', 'name', 'is', 'Simon'] - ->>> 'MyABCnameABCisABCSimon'.split('ABC') -# ['My', 'name', 'is', 'Simon'] - ->>> 'My name is Simon'.split('m') -# ['My na', 'e is Si', 'on'] - ->>> ' My name is Simon'.split() -# ['My', 'name', 'is', 'Simon'] - ->>> ' My name is Simon'.split(' ') -# ['', 'My', '', 'name', 'is', '', 'Simon'] -``` - -## Justifying text with rjust(), ljust() and center() - -```python ->>> 'Hello'.rjust(10) -# ' Hello' - ->>> 'Hello'.rjust(20) -# ' Hello' - ->>> 'Hello World'.rjust(20) -# ' Hello World' - ->>> 'Hello'.ljust(10) -# 'Hello ' - ->>> 'Hello'.center(20) -# ' Hello ' -``` - -An optional second argument to `rjust()` and `ljust()` will specify a fill character apart from a space character: - -```python ->>> 'Hello'.rjust(20, '*') -# '***************Hello' - ->>> 'Hello'.ljust(20, '-') -# 'Hello---------------' - ->>> 'Hello'.center(20, '=') -# '=======Hello========' -``` - -## Removing whitespace with strip(), rstrip(), and lstrip() - -```python ->>> spam = ' Hello World ' ->>> spam.strip() -# 'Hello World' - ->>> spam.lstrip() -# 'Hello World ' - ->>> spam.rstrip() -# ' Hello World' - ->>> spam = 'SpamSpamBaconSpamEggsSpamSpam' ->>> spam.strip('ampS') -# 'BaconSpamEggs' -``` - -## The Count Method - -Counts the number of occurrences of a given character or substring in the string it is applied to. Can be optionally provided start and end index. - -```python ->>> sentence = 'one sheep two sheep three sheep four' ->>> sentence.count('sheep') -# 3 - ->>> sentence.count('e') -# 9 - ->>> sentence.count('e', 6) -# 8 -# returns count of e after 'one sh' i.e 6 chars since beginning of string - ->>> sentence.count('e', 7) -# 7 -``` - -## Replace Method - -Replaces all occurences of a given substring with another substring. Can be optionally provided a third argument to limit the number of replacements. Returns a new string. - -```python ->>> text = "Hello, world!" ->>> text.replace("world", "planet") -# 'Hello, planet!' - ->>> fruits = "apple, banana, cherry, apple" ->>> fruits.replace("apple", "orange", 1) -# 'orange, banana, cherry, apple' - ->>> sentence = "I like apples, Apples are my favorite fruit" ->>> sentence.replace("apples", "oranges") -# 'I like oranges, Apples are my favorite fruit' -``` diff --git a/PythonCheatsheats/oop-basics.md b/PythonCheatsheats/oop-basics.md deleted file mode 100644 index 24147bf..0000000 --- a/PythonCheatsheats/oop-basics.md +++ /dev/null @@ -1,216 +0,0 @@ - --- -title: Python OOP Basics - Python Cheatsheet - -description: Object-Oriented Programming (OOP) is a programming paradigm that revolves around the concept of objects, which are instances of classes. OOP principles are fundamental concepts that guide the design and development of software in an object-oriented way. In Python, OOP is supported by the use of classes and objects. Here are some of the basic OOP principles in Python ---- - - -Python OOP Basics - - - - - Object-Oriented Programming - - - Object-oriented programming (OOP) is a programming paradigm based on the concept of "objects", which can contain data and code. The data is in the form of fields (often known as attributes or properties), and the code is in the form of procedures (often known as methods). - - - -## Encapsulation - -Encapsulation is one of the fundamental concepts of object-oriented programming, which helps to protect the data and methods of an object from unauthorized access and modification. It is a way to achieve data abstraction, which means that the implementation details of an object are hidden from the outside world, and only the essential information is exposed. - -In Python, encapsulation can be achieved by using access modifiers. Access modifiers are keywords that define the accessibility of attributes and methods in a class. The three access modifiers available in Python are public, private, and protected. However, Python does not have an explicit way of defining access modifiers like some other programming languages such as Java and C++. Instead, it uses a convention of using underscore prefixes to indicate the access level. - -In the given code example, the class MyClass has two attributes, _protected_var and __private_var. The _protected_var is marked as protected by using a single underscore prefix. This means that the attribute can be accessed within the class and its subclasses but not outside the class. The __private_var is marked as private by using two underscore prefixes. This means that the attribute can only be accessed within the class and not outside the class, not even in its subclasses. - -When we create an object of the MyClass class, we can access the _protected_var attribute using the object name with a single underscore prefix. However, we cannot access the __private_var attribute using the object name, as it is hidden from the outside world. If we try to access the __private_var attribute, we will get an AttributeError as shown in the code. - -In summary, encapsulation is an important concept in object-oriented programming that helps to protect the implementation details of an object. In Python, we can achieve encapsulation by using access modifiers and using underscore prefixes to indicate the access level. - -```python -# Define a class named MyClass -class MyClass: - - # Constructor method that initializes the class object - def __init__(self): - - # Define a protected variable with an initial value of 10 - # The variable name starts with a single underscore, which indicates protected access - self._protected_var = 10 - - # Define a private variable with an initial value of 20 - # The variable name starts with two underscores, which indicates private access - self.__private_var = 20 - -# Create an object of MyClass class -obj = MyClass() - -# Access the protected variable using the object name and print its value -# The protected variable can be accessed outside the class but -# it is intended to be used within the class or its subclasses -print(obj._protected_var) # output: 10 - -# Try to access the private variable using the object name and print its value -# The private variable cannot be accessed outside the class, even by its subclasses -# This will raise an AttributeError because the variable is not accessible outside the class -print(obj.__private_var) # AttributeError: 'MyClass' object has no attribute '__private_var' - -``` - -## Inheritance - -Inheritance promotes code reuse and allows you to create a hierarchy of classes that share common attributes and methods. It helps in creating clean and organized code by keeping related functionality in one place and promoting the concept of modularity. The base class from which a new class is derived is also known as a parent class, and the new class is known as the child class or subclass. - -In the code, we define a class named Animal which has a constructor method that initializes the class object with a name attribute and a method named speak. The speak method is defined in the Animal class but does not have a body. - -We then define two subclasses named Dog and Cat which inherit from the Animal class. These subclasses override the speak method of the Animal class. - -We create a Dog object with a name attribute "Rover" and a Cat object with a name attribute "Whiskers". We call the speak method of the Dog object using dog.speak(), and it prints "Woof!" because the speak method of the Dog class overrides the speak method of the Animal class. Similarly, we call the speak method of the Cat object using cat.speak(), and it prints "Meow!" because the speak method of the Cat class overrides the speak method of the Animal class. - -``` python -# Define a class named Animal -class Animal: - - # Constructor method that initializes the class object with a name attribute - def __init__(self, name): - self.name = name - - # Method that is defined in the Animal class but does not have a body - # This method will be overridden in the subclasses of Animal - def speak(self): - print("") - -# Define a subclass named Dog that inherits from the Animal class -class Dog(Animal): - - # Override the speak method of the Animal class - def speak(self): - print("Woof!") - -# Define a subclass named Cat that inherits from the Animal class -class Cat(Animal): - - # Override the speak method of the Animal class - def speak(self): - print("Meow!") - -# Create a Dog object with a name attribute "Rover" -dog = Dog("Rover") - -# Create a Cat object with a name attribute "Whiskers" -cat = Cat("Whiskers") - -# Call the speak method of the Dog class and print the output -# The speak method of the Dog class overrides the speak method of the Animal class -# Therefore, when we call the speak method of the Dog object, it will print "Woof!" -dog.speak() # output: Woof! - -# Call the speak method of the Cat class and print the output -# The speak method of the Cat class overrides the speak method of the Animal class -# Therefore, when we call the speak method of the Cat object, it will print "Meow!" -cat.speak() # output: Meow! - -``` - -## Polymorphism - -Polymorphism is an important concept in object-oriented programming that allows you to write code that can work with objects of different classes in a uniform way. In Python, polymorphism is achieved by using method overriding or method overloading. - -Method overriding is when a subclass provides its own implementation of a method that is already defined in its parent class. This allows the subclass to modify the behavior of the method without changing its name or signature. - -Method overloading is when multiple methods have the same name but different parameters. Python does not support method overloading directly, but it can be achieved using default arguments or variable-length arguments. - -Polymorphism makes it easier to write flexible and reusable code. It allows you to write code that can work with different objects without needing to know their specific types. - -```python -#The Shape class is defined with an abstract area method, which is intended to be overridden by subclasses. -class Shape: - def area(self): - pass - -class Rectangle(Shape): - # The Rectangle class is defined with an __init__ method that initializes - # width and height instance variables. - # It also defines an area method that calculates and returns - # the area of a rectangle using the width and height instance variables. - def __init__(self, width, height): - self.width = width # Initialize width instance variable - self.height = height # Initialize height instance variable - - def area(self): - return self.width * self.height # Return area of rectangle - - - # The Circle class is defined with an __init__ method - # that initializes a radius instance variable. - # It also defines an area method that calculates and - # returns the area of a circle using the radius instance variable. -class Circle(Shape): - def __init__(self, radius): - self.radius = radius # Initialize radius instance variable - - def area(self): - return 3.14 * self.radius ** 2 # Return area of circle using pi * r^2 - -# The shapes list is created with one Rectangle object and one Circle object. The for -# loop iterates over each object in the list and calls the area method of each object -# The output will be the area of the rectangle (20) and the area of the circle (153.86). -shapes = [Rectangle(4, 5), Circle(7)] # Create a list of Shape objects -for shape in shapes: - print(shape.area()) # Output the area of each Shape object - -``` - -## Abstraction -Abstraction is an important concept in object-oriented programming (OOP) because it allows you to focus on the essential features of an object or system while ignoring the details that aren't relevant to the current context. By reducing complexity and hiding unnecessary details, abstraction can make code more modular, easier to read, and easier to maintain. - -In Python, abstraction can be achieved by using abstract classes or interfaces. An abstract class is a class that cannot be instantiated directly, but is meant to be subclassed by other classes. It often includes abstract methods that have no implementation, but provide a template for how the subclass should be implemented. This allows the programmer to define a common interface for a group of related classes, while still allowing each class to have its own specific behavior. - -An interface, on the other hand, is a collection of method signatures that a class must implement in order to be considered "compatible" with the interface. Interfaces are often used to define a common set of methods that multiple classes can implement, allowing them to be used interchangeably in certain contexts. - -Python does not have built-in support for abstract classes or interfaces, but they can be implemented using the abc (abstract base class) module. This module provides the ABC class and the abstractmethod decorator, which can be used to define abstract classes and methods. - -Overall, abstraction is a powerful tool for managing complexity and improving code quality in object-oriented programming, and Python provides a range of options for achieving abstraction in your code. - -```python -# Import the abc module to define abstract classes and methods -from abc import ABC, abstractmethod - -# Define an abstract class called Shape that has an abstract method called area -class Shape(ABC): - @abstractmethod - def area(self): - pass - -# Define a Rectangle class that inherits from Shape -class Rectangle(Shape): - def __init__(self, width, height): - self.width = width - self.height = height - - # Implement the area method for Rectangles - def area(self): - return self.width * self.height - -# Define a Circle class that also inherits from Shape -class Circle(Shape): - def __init__(self, radius): - self.radius = radius - - # Implement the area method for Circles - def area(self): - return 3.14 * self.radius ** 2 - -# Create a list of shapes that includes both Rectangles and Circles -shapes = [Rectangle(4, 5), Circle(7)] - -# Loop through each shape in the list and print its area -for shape in shapes: - print(shape.area()) - -``` - -These are some of the basic OOP principles in Python. This page is currently in progress and more -detailed examples and explanations will be coming soon. diff --git a/PythonCheatsheats/reading-and-writing-files.md b/PythonCheatsheats/reading-and-writing-files.md deleted file mode 100644 index a96ab8c..0000000 --- a/PythonCheatsheats/reading-and-writing-files.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: Reading and writing files - Python Cheatsheet -description: To read/write to a file in Python, you will want to use the with statement, which will close the file for you after you are done, managing the available resources for you. ---- - - -Reading and Writing Files - - -## The file Reading/Writing process - -To read/write to a file in Python, you will want to use the `with` -statement, which will close the file for you after you are done, managing the available resources for you. - -## Opening and reading files - -The `open` function opens a file and return a corresponding file object. - -```python ->>> with open('C:\\Users\\your_home_folder\\hi.txt') as hello_file: -... hello_content = hello_file.read() -... ->>> hello_content -'Hello World!' -``` - -Alternatively, you can use the _readlines()_ method to get a list of string values from the file, one string for each line of text: - -```python ->>> with open('sonnet29.txt') as sonnet_file: -... sonnet_file.readlines() -... -# [When, in disgrace with fortune and men's eyes,\n', -# ' I all alone beweep my outcast state,\n', -# And trouble deaf heaven with my bootless cries,\n', And -# look upon myself and curse my fate,'] -``` - -You can also iterate through the file line by line: - -```python ->>> with open('sonnet29.txt') as sonnet_file: -... for line in sonnet_file: -... print(line, end='') -... -# When, in disgrace with fortune and men's eyes, -# I all alone beweep my outcast state, -# And trouble deaf heaven with my bootless cries, -# And look upon myself and curse my fate, -``` - -## Writing to files - -```python ->>> with open('bacon.txt', 'w') as bacon_file: -... bacon_file.write('Hello world!\n') -... -# 13 - ->>> with open('bacon.txt', 'a') as bacon_file: -... bacon_file.write('Bacon is not a vegetable.') -... -# 25 - ->>> with open('bacon.txt') as bacon_file: -... content = bacon_file.read() -... ->>> print(content) -# Hello world! -# Bacon is not a vegetable. -``` diff --git a/PythonCheatsheats/regular-expressions.md b/PythonCheatsheats/regular-expressions.md deleted file mode 100644 index 7e382d1..0000000 --- a/PythonCheatsheats/regular-expressions.md +++ /dev/null @@ -1,393 +0,0 @@ ---- -title: Python Regular Expressions - Python Cheatsheet -description: A regular expression (shortened as regex) is a sequence of characters that specifies a search pattern in text and used by string-searching algorithms. ---- - - -Regular Expressions - - - - - Regular expressions - - - A regular expression (shortened as regex [...]) is a sequence of characters that specifies a search pattern in text. [...] used by string-searching algorithms for "find" or "find and replace" operations on strings, or for input validation. - - - -1. Import the regex module with `import re`. -2. Create a Regex object with the `re.compile()` function. (Remember to use a raw string.) -3. Pass the string you want to search into the Regex object’s `search()` method. This returns a `Match` object. -4. Call the Match object’s `group()` method to return a string of the actual matched text. - -All the regex functions in Python are in the re module: - -```python ->>> import re -``` - -## Regex symbols - -| Symbol | Matches | -| ------------------------ | ------------------------------------------------------ | -| `?` | zero or one of the preceding group. | -| `*` | zero or more of the preceding group. | -| `+` | one or more of the preceding group. | -| `{n}` | exactly n of the preceding group. | -| `{n,}` | n or more of the preceding group. | -| `{,m}` | 0 to m of the preceding group. | -| `{n,m}` | at least n and at most m of the preceding p. | -| `{n,m}?` or `*?` or `+?` | performs a non-greedy match of the preceding p. | -| `^spam` | means the string must begin with spam. | -| `spam$` | means the string must end with spam. | -| `.` | any character, except newline characters. | -| `\d`, `\w`, and `\s` | a digit, word, or space character, respectively. | -| `\D`, `\W`, and `\S` | anything except a digit, word, or space, respectively. | -| `[abc]` | any character between the brackets (such as a, b, ). | -| `[^abc]` | any character that isn’t between the brackets. | - -## Matching regex objects - -```python ->>> phone_num_regex = re.compile(r'\d\d\d-\d\d\d-\d\d\d\d') - ->>> mo = phone_num_regex.search('My number is 415-555-4242.') - ->>> print(f'Phone number found: {mo.group()}') -# Phone number found: 415-555-4242 -``` - -## Grouping with parentheses - -```python ->>> phone_num_regex = re.compile(r'(\d\d\d)-(\d\d\d-\d\d\d\d)') ->>> mo = phone_num_regex.search('My number is 415-555-4242.') - ->>> mo.group(1) -# '415' - ->>> mo.group(2) -# '555-4242' - ->>> mo.group(0) -# '415-555-4242' - ->>> mo.group() -# '415-555-4242' -``` - -To retrieve all the groups at once use the `groups()` method: - -```python ->>> mo.groups() -('415', '555-4242') - ->>> area_code, main_number = mo.groups() - ->>> print(area_code) -415 - ->>> print(main_number) -555-4242 -``` - -## Multiple groups with Pipe - -You can use the `|` character anywhere you want to match one of many expressions. - -```python ->>> hero_regex = re.compile (r'Batman|Tina Fey') - ->>> mo1 = hero_regex.search('Batman and Tina Fey.') ->>> mo1.group() -# 'Batman' - ->>> mo2 = hero_regex.search('Tina Fey and Batman.') ->>> mo2.group() -# 'Tina Fey' -``` - -You can also use the pipe to match one of several patterns as part of your regex: - -```python ->>> bat_regex = re.compile(r'Bat(man|mobile|copter|bat)') ->>> mo = bat_regex.search('Batmobile lost a wheel') - ->>> mo.group() -# 'Batmobile' - ->>> mo.group(1) -# 'mobile' -``` - -## Optional matching with the Question Mark - -The `?` character flags the group that precedes it as an optional part of the pattern. - -```python ->>> bat_regex = re.compile(r'Bat(wo)?man') - ->>> mo1 = bat_regex.search('The Adventures of Batman') ->>> mo1.group() -# 'Batman' - ->>> mo2 = bat_regex.search('The Adventures of Batwoman') ->>> mo2.group() -# 'Batwoman' -``` - -## Matching zero or more with the Star - -The `*` (star or asterisk) means “match zero or more”. The group that precedes the star can occur any number of times in the text. - -```python ->>> bat_regex = re.compile(r'Bat(wo)*man') ->>> mo1 = bat_regex.search('The Adventures of Batman') ->>> mo1.group() -'Batman' - ->>> mo2 = bat_regex.search('The Adventures of Batwoman') ->>> mo2.group() -'Batwoman' - ->>> mo3 = bat_regex.search('The Adventures of Batwowowowoman') ->>> mo3.group() -'Batwowowowoman' -``` - -## Matching one or more with the Plus - -The `+` (or plus) _means match one or more_. The group preceding a plus must appear at least once: - -```python ->>> bat_regex = re.compile(r'Bat(wo)+man') - ->>> mo1 = bat_regex.search('The Adventures of Batwoman') ->>> mo1.group() -# 'Batwoman' - ->>> mo2 = bat_regex.search('The Adventures of Batwowowowoman') ->>> mo2.group() -# 'Batwowowowoman' - ->>> mo3 = bat_regex.search('The Adventures of Batman') ->>> mo3 is None -# True -``` - -## Matching specific repetitions with Curly Brackets - -If you have a group that you want to repeat a specific number of times, follow the group in your regex with a number in curly brackets: - -```python ->>> ha_regex = re.compile(r'(Ha){3}') - ->>> mo1 = ha_regex.search('HaHaHa') ->>> mo1.group() -# 'HaHaHa' - ->>> mo2 = ha_regex.search('Ha') ->>> mo2 is None -# True -``` - -Instead of one number, you can specify a range with minimum and a maximum in between the curly brackets. For example, the regex (Ha){3,5} will match 'HaHaHa', 'HaHaHaHa', and 'HaHaHaHaHa'. - -```python ->>> ha_regex = re.compile(r'(Ha){2,3}') ->>> mo1 = ha_regex.search('HaHaHaHa') ->>> mo1.group() -# 'HaHaHa' -``` - -## Greedy and non-greedy matching - -Python’s regular expressions are greedy by default: in ambiguous situations they will match the longest string possible. The non-greedy version of the curly brackets, which matches the shortest string possible, has the closing curly bracket followed by a question mark. - -```python ->>> greedy_ha_regex = re.compile(r'(Ha){3,5}') - ->>> mo1 = greedy_ha_regex.search('HaHaHaHaHa') ->>> mo1.group() -# 'HaHaHaHaHa' - ->>> non_greedy_ha_regex = re.compile(r'(Ha){3,5}?') ->>> mo2 = non_greedy_ha_regex.search('HaHaHaHaHa') ->>> mo2.group() -# 'HaHaHa' -``` - -## The findall() method - -The `findall()` method will return the strings of every match in the searched string. - -```python ->>> phone_num_regex = re.compile(r'\d\d\d-\d\d\d-\d\d\d\d') # has no groups - ->>> phone_num_regex.findall('Cell: 415-555-9999 Work: 212-555-0000') -# ['415-555-9999', '212-555-0000'] -``` - -## Making your own character classes - -You can define your own character class using square brackets. For example, the character class _[aeiouAEIOU]_ will match any vowel, both lowercase and uppercase. - -```python ->>> vowel_regex = re.compile(r'[aeiouAEIOU]') ->>> vowel_regex.findall('Robocop eats baby food. BABY FOOD.') -# ['o', 'o', 'o', 'e', 'a', 'a', 'o', 'o', 'A', 'O', 'O'] -``` - -You can also include ranges of letters or numbers by using a hyphen. For example, the character class _[a-zA-Z0-9]_ will match all lowercase letters, uppercase letters, and numbers. - -By placing a caret character (`^`) just after the character class’s opening bracket, you can make a negative character class that will match all the characters that are not in the character class: - -```python ->>> consonant_regex = re.compile(r'[^aeiouAEIOU]') ->>> consonant_regex.findall('Robocop eats baby food. BABY FOOD.') -# ['R', 'b', 'c', 'p', ' ', 't', 's', ' ', 'b', 'b', 'y', ' ', 'f', 'd', '.', ' -# ', 'B', 'B', 'Y', ' ', 'F', 'D', '.'] -``` - -## The Caret and Dollar sign characters - -- You can also use the caret symbol `^` at the start of a regex to indicate that a match must occur at the beginning of the searched text. - -- Likewise, you can put a dollar sign `$` at the end of the regex to indicate the string must end with this regex pattern. - -- And you can use the `^` and `$` together to indicate that the entire string must match the regex. - -The `r'^Hello`' regular expression string matches strings that begin with 'Hello': - -```python ->>> begins_with_hello = re.compile(r'^Hello') ->>> begins_with_hello.search('Hello world!') -# <_sre.SRE_Match object; span=(0, 5), match='Hello'> - ->>> begins_with_hello.search('He said hello.') is None -# True -``` - -The `r'\d\$'` regular expression string matches strings that end with a numeric character from 0 to 9: - -```python ->>> whole_string_is_num = re.compile(r'^\d+$') - ->>> whole_string_is_num.search('1234567890') -# <_sre.SRE_Match object; span=(0, 10), match='1234567890'> - ->>> whole_string_is_num.search('12345xyz67890') is None -# True - ->>> whole_string_is_num.search('12 34567890') is None -# True -``` - -## The Wildcard character - -The `.` (or dot) character in a regular expression will match any character except for a newline: - -```python ->>> at_regex = re.compile(r'.at') - ->>> at_regex.findall('The cat in the hat sat on the flat mat.') -['cat', 'hat', 'sat', 'lat', 'mat'] -``` - -## Matching everything with Dot-Star - -```python ->>> name_regex = re.compile(r'First Name: (.*) Last Name: (.*)') - ->>> mo = name_regex.search('First Name: Al Last Name: Sweigart') ->>> mo.group(1) -# 'Al' - ->>> mo.group(2) -'Sweigart' -``` - -The `.*` uses greedy mode: It will always try to match as much text as possible. To match any and all text in a non-greedy fashion, use the dot, star, and question mark (`.*?`). The question mark tells Python to match in a non-greedy way: - -```python ->>> non_greedy_regex = re.compile(r'<.*?>') ->>> mo = non_greedy_regex.search(' for dinner.>') ->>> mo.group() -# '' - ->>> greedy_regex = re.compile(r'<.*>') ->>> mo = greedy_regex.search(' for dinner.>') ->>> mo.group() -# ' for dinner.>' -``` - -## Matching newlines with the Dot character - -The dot-star will match everything except a newline. By passing `re.DOTALL` as the second argument to `re.compile()`, you can make the dot character match all characters, including the newline character: - -```python ->>> no_newline_regex = re.compile('.*') ->>> no_newline_regex.search('Serve the public trust.\nProtect the innocent.\nUphold the law.').group() -# 'Serve the public trust.' - ->>> newline_regex = re.compile('.*', re.DOTALL) ->>> newline_regex.search('Serve the public trust.\nProtect the innocent.\nUphold the law.').group() -# 'Serve the public trust.\nProtect the innocent.\nUphold the law.' -``` - -## Case-Insensitive matching - -To make your regex case-insensitive, you can pass `re.IGNORECASE` or `re.I` as a second argument to `re.compile()`: - -```python ->>> robocop = re.compile(r'robocop', re.I) - ->>> robocop.search('Robocop is part man, part machine, all cop.').group() -# 'Robocop' - ->>> robocop.search('ROBOCOP protects the innocent.').group() -# 'ROBOCOP' - ->>> robocop.search('Al, why does your programming book talk about robocop so much?').group() -# 'robocop' -``` - -## Substituting strings with the sub() method - -The `sub()` method for Regex objects is passed two arguments: - -1. The first argument is a string to replace any matches. -1. The second is the string for the regular expression. - -The `sub()` method returns a string with the substitutions applied: - -```python ->>> names_regex = re.compile(r'Agent \w+') - ->>> names_regex.sub('CENSORED', 'Agent Alice gave the secret documents to Agent Bob.') -# 'CENSORED gave the secret documents to CENSORED.' -``` - -## Managing complex Regexes - -To tell the `re.compile()` function to ignore whitespace and comments inside the regular expression string, “verbose mode” can be enabled by passing the variable `re.VERBOSE` as the second argument to `re.compile()`. - -Now instead of a hard-to-read regular expression like this: - -```python -phone_regex = re.compile(r'((\d{3}|\(\d{3}\))?(\s|-|\.)?\d{3}(\s|-|\.)\d{4}(\s*(ext|x|ext.)\s*\d{2,5})?)') -``` - -you can spread the regular expression over multiple lines with comments like this: - -```python -phone_regex = re.compile(r'''( - (\d{3}|\(\d{3}\))? # area code - (\s|-|\.)? # separator - \d{3} # first 3 digits - (\s|-|\.) # separator - \d{4} # last 4 digits - (\s*(ext|x|ext.)\s*\d{2,5})? # extension - )''', re.VERBOSE) -``` diff --git a/PythonCheatsheats/sets.md b/PythonCheatsheats/sets.md deleted file mode 100644 index d0ab645..0000000 --- a/PythonCheatsheats/sets.md +++ /dev/null @@ -1,158 +0,0 @@ ---- -title: Python Sets - Python Cheatsheet -description: Python comes equipped with several built-in data types to help us organize our data. These structures include lists, dictionaries, tuples and sets. ---- - - -Python Sets - - -Python comes equipped with several built-in data types to help us organize our data. These structures include lists, dictionaries, tuples and **sets**. - - - - From the Python 3 documentation - - - A set is an unordered collection with no duplicate elements. Basic uses include membership testing and eliminating duplicate entries. - - - -Read Python Sets: What, Why and How for a more in-deep reference. - -## Initializing a set - -There are two ways to create sets: using curly braces `{}` and the built-in function `set()` - - - - Empty Sets - - - When creating set, be sure to not use empty curly braces {} or you will get an empty dictionary instead. - - - -```python ->>> s = {1, 2, 3} ->>> s = set([1, 2, 3]) - ->>> s = {} # this will create a dictionary instead of a set ->>> type(s) -# -``` - -## Unordered collections of unique elements - -A set automatically removes all the duplicate values. - -```python ->>> s = {1, 2, 3, 2, 3, 4} ->>> s -# {1, 2, 3, 4} -``` - -And as an unordered data type, they can't be indexed. - -```python ->>> s = {1, 2, 3} ->>> s[0] -# Traceback (most recent call last): -# File "", line 1, in -# TypeError: 'set' object does not support indexing -``` - -## set add and update - -Using the `add()` method we can add a single element to the set. - -```python ->>> s = {1, 2, 3} ->>> s.add(4) ->>> s -# {1, 2, 3, 4} -``` - -And with `update()`, multiple ones: - -```python ->>> s = {1, 2, 3} ->>> s.update([2, 3, 4, 5, 6]) ->>> s -# {1, 2, 3, 4, 5, 6} -``` - -## set remove and discard - -Both methods will remove an element from the set, but `remove()` will raise a `key error` if the value doesn't exist. - -```python ->>> s = {1, 2, 3} ->>> s.remove(3) ->>> s -# {1, 2} - ->>> s.remove(3) -# Traceback (most recent call last): -# File "", line 1, in -# KeyError: 3 -``` - -`discard()` won't raise any errors. - -```python ->>> s = {1, 2, 3} ->>> s.discard(3) ->>> s -# {1, 2} ->>> s.discard(3) -``` - -## set union - -`union()` or `|` will create a new set with all the elements from the sets provided. - -```python ->>> s1 = {1, 2, 3} ->>> s2 = {3, 4, 5} ->>> s1.union(s2) # or 's1 | s2' -# {1, 2, 3, 4, 5} -``` - -## set intersection - -`intersection()` or `&` will return a set with only the elements that are common to all of them. - -```python ->>> s1 = {1, 2, 3} ->>> s2 = {2, 3, 4} ->>> s3 = {3, 4, 5} ->>> s1.intersection(s2, s3) # or 's1 & s2 & s3' -# {3} -``` - -## set difference - -`difference()` or `-` will return only the elements that are unique to the first set (invoked set). - -```python ->>> s1 = {1, 2, 3} ->>> s2 = {2, 3, 4} - ->>> s1.difference(s2) # or 's1 - s2' -# {1} - ->>> s2.difference(s1) # or 's2 - s1' -# {4} -``` - -## set symmetric_difference - -`symmetric_difference()` or `^` will return all the elements that are not common between them. - -```python ->>> s1 = {1, 2, 3} ->>> s2 = {2, 3, 4} ->>> s1.symmetric_difference(s2) # or 's1 ^ s2' -# {1, 4} -``` diff --git a/PythonCheatsheats/setup-py.md b/PythonCheatsheats/setup-py.md deleted file mode 100644 index 19f7fb1..0000000 --- a/PythonCheatsheats/setup-py.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: Python Setup.py - Python Cheatsheet -description: The setup script is the centre of all activity in building, distributing, and installing modules using the Distutils. The main purpose of the setup script is to describe your module distribution to the Distutils, so that the various commands that operate on your modules do the right thing. ---- - - -Python setup.py - - - - - A 'controversial' opinion - - - Using `setup.py` to pack and distribute your python packages can be quite challenging every so often. Tools like Poetry make not only the packaging a lot easier, but also help you to manage your dependencies in a very convenient way. - - - -If you want more information about Poetry you can read the following articles: - -- Python projects with Poetry and VSCode. Part 1 -- Python projects with Poetry and VSCode. Part 2 -- Python projects with Poetry and VSCode. Part 3 - -## Introduction - -The setup script is the center of all activity in building, distributing, and installing modules using the Distutils. The main purpose of the setup script is to describe your module distribution to the Distutils, so that the various commands that operate on your modules do the right thing. - -The `setup.py` file is at the heart of a Python project. It describes all the metadata about your project. There are quite a few fields you can add to a project to give it a rich set of metadata describing the project. However, there are only three required fields: name, version, and packages. The name field must be unique if you wish to publish your package on the Python Package Index (PyPI). The version field keeps track of different releases of the project. The package's field describes where you’ve put the Python source code within your project. - -This allows you to easily install Python packages. Often it's enough to write: - -```bash -python setup.py install -``` - -and module will install itself. - -## Example - -Our initial setup.py will also include information about the license and will re-use the README.txt file for the long_description field. This will look like: - -```python -from distutils.core import setup -setup( - name='pythonCheatsheet', - version='0.1', - packages=['pipenv',], - license='MIT', - long_description=open('README.txt').read(), -) -``` - -Find more information visit the [official documentation](http://docs.python.org/3.11/install/index.html). diff --git a/PythonCheatsheats/string-formatting.md b/PythonCheatsheats/string-formatting.md deleted file mode 100644 index 7c8f426..0000000 --- a/PythonCheatsheats/string-formatting.md +++ /dev/null @@ -1,177 +0,0 @@ ---- -title: Python String Formatting - Python Cheatsheet -description: If your are using Python 3.6+, string f-strings are the recommended way to format strings. ---- - - -Python String Formatting - - - - - From the Python 3 documentation - - - The formatting operations described here (% operator) exhibit a variety of quirks that lead to a number of common errors [...]. Using the newer formatted string literals [...] helps avoid these errors. These alternatives also provide more powerful, flexible and extensible approaches to formatting text. - - - -## % operator - - - - Prefer String Literals - - - For new code, using str.format, or formatted string literals (Python 3.6+) over the % operator is strongly recommended. - - - -```python ->>> name = 'Pete' ->>> 'Hello %s' % name -# "Hello Pete" -``` - -We can use the `%d` format specifier to convert an int value to a string: - -```python ->>> num = 5 ->>> 'I have %d apples' % num -# "I have 5 apples" -``` - -## str.format - -Python 3 introduced a new way to do string formatting that was later back-ported to Python 2.7. This makes the syntax for string formatting more regular. - -```python ->>> name = 'John' ->>> age = 20 - ->>> "Hello I'm {}, my age is {}".format(name, age) -# "Hello I'm John, my age is 20" - ->>> "Hello I'm {0}, my age is {1}".format(name, age) -# "Hello I'm John, my age is 20" -``` - -## Formatted String Literals or f-Strings - -If your are using Python 3.6+, string `f-Strings` are the recommended way to format strings. - - - - From the Python 3 documentation - - - A formatted string literal or f-string is a string literal that is prefixed with `f` or `F`. These strings may contain replacement fields, which are expressions delimited by curly braces {}. While other string literals always have a constant value, formatted strings are really expressions evaluated at run time. - - - -```python ->>> name = 'Elizabeth' ->>> f'Hello {name}!' -# 'Hello Elizabeth!' -``` - -It is even possible to do inline arithmetic with it: - -```python ->>> a = 5 ->>> b = 10 ->>> f'Five plus ten is {a + b} and not {2 * (a + b)}.' -# 'Five plus ten is 15 and not 30.' -``` - -### Multiline f-Strings - -```python ->>> name = 'Robert' ->>> messages = 12 ->>> ( -... f'Hi, {name}. ' -... f'You have {messages} unread messages' -... ) -# 'Hi, Robert. You have 12 unread messages' -``` - -### The `=` specifier - -This will print the expression and its value: - -```python ->>> from datetime import datetime ->>> now = datetime.now().strftime("%b/%d/%Y - %H:%M:%S") ->>> f'date and time: {now=}' -# "date and time: now='Nov/14/2022 - 20:50:01'" -``` - -### Adding spaces or characters - -```python ->>> f"{name.upper() = :-^20}" -# 'name.upper() = -------ROBERT-------' ->>> ->>> f"{name.upper() = :^20}" -# 'name.upper() = ROBERT ' ->>> ->>> f"{name.upper() = :20}" -# 'name.upper() = ROBERT ' -``` - -## Formatting Digits - -Adding thousands separator - -```python ->>> a = 10000000 ->>> f"{a:,}" -# '10,000,000' -``` - -Rounding - -```python ->>> a = 3.1415926 ->>> f"{a:.2f}" -# '3.14' -``` - -Showing as Percentage - -```python ->>> a = 0.816562 ->>> f"{a:.2%}" -# '81.66%' -``` - -### Number formatting table - -| Number | Format | Output | description | -| ---------- | ------- | --------- | --------------------------------------------- | -| 3.1415926 | {:.2f} | 3.14 | Format float 2 decimal places | -| 3.1415926 | {:+.2f} | +3.14 | Format float 2 decimal places with sign | -| -1 | {:+.2f} | -1.00 | Format float 2 decimal places with sign | -| 2.71828 | {:.0f} | 3 | Format float with no decimal places | -| 4 | {:0>2d} | 04 | Pad number with zeros (left padding, width 2) | -| 4 | {:x<4d} | 4xxx | Pad number with x’s (right padding, width 4) | -| 10 | {:x<4d} | 10xx | Pad number with x’s (right padding, width 4) | -| 1000000 | {:,} | 1,000,000 | Number format with comma separator | -| 0.35 | {:.2%} | 35.00% | Format percentage | -| 1000000000 | {:.2e} | 1.00e+09 | Exponent notation | -| 11 | {:11d} | 11 | Right-aligned (default, width 10) | -| 11 | {:<11d} | 11 | Left-aligned (width 10) | -| 11 | {:^11d} | 11 | Center aligned (width 10) | - -## Template Strings - -A simpler and less powerful mechanism, but it is recommended when handling strings generated by users. Due to their reduced complexity, template strings are a safer choice. - -```python ->>> from string import Template ->>> name = 'Elizabeth' ->>> t = Template('Hey $name!') ->>> t.substitute(name=name) -# 'Hey Elizabeth!' -``` diff --git a/PythonCheatsheats/virtual-environments.md b/PythonCheatsheats/virtual-environments.md deleted file mode 100644 index 9270b4c..0000000 --- a/PythonCheatsheats/virtual-environments.md +++ /dev/null @@ -1,180 +0,0 @@ ---- -title: Python Virtual environments - Python Cheatsheet -description: The use of a Virtual Environment is to test python code in encapsulated environments and to also avoid filling the base Python installation with libraries we might use for only one project. ---- - - -Virtual Environment - - -The use of a Virtual Environment is to test python code in encapsulated environments, and to also avoid filling the base Python installation with libraries we might use for only one project. - -## virtualenv - -1. Install virtualenv - - pip install virtualenv - -1. Install virtualenvwrapper-win (Windows) - - pip install virtualenvwrapper-win - -Usage: - -1. Make a Virtual Environment named `HelloWorld` - - mkvirtualenv HelloWorld - - Anything we install now will be specific to this project. And available to the projects we connect to this environment. - -1. Set Project Directory - - To bind our virtualenv with our current working directory we simply enter: - - setprojectdir . - -1. Deactivate - - To move onto something else in the command line type `deactivate` to deactivate your environment. - - deactivate - - Notice how the parenthesis disappear. - -1. Workon - - Open up the command prompt and type `workon HelloWorld` to activate the environment and move into your root project folder - - workon HelloWorld - -## Poetry - - - - From Poetry website - - - Poetry is a tool for dependency management and packaging in Python. It allows you to declare the libraries your project depends on and it will manage (install/update) them for you. - - - -1. Install Poetry - - pip install --user poetry - -2. Create a new project - - poetry new my-project - - This will create a my-project directory: - - my-project - ├── pyproject.toml - ├── README.rst - ├── poetry_demo - │ └── __init__.py - └── tests - ├── __init__.py - └── test_poetry_demo.py - - The pyproject.toml file will orchestrate your project and its dependencies: - - [tool.poetry] - name = "my-project" - version = "0.1.0" - description = "" - authors = ["your name "] - - [tool.poetry.dependencies] - python = "*" - - [tool.poetry.dev-dependencies] - pytest = "^3.4" - -3. Packages - - To add dependencies to your project, you can specify them in the tool.poetry.dependencies section: - - [tool.poetry.dependencies] - pendulum = "^1.4" - - Also, instead of modifying the pyproject.toml file by hand, you can use the add command and it will automatically find a suitable version constraint. - - $ poetry add pendulum - - To install the dependencies listed in the pyproject.toml: - - poetry install - - To remove dependencies: - - poetry remove pendulum - -For more information, check the [documentation](https://poetry.eustace.io/docs/) or read here: - -- Python projects with Poetry and VSCode. Part 1 -- Python projects with Poetry and VSCode. Part 2 -- Python projects with Poetry and VSCode. Part 3 - -## Pipenv - - - - From Pipenv website - - - Pipenv is a tool that aims to bring the best of all packaging worlds (bundler, composer, npm, cargo, yarn, etc.) to the Python world. Windows is a first-class citizen, in our world. - - - -1. Install pipenv - - pip install pipenv - -2. Enter your Project directory and install the Packages for your project - - cd my_project - pipenv install - - Pipenv will install your package and create a Pipfile for you in your project’s directory. The Pipfile is used to track which dependencies your project needs in case you need to re-install them. - -3. Uninstall Packages - - pipenv uninstall - -4. Activate the Virtual Environment associated with your Python project - - pipenv shell - -5. Exit the Virtual Environment - - exit - -Find more information and a video in [docs.pipenv.org](https://docs.pipenv.org/). - -## Anaconda - - - - Anaconda is another popular tool to manage python packages. - - - Where packages, notebooks, projects and environments are shared. Your place for free public conda package hosting. - - - -Usage: - -1. Make a Virtual Environment - - conda create -n HelloWorld - -2. To use the Virtual Environment, activate it by: - - conda activate HelloWorld - - Anything installed now will be specific to the project HelloWorld - -3. Exit the Virtual Environment - - conda deactivate diff --git a/README.md b/README.md new file mode 100644 index 0000000..97b256b --- /dev/null +++ b/README.md @@ -0,0 +1,29 @@ +# Asset Processing Utility + +This tool streamlines the conversion of raw 3D asset source files from supplies (archives or folders) into a configurable library format. +Goals include automatically updating Assets in various DCC's on import as well - minimising end user workload. + +## Features + +* **Multithreaded Processing:** Leverages multiple CPU cores for parallel input processing, especially beneficial for large batches of inputs. +* **Token-based Output Naming:** Offers highly flexible and configurable output directory structures and file naming conventions using a token system (e.g., `[supplier]`, `[assetname]`, `[resolution]`). +* **Adjustable Resolutions:** Automatically resizes texture maps to multiple user-defined resolutions (e.g., 4K, 2K, 1K) during processing. +* **Adjustable image formats:** Allows configuration of prefered image output formats, allowing for overrides depending on various factors (prefer EXR over PNG for 16Bit, Prefer JPG over PNG for resolutions above certain thesholds). User can configure compression settings for each image format. +* **Gloss to Roughness Conversions:** Includes functionality to automatically invert Glossiness maps to Roughness maps, ensuring PBR workflow compatibility. +* **Configurable Channel Packing:** Supports customizable channel packing, allowing users to define rules for merging channels from different source maps into optimized packed textures (e.g., Normal + Roughness into a single NRMRGH map). +* **User-Friendly GUI:** Provides an intuitive Graphical User Interface for easy drag-and-drop input, interactive review and refinement of asset predictions, preset management, and process monitoring. +* **Asset Identification by LLM:** Supports using customizable Large Language Model (LLM) for intelligent and flexible identification of asset and file types. +* **Customizable Asset and File Types:** Allows for extensive customization of asset and file type definitions through configurable presets, enabling adaptation to various asset sources and conventions. +* **Blender Asset Catalog Post-processing Step:** Optionally runs Blender scripts after processing, providing automatic creation of PBR node groups and materials, and marking them as assets for use in Blender's Asset Browser without needing user interaction. + +## Core Workflow + +1. **Input Sources:** Drop asset archives or folders into the GUI. +2. **Identify Inputs:** The tool reads the directory contents and performs an initial identification of asset and file types using configurable presets or an customizable LLM. Users can then review and refine these predictions interactively in a preview table. This involves confirming or correcting asset names, suppliers, types, and individual file types using direct editing, keyboard shortcuts, or triggering re-interpretation for specific sources. +3. **Export to Library:** Once the inputs are correctly arranged and user verified, initiate the processing. The tool then automates tasks such as file classification, image resizings, format conversion, channel packing, and metadata generation, exporting the processed assets to a configurable output directory structure and naming convention. Most processing steps can be configured to fit user requirements. + +In addition to the interactive GUI, the tool also offers a Command-Line Interface (CLI) for batch processing and scripting, and a Directory Monitor for automated processing of files dropped into a watched folder. (Docker Support Planned) + +## Documentation + +For detailed information on installation, configuration, usage of the different interfaces, and the output structure, please refer to the [`Documentation`](Documentation/00_Overview.md) \ No newline at end of file diff --git a/Testfiles/Concrete24.zip b/TestFiles/BoucleChunky001.zip similarity index 51% rename from Testfiles/Concrete24.zip rename to TestFiles/BoucleChunky001.zip index cfa968b..cbd6d0b 100644 Binary files a/Testfiles/Concrete24.zip and b/TestFiles/BoucleChunky001.zip differ diff --git a/TestFiles/Test-BoucleChunky001.json b/TestFiles/Test-BoucleChunky001.json new file mode 100644 index 0000000..770d458 --- /dev/null +++ b/TestFiles/Test-BoucleChunky001.json @@ -0,0 +1,57 @@ +{ + "source_rules": [ + { + "input_path": "BoucleChunky001.zip", + "supplier_identifier": "Dinesen", + "preset_name": null, + "assets": [ + { + "asset_name": "BoucleChunky001", + "asset_type": "Surface", + "files": [ + { + "file_path": "BoucleChunky001_AO_1K_METALNESS.png", + "item_type": "MAP_AO", + "target_asset_name_override": "BoucleChunky001" + }, + { + "file_path": "BoucleChunky001_COL_1K_METALNESS.png", + "item_type": "MAP_COL", + "target_asset_name_override": "BoucleChunky001" + }, + { + "file_path": "BoucleChunky001_DISP16_1K_METALNESS.png", + "item_type": "MAP_DISP", + "target_asset_name_override": "BoucleChunky001" + }, + { + "file_path": "BoucleChunky001_DISP_1K_METALNESS.png", + "item_type": "MAP_DISP", + "target_asset_name_override": "BoucleChunky001" + }, + { + "file_path": "BoucleChunky001_Fabric.png", + "item_type": "EXTRA", + "target_asset_name_override": "BoucleChunky001" + }, + { + "file_path": "BoucleChunky001_METALNESS_1K_METALNESS.png", + "item_type": "MAP_METAL", + "target_asset_name_override": "BoucleChunky001" + }, + { + "file_path": "BoucleChunky001_NRM_1K_METALNESS.png", + "item_type": "MAP_NRM", + "target_asset_name_override": "BoucleChunky001" + }, + { + "file_path": "BoucleChunky001_ROUGHNESS_1K_METALNESS.png", + "item_type": "MAP_ROUGH", + "target_asset_name_override": "BoucleChunky001" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/Testfiles/BarkBrown013.zip b/Testfiles/BarkBrown013.zip deleted file mode 100644 index 3ae0ea3..0000000 Binary files a/Testfiles/BarkBrown013.zip and /dev/null differ diff --git a/Testfiles/CarpetMultiLevelLoopPileHerringbone001.zip b/Testfiles/CarpetMultiLevelLoopPileHerringbone001.zip deleted file mode 100644 index 78ed38a..0000000 Binary files a/Testfiles/CarpetMultiLevelLoopPileHerringbone001.zip and /dev/null differ diff --git a/Testfiles/Dinesen-Test.zip b/Testfiles/Dinesen-Test.zip deleted file mode 100644 index 415dcb8..0000000 Binary files a/Testfiles/Dinesen-Test.zip and /dev/null differ diff --git a/Testfiles/FabricDenim003.zip b/Testfiles/FabricDenim003.zip deleted file mode 100644 index 5752e20..0000000 Binary files a/Testfiles/FabricDenim003.zip and /dev/null differ diff --git a/Tickets/FEAT-008-gui-preview-refinement.md b/Tickets/FEAT-008-gui-preview-refinement.md deleted file mode 100644 index ec254e4..0000000 --- a/Tickets/FEAT-008-gui-preview-refinement.md +++ /dev/null @@ -1,139 +0,0 @@ ---- -ID: FEAT-008 -Type: Feature -Status: Backlog -Priority: Medium -Labels: [gui, enhancement] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: gui/main_window.py, gui/prediction_handler.py ---- - -# [FEAT-008]: Refine GUI Preview Table Display and Sorting - -## Description -Enhance the usability and clarity of the GUI's detailed file preview table. This involves improving the default sorting to group by asset and then status, removing redundant information, simplifying status text, and rearranging columns. - -## Current Behavior -* The preview table lists all files from all assets together without clear grouping. -* The table does not sort by status by default, or the sort order isn't optimized for clarity. -* The table includes a "Predicted Output" column which is considered redundant. -* Status text can be verbose (e.g., "unmatched extra", "[Unmatched Extra (Regex match: #####)]", "Ignored (Superseed by 16bit variant for ####)"). -* The "Original Path" column is not necessarily the rightmost column. - -## Desired Behavior / Goals -* The preview table should group files by the asset they belong to (e.g., based on the input ZIP/folder name or derived asset name). -* Within each asset group, the table should sort rows by 'Status' by default. -* The default secondary sort order for 'Status' should prioritize actionable or problematic statuses, grouping models with their maps: Error > (Mapped & Model) > Ignored > Extra. (Note: Assumes 'Unrecognised' files are displayed as 'Extra'). -* Within the 'Mapped & Model' group, files should be sorted alphabetically by original path or filename to keep related items together. -* The "Predicted Output" column should be removed from the table view. -* Status display text should be made more concise: - * "unmatched extra" should be displayed as "Extra". - * "[Unmatched Extra (Regex match: #####)]" should be displayed as "[Extra=#####]". - * "Ignored (Superseed by 16bit variant for ####)" should be displayed as "Superseeded by 16bit ####". - * Other statuses ("Mapped", "Model", "Error") should remain clear. -* The "Original Path" column should be positioned as the rightmost column in the table. - -## Implementation Notes (Optional) -* Modifications will likely be needed in `gui/main_window.py` (table view setup, column management, data population, sorting proxy model) and `gui/prediction_handler.py` (ensure prediction results include the source asset identifier). -* The table model (`QAbstractTableModel` or similar) needs to store the source asset identifier for each file row. -* Implement a custom multi-level sorting logic using `QSortFilterProxyModel`. - * The primary sort key will be the asset identifier. - * The secondary sort key will be the status, mapping 'Mapped' and 'Model' to the same priority level. - * The tertiary sort key will be the original path/filename. -* Update the column hiding/showing logic to remove the "Predicted Output" column. -* Implement string formatting or replacement logic for the status display text. -* Adjust the column order. -* Consider the performance implications of grouping and multi-level sorting, especially with a large number of assets/files. Add necessary optimizations (e.g., efficient data structures, potentially deferring sorting until needed). - -## Acceptance Criteria (Optional) -* [ ] When assets are added to the queue and detailed preview is active, the table visually groups files by their source asset. -* [ ] Within each asset group, the table automatically sorts by the Status column according to the specified multi-level order. -* [ ] Clicking the Status header cycles through ascending/descending sort based on the custom multi-level order: Asset > Status (Error > (Mapped & Model) > Ignored > Extra) > Filename. -* [ ] The "Predicted Output" column is not present in the detailed preview table. -* [ ] Status text for relevant items displays concisely as: "Extra", "[Extra=#####]", "Superseeded by 16bit ####". -* [ ] The "Original Path" column is visually the last column on the right side of the table. ---- - -## Implementation Plan (Generated 2025-04-22) - -This plan outlines the steps to implement the GUI preview refinements described in this ticket. - -**Goal:** Enhance the GUI's detailed file preview table for better usability and clarity by implementing grouping, refined sorting, simplified status text, and adjusted column layout. - -**Affected Files:** - -* `gui/main_window.py`: Handles table view setup, data population trigger, and column management. -* `gui/preview_table_model.py`: Contains the table model (`PreviewTableModel`) and the sorting proxy model (`PreviewSortFilterProxyModel`). - -**Plan Steps:** - -1. **Correct Data Population in `main_window.py`:** - * **Action:** Modify the `on_prediction_results_ready` slot (around line 893). - * **Change:** Update the code to call `self.preview_model.set_data(results)` instead of populating the old `self.preview_table`. - * **Remove:** Delete code manually setting headers, rows, and items on `self.preview_table`. - * **Keep:** Retain the initial setup of `self.preview_table_view` (lines 400-424). - -2. **Implement Status Text Simplification in `preview_table_model.py`:** - * **Action:** Modify the `data()` method within the `PreviewTableModel` class (around line 56). - * **Change:** Inside the `if role == Qt.ItemDataRole.DisplayRole:` block for `COL_STATUS`, add logic to transform the raw status string: - * "Unmatched Extra" -> "Extra" - * "[Unmatched Extra (Regex match: PATTERN)]" -> "[Extra=PATTERN]" - * "Ignored (Superseed by 16bit variant for FILENAME)" -> "Superseeded by 16bit FILENAME" - * Otherwise, return original status. - * **Note:** Ensure `ROLE_RAW_STATUS` still returns the original status for sorting. - -3. **Refine Sorting Logic in `preview_table_model.py`:** - * **Action:** Modify the `lessThan` method within the `PreviewSortFilterProxyModel` class (around line 166). - * **Change:** Adjust the "Level 2: Sort by Status" logic to use a priority mapping where "Mapped" and "Model" have the same priority index, causing the sort to fall through to Level 3 (Path) for items within this group. - ```python - # Example Priority Mapping - STATUS_PRIORITY = { - "Error": 0, - "Mapped": 1, - "Model": 1, - "Ignored": 2, - "Extra": 3, - "Unrecognised": 3, - "Unmatched Extra": 3, - "[No Status]": 99 - } - # ... comparison logic using STATUS_PRIORITY ... - ``` - * **Remove/Update:** Replace the old `STATUS_ORDER` list with this priority dictionary logic. - -4. **Adjust Column Order in `main_window.py`:** - * **Action:** Modify the `setup_main_panel_ui` method (around lines 404-414). - * **Change:** After setting up `self.preview_table_view`, explicitly move the "Original Path" column to the last visual position using `header.moveSection()`. - * **Verify:** Ensure the "Predicted Output" column remains hidden. - -**Visual Plan (Mermaid):** - -```mermaid -graph TD - A[Start FEAT-008 Implementation] --> B(Refactor `main_window.py::on_prediction_results_ready`); - B --> C{Use `self.preview_model.set_data()`?}; - C -- Yes --> D(Remove manual `QTableWidget` population); - C -- No --> E[ERROR: Incorrect data flow]; - D --> F(Modify `preview_table_model.py::PreviewTableModel::data()`); - F --> G{Implement Status Text Simplification?}; - G -- Yes --> H(Add formatting logic for DisplayRole); - G -- No --> I[ERROR: Status text not simplified]; - H --> J(Modify `preview_table_model.py::PreviewSortFilterProxyModel::lessThan()`); - J --> K{Implement Mapped/Model Grouping & Custom Sort Order?}; - K -- Yes --> L(Use priority mapping for status comparison); - K -- No --> M[ERROR: Sorting incorrect]; - L --> N(Modify `main_window.py::setup_main_panel_ui()`); - N --> O{Move 'Original Path' Column to End?}; - O -- Yes --> P(Use `header.moveSection()`); - O -- No --> Q[ERROR: Column order incorrect]; - P --> R(Verify All Acceptance Criteria); - R --> S[End FEAT-008 Implementation]; - - subgraph main_window.py Modifications - B; D; N; P; - end - - subgraph preview_table_model.py Modifications - F; H; J; L; - end \ No newline at end of file diff --git a/Tickets/FEAT-009-gui-unify-preset-selection.md b/Tickets/FEAT-009-gui-unify-preset-selection.md deleted file mode 100644 index 8229e87..0000000 --- a/Tickets/FEAT-009-gui-unify-preset-selection.md +++ /dev/null @@ -1,76 +0,0 @@ -# FEAT-009: GUI - Unify Preset Editor Selection and Processing Preset - -**Status:** Resolved -**Priority:** Medium -**Assigned:** TBD -**Reporter:** Roo (Architect Mode) -**Date:** 2025-04-22 - -## Description - -This ticket proposes a GUI modification to streamline the preset handling workflow by unifying the preset selection mechanism. Currently, the preset selected for editing in the left panel can be different from the preset selected for processing/previewing in the right panel. This change aims to eliminate this duality, making the preset loaded in the editor the single source of truth for both editing and processing actions. - -## Current Behavior - -1. **Preset Editor Panel (Left):** Users select a preset from the 'Preset List'. This action loads the selected preset's details into the 'Preset Editor Tabs' for viewing or modification (See `readme.md` lines 257-260). -2. **Processing Panel (Right):** A separate 'Preset Selector' dropdown exists. Users select a preset from this dropdown *specifically* for processing the queued assets and generating the file preview in the 'Preview Table' (See `readme.md` lines 262, 266). -3. This allows for a scenario where the preset being edited is different from the preset used for previewing and processing, potentially causing confusion. - -## Proposed Behavior - -1. **Unified Selection:** Selecting a preset from the 'Preset List' in the left panel will immediately make it the active preset for *both* editing (loading its details into the 'Preset Editor Tabs') *and* processing/previewing. -2. **Dropdown Removal:** The separate 'Preset Selector' dropdown in the right 'Processing Panel' will be removed. -3. **Dynamic Updates:** The 'Preview Table' in the right panel will dynamically update based on the preset selected in the left panel's 'Preset List'. -4. **Processing Logic:** The 'Start Processing' action will use the currently active preset (selected from the left panel's list). - -## Rationale - -* **Improved User Experience:** Simplifies the UI by removing a redundant control and creates a more intuitive workflow. -* **Reduced Complexity:** Eliminates the need to manage two separate preset states (editing vs. processing). -* **Consistency:** Ensures that the preview and processing actions always reflect the preset currently being viewed/edited. - -## Implementation Notes/Tasks - -* **UI Modification (`gui/main_window.py`):** - * Remove the `QComboBox` (or similar widget) used for the 'Preset Selector' from the right 'Processing Panel' layout. -* **Signal/Slot Connection (`gui/main_window.py`):** - * Ensure the signal emitted when a preset is selected in the left panel's 'Preset List' (`QListWidget` or similar) is connected to slots responsible for: - * Loading the preset into the editor tabs. - * Updating the application's state to reflect the new active preset for processing. - * Triggering an update of the 'Preview Table'. -* **State Management:** - * Modify how the currently active preset for processing is stored and accessed. It should now directly reference the preset selected in the left panel list. -* **Handler Updates (`gui/prediction_handler.py`, `gui/processing_handler.py`):** - * Ensure these handlers correctly retrieve and use the single, unified active preset state when generating previews or initiating processing. - -## Acceptance Criteria - -1. The 'Preset Selector' dropdown is no longer visible in the right 'Processing Panel'. -2. Selecting a preset in the 'Preset List' (left panel) successfully loads its details into the 'Preset Editor Tabs'. -3. Selecting a preset in the 'Preset List' (left panel) updates the 'Preview Table' (right panel) to reflect the rules of the newly selected preset. -4. Initiating processing via the 'Start Processing' button uses the preset currently selected in the 'Preset List' (left panel). -5. The application remains stable and performs processing correctly with the unified preset selection. - -## Workflow Diagram (Mermaid) - -```mermaid -graph TD - subgraph "Current GUI" - A[Preset List (Left Panel)] -- Selects --> B(Preset Editor Tabs); - C[Preset Selector Dropdown (Right Panel)] -- Selects --> D(Active Processing Preset); - D -- Affects --> E(Preview Table); - D -- Used by --> F(Processing Logic); - end - - subgraph "Proposed GUI" - G[Preset List (Left Panel)] -- Selects --> H(Preset Editor Tabs); - G -- Also Sets --> I(Active Preset for Editing & Processing); - I -- Affects --> J(Preview Table); - I -- Used by --> K(Processing Logic); - L(Preset Selector Dropdown Removed); - end - - A --> G; - B --> H; - E --> J; - F --> K; \ No newline at end of file diff --git a/Tickets/FEAT-GUI-NoDefaultPreset.md b/Tickets/FEAT-GUI-NoDefaultPreset.md deleted file mode 100644 index 0a333d2..0000000 --- a/Tickets/FEAT-GUI-NoDefaultPreset.md +++ /dev/null @@ -1,49 +0,0 @@ -# FEAT-GUI-NoDefaultPreset: Prevent Default Preset Selection in GUI - -## Objective - -Modify the Graphical User Interface (GUI) to prevent any preset from being selected by default on application startup. Instead, the GUI should prompt the user to explicitly select a preset from the list before displaying the detailed file preview. This aims to avoid accidental processing with an unintended preset. - -## Problem Description - -Currently, when the GUI application starts, a preset from the available list is automatically selected. This can lead to user confusion if they are not aware of this behavior and proceed to add assets and process them using a preset they did not intend to use. The preview table also populates automatically based on this default selection, which might not be desired until a conscious preset choice is made. - -## Failed Attempts - -1. **Attempt 1: Remove Default Selection Logic and Add Placeholder Text:** - * **Approach:** Removed code that explicitly set a default selected item in the preset list during initialization. Added a `QLabel` with placeholder text ("Please select a preset...") to the preview area and attempted to use `setPlaceholderText` on the `QTableView` (this was incorrect as `QTableView` does not have this method). Managed visibility of the placeholder label and table view. - * **Result:** The `setPlaceholderText` call failed with an `AttributeError`. Even after removing the erroneous line and adding a dedicated `QLabel`, a preset was still being selected automatically in the list on startup, and the placeholder was not consistently shown. This suggested that simply populating the `QListWidget` might implicitly trigger a selection. - -2. **Attempt 2: Explicitly Clear Selection and Refine Visibility Logic:** - * **Approach:** Added explicit calls (`setCurrentItem(None)`, `clearSelection()`) after populating the preset list to ensure no item was selected. Refined the visibility logic for the placeholder label and table view in `_clear_editor` and `_load_selected_preset_for_editing`. Added logging to track selection and visibility changes. - * **Result:** Despite explicitly clearing the selection, testing indicated that a preset was still being selected on startup, and the placeholder was not consistently displayed. This reinforced the suspicion that the `QListWidget`'s behavior upon population was automatically triggering a selection and the associated signal. - -## Proposed Plan: Implement "-- Select a Preset --" Placeholder Item - -This approach makes the "no selection" state an explicit, selectable item in the preset list, giving us more direct control over the initial state and subsequent behavior. - -1. **Modify `populate_presets` Method:** - * Add a `QListWidgetItem` with the text "-- Select a Preset --" at the very beginning of the list (index 0) after clearing the list but before adding actual preset items. - * Store a special, non-`Path` value (e.g., `None` or a unique string like `"__PLACEHOLDER__"`) in this placeholder item's `UserRole` data to distinguish it from real presets. - * After adding all real preset items, explicitly set the current item to this placeholder item using `self.editor_preset_list.setCurrentRow(0)`. - -2. **Modify `_load_selected_preset_for_editing` Method (Slot for `currentItemChanged`):** - * At the beginning of the method, check if the `current_item` is the placeholder item by examining its `UserRole` data. - * If the placeholder item is selected: - * Call `self._clear_editor()` to reset all editor fields. - * Call `self.preview_model.clear_data()` to ensure the preview table model is empty. - * Explicitly set `self.preview_placeholder_label.setVisible(True)` and `self.preview_table_view.setVisible(False)`. - * Return from the method without proceeding to load a preset or call `update_preview`. - * If a real preset item is selected, proceed with the existing logic: get the `Path` from the item's data, call `_load_preset_for_editing(preset_path)`, call `self.update_preview()`, set `self.preview_placeholder_label.setVisible(False)` and `self.preview_table_view.setVisible(True)`. - -3. **Modify `start_processing` Method:** - * Before initiating the processing, check if the currently selected item in `editor_preset_list` is the placeholder item. - * If the placeholder item is selected, display a warning message to the user (e.g., "Please select a valid preset before processing.") using the status bar and return from the method. - * If a real preset is selected, proceed with the existing processing logic. - -4. **Modify `update_preview` Method:** - * Add a check at the beginning of the method. Get the `current_item` from `editor_preset_list`. If it is the placeholder item, clear the preview model (`self.preview_model.clear_data()`) and return immediately. This prevents the prediction handler from running when no valid preset is selected. - -## Next Steps - -Implement the proposed plan by modifying the specified methods in `gui/main_window.py`. Test the GUI on startup and when selecting different items in the preset list to ensure the desired behavior is achieved. \ No newline at end of file diff --git a/Tickets/ISSUE-004-color-channel-swapping.md b/Tickets/ISSUE-004-color-channel-swapping.md deleted file mode 100644 index ac851e3..0000000 --- a/Tickets/ISSUE-004-color-channel-swapping.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -ID: ISSUE-004 -Type: Issue -Status: Resolved -Priority: High -Labels: [bug, core, image-processing] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: ---- - -# [ISSUE-004]: Color channel swapping in image processing (Normal Maps, Stats) - -## Description -There appears to be a general issue with how color channels (specifically Red and Blue) are handled in several parts of the image processing pipeline. This has been observed in normal map channel packing and potentially in the calculation of image statistics per channel, where the Red and Blue channels seem to be swapped relative to their intended order. - -## Current Behavior -When processing images where individual color channels are accessed or manipulated (e.g., during normal map channel packing or calculating per-channel image statistics), the Red and Blue channels appear to be swapped. For example, in normal map packing, the channel intended for Blue might contain Red data, and vice versa, while Green remains correct. This suggests a consistent R/B channel inversion issue in the underlying image processing logic. - -## Desired Behavior / Goals -The tool should correctly handle color channels according to standard RGB ordering in all image processing operations, including channel packing and image statistics calculation. The Red, Green, and Blue channels should consistently correspond to their intended data. - -## Implementation Notes (Optional) -(This likely points to an issue in the image loading, channel splitting, merging, or processing functions, possibly related to the library used for image manipulation (e.g., OpenCV). Need to investigate how channels are accessed and ordered in relevant code sections like `_process_maps`, `_merge_maps`, and image statistics calculation.) - -## Acceptance Criteria (Optional) -* [ ] Process an asset with a normal map and a map requiring channel packing (e.g., NRMRGH). -* [ ] Verify that the channels in the output normal map and packed map are in the correct R, G, B order. -* [ ] Verify that the calculated image statistics (Min/Max/Mean) for Red and Blue channels accurately reflect the data in those specific channels, not the swapped data. -## Resolution -The root cause was identified as a mismatch between OpenCV's default BGR channel order upon image loading (`cv2.imread`) and subsequent code assuming an RGB channel order, particularly in channel indexing during map merging (`_merge_maps`) and potentially in statistics calculation (`_calculate_image_stats`). - -The fix involved the following changes in `asset_processor.py`: -1. **BGR to RGB Conversion:** Immediately after loading a 3-channel image using `cv2.imread` in the `_process_maps` function, the image is converted to RGB color space using `img_processed = cv2.cvtColor(img_loaded, cv2.COLOR_BGR2RGB)`. Grayscale or 4-channel images are handled appropriately without conversion. -2. **Updated Channel Indexing:** The channel indexing logic within the `_merge_maps` function was updated to reflect the RGB order (Red = index 0, Green = index 1, Blue = index 2) when extracting channels from the now-RGB source images for merging. -3. **Statistics Assumption Update:** Comments in `_calculate_image_stats` were updated to reflect that the input data is now expected in RGB order. - -This ensures consistent RGB channel ordering throughout the processing pipeline after the initial load. \ No newline at end of file diff --git a/Tickets/ISSUE-010-color-channel-regression.md b/Tickets/ISSUE-010-color-channel-regression.md deleted file mode 100644 index cfae372..0000000 --- a/Tickets/ISSUE-010-color-channel-regression.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -ID: ISSUE-010 -Type: Issue -Status: Resolved -Priority: High -Labels: [bug, core, image-processing, regression, resolved] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: #ISSUE-004, asset_processor.py ---- - -# [ISSUE-010]: Color Channel Swapping Regression for RGB/Merged Maps after ISSUE-004 Fix - -## Description -The resolution implemented for `ISSUE-004` successfully corrected the BGR/RGB channel handling for image statistics calculation and potentially normal map packing. However, this fix appears to have introduced a regression where standard RGB color maps and maps generated through merging operations are now being saved with swapped Blue and Red channels (BGR order) instead of the expected RGB order. - -## Current Behavior -- Standard RGB texture maps (e.g., Diffuse, Albedo) loaded and processed are saved with BGR channel order. -- Texture maps created by merging channels (e.g., ORM, NRMRGH) are saved with BGR channel order for their color components. -- Image statistics (`metadata.json`) are calculated correctly based on RGB data. -- Grayscale maps are handled correctly. -- RGBA maps are assumed to be handled correctly (needs verification). - -## Desired Behavior / Goals -- All processed color texture maps (both original RGB and merged maps) should be saved with the standard RGB channel order. -- Image statistics should continue to be calculated correctly based on RGB data. -- Grayscale and RGBA maps should retain their correct handling. -- The fix should not reintroduce the problems solved by `ISSUE-004`. - -## Implementation Notes (Optional) -The issue likely stems from the universal application of `cv2.cvtColor(img_loaded, cv2.COLOR_BGR2RGB)` in `_process_maps` introduced in the `ISSUE-004` fix. While this ensures consistent RGB data for internal operations like statistics and merging (which now expects RGB), the final saving step (`cv2.imwrite`) might implicitly expect BGR data for color images, or the conversion needs to be selectively undone before saving certain map types. - -Investigation needed in `asset_processor.py`: -- Review `_process_maps`: Where is the BGR->RGB conversion happening? Is it applied to all 3-channel images? -- Review `_process_maps` saving logic: How are different map types saved? Does `cv2.imwrite` expect RGB or BGR? -- Review `_merge_maps`: How are channels combined? Does the saving logic here also need adjustment? -- Determine if the BGR->RGB conversion should be conditional or if a final RGB->BGR conversion is needed before saving specific map types. - -## Acceptance Criteria (Optional) -* [x] Process an asset containing standard RGB maps (e.g., Color/Albedo). Verify the output map has correct RGB channel order. -* [x] Process an asset requiring map merging (e.g., ORM from Roughness, Metallic, AO). Verify the output merged map has correct RGB(A) channel order. -* [x] Verify image statistics in `metadata.json` remain correct (reflecting RGB values). -* [x] Verify grayscale maps are processed and saved correctly. -* [x] Verify normal maps are processed and saved correctly (as per ISSUE-004 fix). -* [ ] (Optional) Verify RGBA maps are processed and saved correctly. ---- - -## Resolution -The issue was resolved by implementing conditional RGB to BGR conversion immediately before saving images using `cv2.imwrite` within the `_process_maps` and `_merge_maps` methods in `asset_processor.py`. - -The logic checks if the image is 3-channel and if the target output format is *not* EXR. If both conditions are true, the image data is converted from the internal RGB representation back to BGR, which is the expected channel order for `cv2.imwrite` when saving formats like PNG, JPG, and TIF. - -This approach ensures that color maps are saved with the correct channel order in standard formats while leaving EXR files (which handle RGB correctly) and grayscale/single-channel images unaffected. It also preserves the internal RGB representation used for accurate image statistics calculation and channel merging, thus not reintroducing the issues fixed by `ISSUE-004`. - -## Implementation Plan (Generated 2025-04-22) - -**Goal:** Correct the BGR/RGB channel order regression for saved color maps (introduced by the `ISSUE-004` fix) while maintaining the correct handling for statistics, grayscale maps, and EXR files. - -**Core Idea:** Convert the image data back from RGB to BGR *just before* saving with `cv2.imwrite`, but only for 3-channel images and formats where this conversion is necessary (e.g., PNG, JPG, TIF, but *not* EXR). - -**Plan Steps:** - -1. **Modify `_process_maps` Saving Logic (`asset_processor.py`):** - * Before the primary `cv2.imwrite` call (around line 1182) and the fallback call (around line 1206), add logic to check if the image is 3-channel and the output format is *not* 'exr'. If true, convert the image from RGB to BGR using `cv2.cvtColor` and add a debug log message. -2. **Modify `_merge_maps` Saving Logic (`asset_processor.py`):** - * Apply the same conditional RGB to BGR conversion logic before the primary `cv2.imwrite` call (around line 1505) and the fallback call (around line 1531), including a debug log message. -3. **Verification Strategy:** - * Test with various 3-channel color maps (Albedo, Emission, etc.) and merged maps (NRMRGH, etc.) saved as PNG/JPG/TIF. - * Verify correct RGB order in outputs. - * Verify grayscale, EXR, and statistics calculation remain correct. - -**Mermaid Diagram:** - -```mermaid -graph TD - subgraph "_process_maps / _merge_maps Saving Step" - A[Prepare Final Image Data (in RGB)] --> B{Is it 3-Channel?}; - B -- Yes --> C{Output Format != 'exr'?}; - B -- No --> E[Save Image using cv2.imwrite]; - C -- Yes --> D(Convert Image RGB -> BGR); - C -- No --> E; - D --> E; - end \ No newline at end of file diff --git a/Tickets/ISSUE-011-blender-nodegroup-empty-assets.md b/Tickets/ISSUE-011-blender-nodegroup-empty-assets.md deleted file mode 100644 index ecc3cdc..0000000 --- a/Tickets/ISSUE-011-blender-nodegroup-empty-assets.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -ID: ISSUE-011 -Type: Issue -Status: Backlog -Priority: Medium -Labels: [blender, bug] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: ---- - -# [ISSUE-011]: Blender nodegroup script creates empty assets for skipped items - -## Description -The Blender nodegroup creation script (`blenderscripts/create_nodegroups.py`) incorrectly generates empty asset entries in the target .blend file for assets that were skipped during the main processing pipeline. This occurs even though the main asset processor correctly identifies and skips these assets based on the overwrite flag. - -## Current Behavior -When running the asset processor with the `--overwrite` flag set to false, if an asset's output directory and metadata.json already exist, the main processing pipeline correctly skips processing that asset. However, the subsequent Blender nodegroup creation script still attempts to create a nodegroup for this skipped asset, resulting in an empty or incomplete asset entry in the target .blend file. - -## Desired Behavior / Goals -The Blender nodegroup creation script should only attempt to create nodegroups for assets that were *actually processed* by the main asset processor, not those that were skipped. It should check if the asset was processed successfully before attempting nodegroup creation. - -## Implementation Notes (Optional) -The `main.py` script, which orchestrates the processing and calls the Blender scripts, needs to pass information about which assets were successfully processed to the Blender nodegroup script. The Blender script should then filter its operations based on this information. - -## Acceptance Criteria (Optional) -* [ ] Running the asset processor with `--overwrite` false on inputs that already have processed outputs should result in the main processing skipping the asset. -* [ ] The Blender nodegroup script, when run after a skipped processing run, should *not* create or update a nodegroup for the skipped asset. -* [ ] Only assets that were fully processed should have corresponding nodegroups created/updated by the Blender script. \ No newline at end of file diff --git a/Tickets/ISSUE-012-mask-map-alpha-extraction.md b/Tickets/ISSUE-012-mask-map-alpha-extraction.md deleted file mode 100644 index 534b1be..0000000 --- a/Tickets/ISSUE-012-mask-map-alpha-extraction.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -ID: ISSUE-012 -Type: Issue -Status: Backlog -Priority: High -Labels: [core, bug, image processing] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: ---- - -# [ISSUE-012]: MASK map processing fails to extract alpha channel from RGBA images - -## Description -When processing texture sets that include MASK maps provided as RGBA images, the asset processor is expected to extract the alpha channel to represent the mask. However, the current implementation appears to be converting the RGBA image to grayscale instead of isolating the alpha channel. This results in incorrect MASK maps in the processed output. - -## Current Behavior -The asset processor, when encountering an RGBA image classified as a MASK map, converts the image to grayscale. The alpha channel information is lost or ignored during this process. - -## Desired Behavior / Goals -For RGBA images classified as MASK maps, the asset processor should extract the alpha channel and use it as the content for the output MASK map. The resulting MASK map should be a single-channel (grayscale) image representing the transparency or mask information from the original alpha channel. - -## Implementation Notes (Optional) -The image processing logic within `asset_processor.py` (likely in the `_process_maps` method or a related helper function) needs to be updated to specifically handle RGBA input images for MASK map types. It should identify the alpha channel and save it as a single-channel output image. Libraries like OpenCV or Pillow should provide functionality for accessing individual channels. - -## Acceptance Criteria (Optional) -* [ ] Process an asset with an RGBA image designated as a MASK map according to the preset. -* [ ] Verify that the output MASK map is a single-channel grayscale image. -* [ ] Confirm that the grayscale values in the output MASK map correspond to the alpha values of the original RGBA input image. \ No newline at end of file diff --git a/Tickets/ISSUE-012-plan.md b/Tickets/ISSUE-012-plan.md deleted file mode 100644 index 26a55ba..0000000 --- a/Tickets/ISSUE-012-plan.md +++ /dev/null @@ -1,53 +0,0 @@ -# Plan for ISSUE-012: MASK map processing fails to extract alpha channel from RGBA images - -## Issue Description -When processing texture sets that include MASK maps provided as RGBA images, the asset processor is expected to extract the alpha channel to represent the mask. However, the current implementation appears to be converting the RGBA image to grayscale instead of isolating the alpha channel, resulting in incorrect MASK maps. This issue affects all RGBA images classified as MASK, including plain 'MASK' and MASK variants (e.g., MASK-1). - -## Analysis -Based on the code in `asset_processor.py`, specifically the `_load_and_transform_source` method, the issue likely stems from the order of operations. The current logic converts 4-channel BGRA images to 3-channel RGB *before* checking specifically for MASK types and attempting to extract the alpha channel. This means the alpha channel is lost before the code gets a chance to extract it. The condition `if map_type.upper() == 'MASK':` is too strict and does not cover MASK variants. - -## Detailed Plan - -1. **Analyze `_load_and_transform_source`:** Re-examine the `_load_and_transform_source` method in `asset_processor.py` to confirm the exact sequence of image loading, color space conversions, and MASK-specific handling. (Completed during initial analysis). -2. **Modify MASK Handling Condition:** Change the current condition `if map_type.upper() == 'MASK':` to use the `_get_base_map_type` helper function, so it correctly identifies all MASK variants (e.g., 'MASK', 'MASK-1', 'MASK-2') and applies the special handling logic to them. The condition will become `if _get_base_map_type(map_type) == 'MASK':`. -3. **Reorder and Refine Logic:** - * The image will still be loaded with `cv2.IMREAD_UNCHANGED` for MASK types to ensure the alpha channel is initially present. - * Immediately after loading, and *before* any general color space conversions (like BGR->RGB), check if the base map type is 'MASK' and if the loaded image is 4-channel (RGBA/BGRA). - * If both conditions are true, extract the alpha channel (`img_loaded[:, :, 3]`) and use this single-channel data for subsequent processing steps (`img_prepared`). - * If the base map type is 'MASK' but the loaded image is 3-channel (RGB/BGR), convert it to grayscale (`cv2.cvtColor(img_loaded, cv2.COLOR_BGR2GRAY)`) and use this for `img_prepared`. - * If the base map type is 'MASK' and the loaded image is already 1-channel (grayscale), keep it as is. - * If the base map type is *not* 'MASK', the existing BGR->RGB conversion logic for 3/4 channel images will be applied as before. -4. **Review Subsequent Steps:** Verify that the rest of the `_load_and_transform_source` method (Gloss->Rough inversion, resizing, dtype conversion) correctly handles the single-channel image data that will now be produced for RGBA MASK inputs. -5. **Testing:** Use the acceptance criteria outlined in `ISSUE-012` to ensure the fix works correctly. - -## Proposed Code Logic Flow - -```mermaid -graph TD - A[Load Image (IMREAD_UNCHANGED for MASK)] --> B{Loading Successful?}; - B -- No --> C[Handle Load Error]; - B -- Yes --> D[Get Original Dtype & Shape]; - D --> E{Base Map Type is MASK?}; - E -- Yes --> F{Loaded Image is 4-Channel?}; - F -- Yes --> G[Extract Alpha Channel]; - F -- No --> H{Loaded Image is 3-Channel?}; - H -- Yes --> I[Convert BGR/RGB to Grayscale]; - H -- No --> J[Keep as is (Assume Grayscale)]; - G --> K[Set img_prepared to Mask Data]; - I --> K; - J --> K; - E -- No --> L{Loaded Image is 3 or 4-Channel?}; - L -- Yes --> M[Convert BGR/BGRA to RGB]; - L -- No --> N[Keep as is]; - M --> O[Set img_prepared to RGB Data]; - N --> O; - K --> P[Proceed with other transformations (Gloss, Resize, etc.)]; - O --> P; - P --> Q[Cache Result & Return]; - C --> Q; -``` - -## Acceptance Criteria (from ISSUE-012) -* [ ] Process an asset with an RGBA image designated as a MASK map according to the preset. -* [ ] Verify that the output MASK map is a single-channel grayscale image. -* [ ] Confirm that the grayscale values in the output MASK map correspond to the alpha values of the original RGBA input image. \ No newline at end of file diff --git a/Tickets/ISSUE-013-merged-map-roughness-stats.md b/Tickets/ISSUE-013-merged-map-roughness-stats.md deleted file mode 100644 index b1d7b83..0000000 --- a/Tickets/ISSUE-013-merged-map-roughness-stats.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -ID: ISSUE-013 -Type: Issue -Status: Backlog -Priority: Medium -Labels: [core, bug, refactor, image processing] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: #REFACTOR-001-merge-from-source - ---- - -# [ISSUE-013]: Image statistics calculation missing for roughness maps after merged map refactor - -## Description -Following the recent refactor of the merged map processing logic, the calculation of image statistics (Min/Max/Mean) for roughness maps appears to have been omitted or broken. This data is valuable for quality control and metadata, and needs to be reimplemented for roughness maps, particularly when they are part of a merged texture like NRMRGH. - -## Current Behavior -Image statistics (Min/Max/Mean) are not being calculated or stored for roughness maps after the merged map refactor. - -## Desired Behavior / Goals -Reimplement the image statistics calculation for roughness maps. Ensure that statistics are calculated correctly for standalone roughness maps and for roughness data when it is part of a merged map (e.g., the green channel in an NRMRGH map). The calculated statistics should be included in the `metadata.json` file for the asset. - -## Implementation Notes (Optional) -Review the changes made during the merged map refactor (`REFACTOR-001-merge-from-source`). Identify where the image statistics calculation for roughness maps was previously handled and integrate it into the new merged map processing flow. Special consideration may be needed to extract the correct channel (green for NRMRGH) for calculation. The `_process_maps` and `_merge_maps` methods in `asset_processor.py` are likely relevant areas. - -## Acceptance Criteria (Optional) -* [ ] Process an asset that includes a roughness map (standalone or as part of a merged map). -* [ ] Verify that Min/Max/Mean statistics for the roughness data are calculated. -* [ ] Confirm that the calculated roughness statistics are present and correct in the output `metadata.json` file. \ No newline at end of file diff --git a/Tickets/ISSUE-013-plan.md b/Tickets/ISSUE-013-plan.md deleted file mode 100644 index 5f36b0b..0000000 --- a/Tickets/ISSUE-013-plan.md +++ /dev/null @@ -1,67 +0,0 @@ -# Plan to Resolve ISSUE-013: Merged Map Roughness Statistics - -**Objective:** Reimplement the calculation and inclusion of image statistics (Min/Max/Mean) for roughness data, covering both standalone roughness maps and roughness data used as a source channel in merged maps (specifically the green channel for NRMRGH), and ensure these statistics are present in the output `metadata.json` file. - -**Proposed Changes:** - -1. **Modify `_merge_maps_from_source`:** - * Within the loop that processes each resolution for a merge rule, after successfully loading and transforming the source images into `loaded_inputs_data`, iterate through the `inputs_mapping` for the current rule. - * Identify if any of the source map types in the `inputs_mapping` is 'ROUGH'. - * If 'ROUGH' is used as a source, retrieve the corresponding loaded image data from `loaded_inputs_data`. - * Calculate image statistics (Min/Max/Mean) for this ROUGH source image data using the existing `_calculate_image_stats` helper function. - * Store these calculated statistics temporarily, associated with the output merged map type (e.g., 'NRMRGH') and the target channel it's mapped to (e.g., 'G'). - -2. **Update Asset Metadata Structure:** - * Introduce a new key in the asset's metadata dictionary (e.g., `merged_map_channel_stats`) to store statistics for specific channels within merged maps. This structure will hold the stats per merged map type, per channel, and per resolution (specifically the stats resolution defined in the config). - -3. **Modify `_generate_metadata_file`:** - * When generating the final `metadata.json` file, retrieve the accumulated `merged_map_channel_stats` from the asset's metadata dictionary. - * Include these statistics under a dedicated section in the output JSON, alongside the existing `image_stats_1k` for individual maps. - -**Data Flow for Statistics Calculation:** - -```mermaid -graph TD - A[Source Image Files] --> B{_load_and_transform_source}; - B --> C[Resized/Transformed Image Data (float32)]; - C --> D{Is this source for a ROUGH map?}; - D -- Yes --> E{_calculate_image_stats}; - E --> F[Calculated Stats (Min/Max/Mean)]; - F --> G[Store in merged_map_channel_stats (in asset metadata)]; - C --> H{_merge_maps_from_source}; - H --> I[Merged Image Data]; - I --> J[_save_image]; - J --> K[Saved Merged Map File]; - G --> L[_generate_metadata_file]; - L --> M[metadata.json]; - - subgraph Individual Map Processing - A --> B; - B --> C; - C --> N{Is this an individual map?}; - N -- Yes --> E; - E --> O[Store in image_stats_1k (in asset metadata)]; - C --> P[_save_image]; - P --> Q[Saved Individual Map File]; - O --> L; - Q --> S[Organize Output Files]; - end - - subgraph Merged Map Processing - A --> B; - B --> C; - C --> D; - D -- Yes --> E; - E --> G; - C --> H; - H --> I; - I --> J; - J --> K; - K --> S; - end - - L --> M; - M --> S; -``` - -This plan ensures that statistics are calculated for the roughness data at the appropriate stage (after loading and transformation, before merging) and correctly included in the metadata file, addressing the issue described in ISSUE-013. \ No newline at end of file diff --git a/Tickets/ISSUE-GUI-PreviewNotUpdatingOnDrop.md b/Tickets/ISSUE-GUI-PreviewNotUpdatingOnDrop.md deleted file mode 100644 index 927eb39..0000000 --- a/Tickets/ISSUE-GUI-PreviewNotUpdatingOnDrop.md +++ /dev/null @@ -1,49 +0,0 @@ -# Issue: GUI Preview Not Updating on File Drop Without Preset Selection - -**ID:** ISSUE-GUI-PreviewNotUpdatingOnDrop -**Date:** 2025-04-28 -**Status:** Open -**Priority:** High - -**Description:** -The file preview list (`QTableView` in the main panel) does not populate when files or folders are dropped onto the application if a valid preset has not been explicitly selected *after* the application starts and *before* the drop event. - -**Root Cause:** -- The `add_input_paths` method in `gui/main_window.py` correctly calls `update_preview` after files are added via drag-and-drop. -- However, the `update_preview` method checks `self.editor_preset_list.currentItem()` to get the selected preset. -- If no preset is selected, or if the placeholder "--- Select a Preset ---" is selected, `update_preview` correctly identifies this and returns *before* starting the `PredictionHandler` thread. -- This prevents the file prediction from running and the preview table from being populated, even if assets have been added. - -**Expected Behavior:** -Dropping files should trigger a preview update based on the last valid preset selected by the user, or potentially prompt the user to select a preset if none has ever been selected. The preview should not remain empty simply because the user hasn't re-clicked the preset list immediately before dropping files. - -**Proposed Solution:** -1. Modify `MainWindow` to store the last validly selected preset path. -2. Update the `update_preview` method: - - Instead of relying solely on `currentItem()`, use the stored last valid preset path if `currentItem()` is invalid or the placeholder. - - If no valid preset has ever been selected, display a clear message in the status bar or a dialog box prompting the user to select a preset first. - - Ensure the prediction handler is triggered with the correct preset name based on this updated logic. - -**Affected Files:** -- `gui/main_window.py` - -**Debugging Steps Taken:** -- Confirmed `AssetProcessor.__init__` error was fixed. -- Added logging to `PredictionHandler` and `MainWindow`. -- Logs confirmed `update_preview` is called on drop, but exits early due to no valid preset being selected via `currentItem()`. -## Related Changes (Session 2025-04-28) - -During a recent session, work was done to implement a feature allowing GUI editing of the file preview list, specifically the file status and predicted output name. - -This involved defining a data interface (`ProjectNotes/Data_Structures/Preview_Edit_Interface.md`) and modifying several GUI and backend components. - -Key files modified for this feature include: -- `gui/prediction_handler.py` -- `gui/preview_table_model.py` -- `gui/main_window.py` -- `gui/processing_handler.py` -- Core `asset_processor` logic - -The changes involved passing the editable `file_list` and `asset_properties` data structure through the prediction handler, to the preview table model, and finally to the processing handler. The preview table model was made editable to allow user interaction. The backend logic was updated to utilize this editable data. - -Debugging steps taken during this implementation included fixing indentation errors and resolving an `AssetProcessor` instantiation issue that occurred during the prediction phase. \ No newline at end of file diff --git a/Tickets/REFACTOR-001-merge-from-source.md b/Tickets/REFACTOR-001-merge-from-source.md deleted file mode 100644 index 8c1f27a..0000000 --- a/Tickets/REFACTOR-001-merge-from-source.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -ID: REFACTOR-001 -Type: Refactor -Status: Resolved -Priority: Medium -Labels: [refactor, core, image-processing, quality] -Created: 2025-04-22 -Updated: 2025-04-22 # Keep original update date, or use today's? Using today's for now. -Related: #ISSUE-010 #asset_processor.py #config.py ---- - -# [REFACTOR-001]: Refactor Map Merging to Use Source Files Directly - -## Description - -Currently, the `_merge_maps` function in `asset_processor.py` loads map data from temporary files that have already been processed by `_process_maps` (resized, format converted, etc.). This intermediate save/load step can introduce quality degradation, especially if the intermediate files are saved using lossy compression (e.g., JPG) or if bit depth conversions occur before merging. This is particularly noticeable in merged maps like NRMRGH where subtle details from the source Normal map might be lost or altered due to recompression. - -## Goals - -1. **Improve Quality:** Modify the map merging process to load channel data directly from the *selected original source files* (after classification and 16-bit prioritization) instead of intermediate processed files, thus avoiding potential quality loss from recompression. -2. **Maintain Modularity:** Refactor the processing logic to avoid significant code duplication between individual map processing and the merging process. -3. **Preserve Functionality:** Ensure all existing functionality (resizing, gloss inversion, format/bit depth rules, merging logic) is retained and applied correctly in the new structure. - -## Proposed Solution: Restructure with Helper Functions - -Introduce two helper functions within the `AssetProcessor` class: - -1. **`_load_and_transform_source(source_path_rel, map_type, target_resolution_key)`:** - * Responsible for loading the specified source file. - * Performs initial preparation: BGR->RGB conversion, Gloss->Roughness inversion (if applicable), MASK extraction (if applicable). - * Resizes the prepared data to the target resolution. - * Returns the resized NumPy array and original source dtype. - -2. **`_save_image(image_data, map_type, resolution_key, asset_base_name, source_info, output_bit_depth_rule, temp_dir)`:** - * Encapsulates all logic for saving an image. - * Determines final output format and bit depth based on rules and source info. - * Performs final data type conversions (e.g., to uint8, uint16, float16). - * Performs final color space conversion (RGB->BGR for non-EXR). - * Constructs the output filename. - * Saves the image using `cv2.imwrite`, including fallback logic (e.g., EXR->PNG). - * Returns details of the saved temporary file. - -**Modified Workflow:** - -```mermaid -graph TD - A[Input Files] --> B(_inventory_and_classify_files); - B --> C{Selected Source Maps Info}; - - subgraph Core Processing Logic - C --> PIM(_process_individual_map); - C --> MFS(_merge_maps_from_source); - PIM --> LTS([_load_and_transform_source]); - MFS --> LTS; - end - - LTS --> ImgData{Loaded, Prepared, Resized Image Data}; - - subgraph Saving Logic - PIM --> SI([_save_image]); - MFS --> SI; - ImgData --> SI; // Pass image data to save helper - end - - SI --> SaveResults{Saved Temp File Details}; - - SaveResults --> Results(Processing Results); // Collect results from saving - - Results --> Meta(_generate_metadata_file); - Meta --> Org(_organize_output_files); - Org --> Final[Final Output Structure]; -``` - -**Function Changes:** - -* **`_process_maps`** was renamed to `_process_individual_map`, responsible only for maps *not* used in merges. It calls `_load_and_transform_source` and `_save_image`. -* **`_merge_maps`** was replaced by `_merge_maps_from_source`. It identifies required source paths, calls `_load_and_transform_source` for each input at each target resolution, merges the results, determines saving parameters, and calls `_save_image`. -* The main `process` loop coordinates calls to the new functions and handles caching. - -## Resolution (2025-04-22) - -The refactoring described above was implemented in `asset_processor.py`. The `_merge_maps_from_source` function now utilizes the `_load_and_transform_source` and `_save_image` helpers, loading data directly from the classified source files instead of intermediate processed files. User testing confirmed the changes work correctly and improve merged map quality. - -## Acceptance Criteria (Optional) - -* [X] Merged maps (e.g., NRMRGH) are generated correctly using data loaded directly from selected source files. -* [X] Visual inspection confirms improved quality/reduced artifacts in merged maps compared to the previous method. -* [X] Individual maps (not part of any merge rule) are still processed and saved correctly. -* [X] All existing configuration options (resolutions, bit depth rules, format rules, gloss inversion) function as expected within the new structure. -* [X] Processing time remains within acceptable limits (potential performance impact of repeated source loading/resizing needs monitoring). -* [X] Code remains modular and maintainable, with minimal duplication of core logic. \ No newline at end of file diff --git a/Tickets/REFACTOR-001-plan.md b/Tickets/REFACTOR-001-plan.md deleted file mode 100644 index c31dc6b..0000000 --- a/Tickets/REFACTOR-001-plan.md +++ /dev/null @@ -1,68 +0,0 @@ -# Refactoring Plan for REFACTOR-001: Merge Maps From Source - -This plan details the steps to implement the refactoring described in `Tickets/REFACTOR-001-merge-from-source.md`. The goal is to improve merged map quality by loading data directly from original source files, avoiding intermediate compression artifacts. - -## Final Plan Summary: - -1. **Implement `_load_and_transform_source` Helper:** - * Loads source image, performs initial prep (BGR->RGB, gloss inversion), resizes to target resolution. - * Includes an in-memory cache (passed from `process`) using a `(source_path, resolution_key)` key to store and retrieve resized results, avoiding redundant loading and resizing within a single `process` call. - -2. **Implement `_save_image` Helper:** - * Encapsulates all saving logic: determining output format/bit depth based on rules, final data type/color space conversions, filename construction, saving with `cv2.imwrite`, and fallback logic (e.g., EXR->PNG). - -3. **Refactor `_process_maps` (Potential Rename):** - * Modify to process only maps *not* used as inputs for any merge rule. - * Calls `_load_and_transform_source` (passing cache) and `_save_image`. - -4. **Replace `_merge_maps` with `_merge_maps_from_source`:** - * Iterates through merge rules. - * Calls `_load_and_transform_source` (passing cache) for each required *source* input at target resolutions. - * Merges the resulting channel data. - * Calls `_save_image` to save the final merged map. - -5. **Update `process` Method:** - * Initializes an empty cache dictionary (`loaded_data_cache = {}`) at the beginning of the method. - * Passes this cache dictionary to all calls to `_load_and_transform_source` within its scope. - * Coordinates calls to the refactored/new processing and merging functions. - * Ensures results are collected correctly for metadata generation. - -## New Workflow Visualization: - -```mermaid -graph TD - A2[Input Files] --> B2(_inventory_and_classify_files); - B2 --> C2{Classified Maps Info}; - - subgraph Processing Logic - C2 --> D2(_process_individual_map); - C2 --> E2(_merge_maps_from_source); - D2 --> F2([_load_and_transform_source w/ Cache]); - E2 --> F2; - end - - F2 --> G2{Loaded/Transformed Data (Cached)}; - - subgraph Saving Logic - G2 --> H2([_save_image]); - D2 --> H2; - E2 --> H2; - end - - H2 -- Saves Temp Files --> I2{Processed/Merged Map Details}; - - I2 --> J2(_generate_metadata_file); - J2 --> K2(_organize_output_files); -## Current Status (as of 2025-04-22 ~19:45 CET) - -* **DONE:** Step 1: Implemented `_load_and_transform_source` helper function (including caching logic) and inserted into `asset_processor.py`. -* **DONE:** Step 2: Implemented `_save_image` helper function and inserted into `asset_processor.py`. -* **DONE:** Step 5 (Partial): Updated `process` method to initialize `loaded_data_cache` and updated calls to use new function names (`_process_individual_maps`, `_merge_maps_from_source`) and pass the cache. -* **DONE:** Renamed function definitions: `_process_maps` -> `_process_individual_maps`, `_merge_maps` -> `_merge_maps_from_source`, and added `loaded_data_cache` parameter. -* **DONE:** Corrected syntax errors introduced during previous `apply_diff` operations (related to docstrings). - -## Remaining Steps: - -1. **DONE:** Modify `_process_individual_maps` Logic: Updated the internal logic to correctly utilize `_load_and_transform_source` (with cache) and `_save_image` for maps not involved in merging. -2. **DONE:** Modify `_merge_maps_from_source` Logic: Updated the internal logic to correctly utilize `_load_and_transform_source` (with cache) for *source* files, perform the channel merge, and then use `_save_image` for the merged result. -3. **DONE:** Testing: User confirmed the refactored code works correctly. \ No newline at end of file diff --git a/Tickets/Resolved/BUG-GUI-PreviewToggleCrash.md b/Tickets/Resolved/BUG-GUI-PreviewToggleCrash.md deleted file mode 100644 index 5cc36cd..0000000 --- a/Tickets/Resolved/BUG-GUI-PreviewToggleCrash.md +++ /dev/null @@ -1,43 +0,0 @@ -# BUG: GUI - Persistent Crash When Toggling "Disable Detailed Preview" - -**Ticket Type:** Bug -**Priority:** High -**Status:** Resolved - -**Description:** -The GUI application crashes with a `Fatal Python error: _PyThreadState_Attach: non-NULL old thread state` when toggling the "Disable Detailed Preview" option in the View menu. This issue persisted despite attempted fixes aimed at resolving potential threading conflicts. - -This was a follow-up to a previous ticket regarding the "Disable Detailed Preview" feature regression (refer to ISSUE-GUI-DisableDetailedPreview-Regression.md). While the initial fix addressed the preview display logic, it did not eliminate the crash. - -**Symptoms:** -The application terminates unexpectedly with the fatal Python error traceback when the "Disable Detailed Preview" menu item is toggled on or off, particularly after assets have been added to the queue and the detailed preview has been generated or is in the process of being generated. - -**Steps to Reproduce:** -1. Launch the GUI (`python -m gui.main_window`). -2. (Optional but recommended for diagnosis) Check the "Verbose Logging (DEBUG)" option in the View menu. -3. Add one or more asset files (ZIPs or folders) to the drag and drop area. -4. Wait for the detailed preview to populate (or start populating). -5. Toggle the "Disable Detailed Preview" option in the View menu. The crash should occur. -6. Toggle the option again if the first toggle didn't cause the crash. - -**Attempted Fixes:** -1. Modified `gui/preview_table_model.py` to introduce a `_simple_mode` flag and `set_simple_mode` method to control the data and column presentation for detailed vs. simple views. -2. Modified `gui/main_window.py` (`update_preview` method) to: - * Utilize the `PreviewTableModel.set_simple_mode` method based on the "Disable Detailed Preview" menu action state. - * Configure the `QTableView`'s column visibility and resize modes according to the selected preview mode. - * Request cancellation of the `PredictionHandler` via `prediction_handler.request_cancel()` if it is running when `update_preview` is called. (Note: `request_cancel` did not exist in `PredictionHandler`). -3. Added extensive logging with timestamps and thread IDs to `gui/main_window.py`, `gui/preview_table_model.py`, and `gui/prediction_handler.py` to diagnose threading behavior. - -**Diagnosis:** -Analysis of logs revealed that the crash occurred consistently when toggling the preview back ON, specifically during the `endResetModel` call within `PreviewTableModel.set_simple_mode(False)`. The root cause was identified as a state inconsistency in the `QTableView` (or associated models) caused by a redundant call to `PreviewTableModel.set_data` immediately following `PreviewTableModel.set_simple_mode(True)` within the `MainWindow.update_preview` method when switching *to* simple mode. This resulted in two consecutive `beginResetModel`/`endResetModel` calls on the main thread, leaving the model/view in an unstable state that triggered the crash on the subsequent toggle. Additionally, it was found that `PredictionHandler` lacked a `request_cancel` method, although this was not the direct cause of the crash. - -**Resolution:** -1. Removed the redundant call to `self.preview_model.set_data(list(self.current_asset_paths))` within the `if simple_mode_enabled:` block in `MainWindow.update_preview`. The `set_simple_mode(True)` call is sufficient to switch the model's internal mode. -2. Added an explicit call to `self.preview_model.set_data(list(self.current_asset_paths))` within the `MainWindow.add_input_paths` method, specifically for the case when the GUI is in simple preview mode. This ensures the simple view is updated correctly when new files are added without relying on the problematic `set_data` call in `update_preview`. -3. Corrected instances of `QThread.currentThreadId()` to `QThread.currentThread()` in logging statements across the relevant files. -4. Added the missing `QThread` import in `gui/prediction_handler.py`. - -**Relevant Files/Components:** -* `gui/main_window.py` -* `gui/preview_table_model.py` -* `gui/prediction_handler.py` \ No newline at end of file diff --git a/Tickets/Resolved/FEAT-003-selective-nodegroup-generation-tagging.md b/Tickets/Resolved/FEAT-003-selective-nodegroup-generation-tagging.md deleted file mode 100644 index f4d29f5..0000000 --- a/Tickets/Resolved/FEAT-003-selective-nodegroup-generation-tagging.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -ID: FEAT-003 -Type: Feature -Status: Complete -Priority: Medium -Labels: [feature, blender, metadata] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: ---- - -# [FEAT-003]: Selective Nodegroup Generation and Category Tagging in Blender - -## Description -Enhance the Blender nodegroup creation script (`blenderscripts/create_nodegroups.py`) to only generate nodegroups for assets classified as "Surface" or "Decal" based on the `category` field in their `metadata.json` file. Additionally, store the asset's category (Surface, Decal, or Asset) as a tag on the generated Blender asset for better organization and filtering within Blender. - -## Current Behavior -The current nodegroup generation script processes all assets found in the processed asset library root, regardless of their `category` specified in `metadata.json`. It does not add the asset's category as a tag in Blender. - -## Desired Behavior / Goals -1. The script should read the `category` field from the `metadata.json` file for each processed asset. -2. If the `category` is "Surface" or "Decal", the script should proceed with generating the nodegroup. -3. If the `category` is "Asset" (or any other category), the script should skip nodegroup generation for that asset. -4. The script should add the asset's `category` (e.g., "Surface", "Decal", "Asset") as a tag to the corresponding generated Blender asset. - -## Implementation Notes (Optional) -(This will require modifying `blenderscripts/create_nodegroups.py` to read the `metadata.json` file, check the `category` field, and use the Blender Python API (`bpy`) to add tags to the created asset.) - -## Acceptance Criteria (Optional) -* [x] Run the nodegroup generation script on a processed asset library containing assets of different categories (Surface, Decal, Asset). -* [x] Verify that nodegroups are created only for Surface and Decal assets. -* [x] Verify that assets in the Blender file (both those with generated nodegroups and those skipped) have a tag corresponding to their category from `metadata.json`. \ No newline at end of file diff --git a/Tickets/Resolved/FEAT-004-handle-multi-asset-inputs.md b/Tickets/Resolved/FEAT-004-handle-multi-asset-inputs.md deleted file mode 100644 index 18d4895..0000000 --- a/Tickets/Resolved/FEAT-004-handle-multi-asset-inputs.md +++ /dev/null @@ -1,162 +0,0 @@ ---- -ID: FEAT-004 -Type: Feature -Status: Complete -Priority: Medium -Labels: [core, gui, cli, feature, enhancement] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: #ISSUE-001 ---- - -# [FEAT-004]: Handle Multi-Asset Inputs Based on Source Naming Index - -## Description -Currently, when an input ZIP or folder contains files from multiple distinct assets (as identified by the `source_naming.part_indices.base_name` rule in the preset), the tool's fallback logic uses `os.path.commonprefix` to determine a single, often incorrect, asset name. This prevents the tool from correctly processing inputs containing multiple assets and leads to incorrect predictions in the GUI. - -## Current Behavior -When processing an input containing files from multiple assets (e.g., `3-HeartOak...` and `3-Oak-Classic...` in the same ZIP), the `_determine_base_metadata` method identifies multiple potential base names based on the configured index. It then falls back to calculating the common prefix of all relevant file stems, resulting in a truncated or incorrect asset name (e.g., "3-"). The processing pipeline and GUI prediction then proceed using this incorrect name. - -## Desired Behavior / Goals -The tool should accurately detect when a single input (ZIP/folder) contains files belonging to multiple distinct assets, as defined by the `source_naming.part_indices.base_name` rule. For each distinct base name identified, the tool should process the corresponding subset of files as a separate, independent asset. This includes generating a correct output directory structure and a complete `metadata.json` file for each detected asset within the input. The GUI preview should also accurately reflect the presence of multiple assets and their predicted names. - -## Implementation Notes (Optional) -* Modify `AssetProcessor._determine_base_metadata` to return a list of distinct base names and a mapping of files to their determined base names. -* Adjust the main processing orchestration (`main.py`, `gui/processing_handler.py`) to iterate over the list of distinct base names returned by `_determine_base_metadata`. -* For each distinct base name, create a new processing context (potentially a new `AssetProcessor` instance or a modified approach) that operates only on the files associated with that specific base name. -* Ensure temporary workspace handling and cleanup correctly manage files for multiple assets from a single input. -* Update `AssetProcessor.get_detailed_file_predictions` to correctly identify and group files by distinct base names for accurate GUI preview display. -* Consider edge cases: what if some files don't match any determined base name? (They should likely still go to 'Extra/'). What if the index method yields no names? (Fallback to input name as currently). - -## Acceptance Criteria (Optional) -* [ ] Processing a ZIP file containing files for two distinct assets (e.g., 'AssetA' and 'AssetB') using a preset with `base_name_index` results in two separate output directories (`//AssetA/` and `//AssetB/`), each containing the correctly processed files and metadata for that asset. -* [ ] The GUI preview accurately lists the files from the multi-asset input and shows the correct predicted asset name for each file based on its determined base name (e.g., files belonging to 'AssetA' show 'AssetA' as the predicted name). -* [ ] The CLI processing of a multi-asset input correctly processes and outputs each asset separately. -* [ ] The tool handles cases where some files in a multi-asset input do not match any determined base name (e.g., they are correctly classified as 'Unrecognised' or 'Extra'). ---- -## Implementation Plan (Generated by Architect Mode) - -**Goal:** Modify the tool to correctly identify and process multiple distinct assets within a single input (ZIP/folder) based on the `source_naming.part_indices.base_name` rule, placing unmatched files into the `Extra/` folder of each processed asset. - -**Phase 1: Core Logic Refactoring (`asset_processor.py`)** - -1. **Refactor `_determine_base_metadata`:** - * **Input:** Takes the list of all file paths (relative to temp dir) found after extraction. - * **Logic:** - * Iterates through relevant file stems (maps, models). - * Uses the `source_naming_separator` and `source_naming_indices['base_name']` to extract potential base names for each file stem. - * Identifies the set of *distinct* base names found across all files. - * Creates a mapping: `Dict[Path, Optional[str]]` where keys are relative file paths and values are the determined base name string (or `None` if a file doesn't match any base name according to the index rule). - * **Output:** Returns a tuple: `(distinct_base_names: List[str], file_to_base_name_map: Dict[Path, Optional[str]])`. - * **Remove:** Logic setting `self.metadata["asset_name"]`, `asset_category`, and `archetype`. - -2. **Create New Method `_determine_single_asset_metadata`:** - * **Input:** Takes a specific `asset_base_name` (string) and the list of `classified_files` *filtered* for that asset. - * **Logic:** Contains the logic previously in `_determine_base_metadata` for determining `asset_category` and `archetype` based *only* on the files associated with the given `asset_base_name`. - * **Output:** Returns a dictionary containing `{"asset_category": str, "archetype": str}` for the specific asset. - -3. **Modify `_inventory_and_classify_files`:** - * No major changes needed here initially, as it classifies based on file patterns independent of the final asset name. However, ensure the `classified_files` structure remains suitable for later filtering. - -4. **Refactor `AssetProcessor.process` Method:** - * Change the overall flow to handle multiple assets. - * **Steps:** - 1. `_setup_workspace()` - 2. `_extract_input()` - 3. `_inventory_and_classify_files()` -> Get initial `self.classified_files` (all files). - 4. Call the *new* `_determine_base_metadata()` using all relevant files -> Get `distinct_base_names` list and `file_to_base_name_map`. - 5. Initialize an overall status dictionary (e.g., `{"processed": [], "skipped": [], "failed": []}`). - 6. **Loop** through each `current_asset_name` in `distinct_base_names`: - * Log the start of processing for `current_asset_name`. - * **Filter Files:** Create temporary filtered lists of maps, models, etc., from `self.classified_files` based on the `file_to_base_name_map` for the `current_asset_name`. - * **Determine Metadata:** Call `_determine_single_asset_metadata(current_asset_name, filtered_files)` -> Get category/archetype for this asset. Store these along with `current_asset_name` and supplier name in a temporary `current_asset_metadata` dict. - * **Skip Check:** Perform the skip check logic specifically for `current_asset_name` using the `output_base_path`, supplier name, and `current_asset_name`. If skipped, update overall status and `continue` to the next asset name. - * **Process:** Call `_process_maps()`, `_merge_maps()`, passing the *filtered* file lists and potentially the `current_asset_metadata`. These methods need to operate only on the provided subset of files. - * **Generate Metadata:** Call `_generate_metadata_file()`, passing the `current_asset_metadata` and the results from map/merge processing for *this asset*. This method will now write `metadata.json` specific to `current_asset_name`. - * **Organize Output:** Call `_organize_output_files()`, passing the `current_asset_name`. This method needs modification: - * It will move the processed files for the *current asset* to the correct subfolder (`///`). - * It will also identify files from the *original* input whose base name was `None` in the `file_to_base_name_map` (the "unmatched" files). - * It will copy these "unmatched" files into the `Extra/` subfolder for the *current asset being processed in this loop iteration*. - * Update overall status based on the success/failure of this asset's processing. - 7. `_cleanup_workspace()` (only after processing all assets from the input). - 8. **Return:** Return the overall status dictionary summarizing results across all detected assets. - -5. **Adapt `_process_maps`, `_merge_maps`, `_generate_metadata_file`, `_organize_output_files`:** - * Ensure these methods accept and use the filtered file lists and the specific `asset_name` for the current iteration. - * `_organize_output_files` needs the logic to handle copying the "unmatched" files into the current asset's `Extra/` folder. - -**Phase 2: Update Orchestration (`main.py`, `gui/processing_handler.py`)** - -1. **Modify `main.process_single_asset_wrapper`:** - * The call `processor.process()` will now return the overall status dictionary. - * The wrapper needs to interpret this dictionary to return a single representative status ("processed" if any succeeded, "skipped" if all skipped, "failed" if any failed) and potentially a consolidated error message for the main loop/GUI. - -2. **Modify `gui.processing_handler.ProcessingHandler.run`:** - * No major changes needed here, as it relies on `process_single_asset_wrapper`. The status updates emitted back to the GUI might need slight adjustments if more detailed per-asset status is desired in the future, but for now, the overall status from the wrapper should suffice. - -**Phase 3: Update GUI Prediction (`asset_processor.py`, `gui/prediction_handler.py`, `gui/main_window.py`)** - -1. **Modify `AssetProcessor.get_detailed_file_predictions`:** - * This method must now perform the multi-asset detection: - * Call the refactored `_determine_base_metadata` to get the `distinct_base_names` and `file_to_base_name_map`. - * Iterate through all classified files (maps, models, extra, ignored). - * For each file, look up its corresponding base name in the `file_to_base_name_map`. - * The returned dictionary for each file should now include: - * `original_path`: str - * `predicted_asset_name`: str | None (The base name determined for this file, or None if unmatched) - * `predicted_output_name`: str | None (The predicted final filename, e.g., `AssetName_Color_4K.png`, or original name for models/extra) - * `status`: str ("Mapped", "Model", "Extra", "Unrecognised", "Ignored", **"Unmatched Extra"** - new status for files with `None` base name). - * `details`: str | None - -2. **Update `gui.prediction_handler.PredictionHandler`:** - * Ensure it correctly passes the results from `get_detailed_file_predictions` (including the new `predicted_asset_name` and `status` values) back to the main window via signals. - -3. **Update `gui.main_window.MainWindow`:** - * Modify the preview table model/delegate to display the `predicted_asset_name`. A new column might be needed. - * Update the logic that colors rows or displays status icons to handle the new "Unmatched Extra" status distinctly from regular "Extra" or "Unrecognised". - -**Visual Plan (`AssetProcessor.process` Sequence)** - -```mermaid -sequenceDiagram - participant Client as Orchestrator (main.py / GUI Handler) - participant AP as AssetProcessor - participant Config as Configuration - participant FS as File System - - Client->>AP: process(input_path, config, output_base, overwrite) - AP->>AP: _setup_workspace() - AP->>FS: Create temp_dir - AP->>AP: _extract_input() - AP->>FS: Extract/Copy files to temp_dir - AP->>AP: _inventory_and_classify_files() - AP-->>AP: self.classified_files (all files) - AP->>AP: _determine_base_metadata() - AP-->>AP: distinct_base_names, file_to_base_name_map - - AP->>AP: Initialize overall_status = {} - loop For each current_asset_name in distinct_base_names - AP->>AP: Log start for current_asset_name - AP->>AP: Filter self.classified_files using file_to_base_name_map - AP-->>AP: filtered_files_for_asset - AP->>AP: _determine_single_asset_metadata(current_asset_name, filtered_files_for_asset) - AP-->>AP: current_asset_metadata (category, archetype) - AP->>AP: Perform Skip Check for current_asset_name - alt Skip Check == True - AP->>AP: Update overall_status (skipped) - AP->>AP: continue loop - end - AP->>AP: _process_maps(filtered_files_for_asset, current_asset_metadata) - AP-->>AP: processed_map_details_asset - AP->>AP: _merge_maps(filtered_files_for_asset, current_asset_metadata) - AP-->>AP: merged_map_details_asset - AP->>AP: _generate_metadata_file(current_asset_metadata, processed_map_details_asset, merged_map_details_asset) - AP->>FS: Write metadata.json for current_asset_name - AP->>AP: _organize_output_files(current_asset_name, file_to_base_name_map) - AP->>FS: Move processed files for current_asset_name - AP->>FS: Copy unmatched files to Extra/ for current_asset_name - AP->>AP: Update overall_status (processed/failed for this asset) - end - AP->>AP: _cleanup_workspace() - AP->>FS: Delete temp_dir - AP-->>Client: Return overall_status dictionary \ No newline at end of file diff --git a/Tickets/Resolved/FEAT-011-pot-resizing.md b/Tickets/Resolved/FEAT-011-pot-resizing.md deleted file mode 100644 index 15b3a78..0000000 --- a/Tickets/Resolved/FEAT-011-pot-resizing.md +++ /dev/null @@ -1,56 +0,0 @@ -# Ticket: FEAT-011 - Implement Power-of-Two Texture Resizing - -**Status:** Open -**Priority:** High -**Assignee:** TBD -**Reporter:** Roo (Architect Mode) - -## Description - -The current asset processing pipeline resizes textures based on a target maximum dimension (e.g., 4K = 4096px) while maintaining the original aspect ratio. This results in non-power-of-two (NPOT) dimensions for non-square textures, which is suboptimal for rendering performance and compatibility with certain systems. - -This feature implements a "Stretch/Squash" approach to ensure all output textures have power-of-two (POT) dimensions for each target resolution key. - -## Proposed Solution - -1. **Resizing Logic Change:** - * Modify the `calculate_target_dimensions` helper function in `asset_processor.py`. - * **Step 1:** Calculate intermediate dimensions (`scaled_w`, `scaled_h`) by scaling the original image (orig_w, orig_h) to fit within the target resolution key's maximum dimension (e.g., 4096 for "4K") while maintaining the original aspect ratio (using existing logic). - * **Step 2:** Implement a new helper function `get_nearest_pot(value: int) -> int` to find the closest power-of-two value for a given integer. - * **Step 3:** Apply `get_nearest_pot()` to `scaled_w` to get the final target power-of-two width (`pot_w`). - * **Step 4:** Apply `get_nearest_pot()` to `scaled_h` to get the final target power-of-two height (`pot_h`). - * **Step 5:** Return `(pot_w, pot_h)` from `calculate_target_dimensions`. The `_process_maps` function will then use these POT dimensions in `cv2.resize`. - -2. **Helper Function `get_nearest_pot`:** - * This function will take an integer `value`. - * It will find the powers of two immediately below (`lower_pot`) and above (`upper_pot`) the value. - * It will return the power of two that is numerically closer to the original `value`. (e.g., `get_nearest_pot(1365)` would return 1024, as `1365 - 1024 = 341` and `2048 - 1365 = 683`). - -3. **Filename Convention:** - * The original resolution tag (e.g., `_4K`, `_2K`) defined in `config.py` will be kept in the output filename, even though the final dimensions are POT. This maintains consistency with the processing target. - -4. **Metadata:** - * The existing aspect ratio change metadata calculation (`_normalize_aspect_ratio_change`) will remain unchanged. This metadata can be used downstream to potentially correct the aspect ratio distortion introduced by the stretch/squash resizing. - -## Implementation Diagram - -```mermaid -graph TD - A[Original Dimensions (W, H)] --> B{Target Resolution Key (e.g., "4K")}; - B --> C{Get Max Dimension (e.g., 4096)}; - A & C --> D[Calculate Scaled Dimensions (scaled_w, scaled_h) - Maintain Aspect Ratio]; - D --> E[scaled_w]; - D --> F[scaled_h]; - E --> G[Find Nearest POT(scaled_w) -> pot_w]; - F --> H[Find Nearest POT(scaled_h) -> pot_h]; - G & H --> I[Final POT Dimensions (pot_w, pot_h)]; - I --> J[Use (pot_w, pot_h) in cv2.resize]; -``` - -## Acceptance Criteria - -* All textures output by the `_process_maps` function have power-of-two dimensions (width and height are both powers of 2). -* The resizing uses the "Stretch/Squash" method based on the nearest POT value for each dimension calculated *after* initial aspect-preserving scaling. -* The output filename retains the original resolution key (e.g., `_4K`). -* The `get_nearest_pot` helper function correctly identifies the closest power of two. -* The aspect ratio metadata calculation remains unchanged. \ No newline at end of file diff --git a/Tickets/Resolved/ISSUE-001-source-naming-rules-not-respected.md b/Tickets/Resolved/ISSUE-001-source-naming-rules-not-respected.md deleted file mode 100644 index 6bc8a3b..0000000 --- a/Tickets/Resolved/ISSUE-001-source-naming-rules-not-respected.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -ID: ISSUE-001 -Type: Issue -Status: Backlog -Priority: Medium -Labels: [bug, config, gui] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: ---- - -# [ISSUE-001]: Source file naming rules from JSON are not respected - -## Description -The tool is not correctly applying the "Source file naming rules" defined in the JSON presets. Specifically, the "base Name index" and "Map type index" values within the `source_naming` section of the preset JSON are not being respected during file processing. - -## Current Behavior -When processing assets, the tool (observed in the GUI) does not use the specified "base Name index" and "Map type index" from the active preset's `source_naming` rules to determine the asset's base name and individual map types from the source filenames. - -## Desired Behavior / Goals -The tool should accurately read and apply the "base Name index" and "Map type index" values from the selected preset's `source_naming` rules to correctly parse asset base names and map types from source filenames. - -## Implementation Notes (Optional) -(Add any thoughts on how this could be implemented, technical challenges, relevant code sections, or ideas for a solution.) - -## Acceptance Criteria (Optional) -(Define clear, testable criteria that must be met for the ticket to be considered complete.) -* [ ] Processing an asset with a preset that uses specific `base_name_index` and `map_type_index` values results in the correct asset name and map types being identified according to those indices. -* [ ] This behavior is consistent in both the GUI and CLI. \ No newline at end of file diff --git a/Tickets/Resolved/ISSUE-002-incorrect-col-numbering-multiple-assets.md b/Tickets/Resolved/ISSUE-002-incorrect-col-numbering-multiple-assets.md deleted file mode 100644 index 9a15054..0000000 --- a/Tickets/Resolved/ISSUE-002-incorrect-col-numbering-multiple-assets.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -ID: ISSUE-002 -Type: Issue -Status: Mostly Resolved -Priority: High -Labels: [bug, core, file-classification] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: ---- - -# [ISSUE-002]: Incorrect COL-# numbering with multiple assets in one directory - -## Description -When processing a directory containing multiple distinct asset sets (e.g., `Assetname1` and `Assetname2`), the numbering for map types that require variant suffixes (like "COL") is incorrectly incremented across all assets in the directory rather than being reset for each individual asset. - -## Current Behavior -If an input directory contains files for `Assetname1` and `Assetname2`, and both have multiple "COL" maps, the numbering continues sequentially across both sets. For example, `Assetname1` might get `_COL-1`, `_COL-2`, while `Assetname2` incorrectly gets `_COL-3`, `_COL-4`, instead of starting its own sequence (`_COL-1`, `_COL-2`). The "COL value accumulates across directory, not comparing names". - -## Desired Behavior / Goals -The tool should correctly identify distinct asset sets within a single input directory and apply variant numbering (like "COL-#") independently for each asset set. The numbering should reset for each new asset encountered in the directory. - -## Implementation Notes (Optional) -(This likely requires adjusting the file classification or inventory logic to group files by asset name before applying variant numbering rules.) - -## Acceptance Criteria (Optional) -* [ ] Processing a directory containing multiple asset sets with variant map types results in correct, independent numbering for each asset set (e.g., `Assetname1_COL-1`, `Assetname1_COL-2`, `Assetname2_COL-1`, `Assetname2_COL-2`). -* [ ] The numbering is based on the files belonging to a specific asset name, not the overall count of variant maps in the entire input directory. \ No newline at end of file diff --git a/Tickets/Resolved/ISSUE-005-alpha-mask-channel-incorrect.md b/Tickets/Resolved/ISSUE-005-alpha-mask-channel-incorrect.md deleted file mode 100644 index 646eff9..0000000 --- a/Tickets/Resolved/ISSUE-005-alpha-mask-channel-incorrect.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -ID: ISSUE-005 -Type: Issue -Status: Resolved -Priority: High -Labels: [bug, core, image-processing] -Created: 2025-04-22 -Updated: 2025-04-22 -Related: ---- - -# [ISSUE-005]: Alpha Mask channel not processed correctly - -## Description -When processing source images that contain an alpha channel intended for use as a MASK map, the tool's output for the MASK map is an RGBA image instead of a grayscale image derived solely from the alpha channel. - -## Current Behavior -If a source image (e.g., a PNG or TIF) has an alpha channel and is classified as a MASK map type, the resulting output MASK file retains the RGB channels (potentially with incorrect data or black/white values) in addition to the alpha channel, resulting in an RGBA output image. - -## Desired Behavior / Goals -When processing a source image with an alpha channel for a MASK map type, the tool should extract only the alpha channel data and output a single-channel (grayscale) image representing the mask. The RGB channels from the source should be discarded for the MASK output. - -## Implementation Notes (Optional) -(This requires modifying the image processing logic for MASK map types to specifically isolate and save only the alpha channel as a grayscale image. Need to check the relevant sections in `asset_processor.py` related to map processing and saving.) - -## Acceptance Criteria (Optional) -* [ ] Process an asset containing a source image with an alpha channel intended as a MASK map. -* [ ] Verify that the output MASK file is a grayscale image (single channel) and accurately represents the alpha channel data from the source. -* [ ] Verify that the output MASK file does not contain redundant or incorrect RGB channel data. \ No newline at end of file diff --git a/Tickets/Resolved/ISSUE-006-col-increment-multi-asset.md b/Tickets/Resolved/ISSUE-006-col-increment-multi-asset.md deleted file mode 100644 index 7e8b7ec..0000000 --- a/Tickets/Resolved/ISSUE-006-col-increment-multi-asset.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -ID: ISSUE-006 # e.g., FEAT-001, ISSUE-002 -Type: Issue # Choose one: Issue or Feature -Status: Mostly Resolved # Choose one -Priority: Medium # Choose one -Labels: [core, bug, map_processing, multi_asset] # Add relevant labels from the list or define new ones -Created: 2025-04-22 -Updated: 2025-04-22 -Related: #FEAT-004-handle-multi-asset-inputs.md # Links to other tickets (e.g., #ISSUE-YYY), relevant files, or external URLs ---- - -# [ISSUE-006]: COL-# Suffixes Incorrectly Increment Across Multi-Asset Inputs - -## Description -When processing an input (ZIP or folder) that contains files for multiple distinct assets, the numeric suffixes applied to map types listed in `RESPECT_VARIANT_MAP_TYPES` (such as "COL") are currently incremented globally across all files in the input, rather than being reset and incremented independently for each detected asset group. - -## Current Behavior -If an input contains files for AssetA (e.g., AssetA_COL.png, AssetA_COL_Variant.png) and AssetB (e.g., AssetB_COL.png), the output might incorrectly number them as AssetA_COL-1.png, AssetA_COL-2.png, and AssetB_COL-3.png. The expectation is that numbering should restart for each asset, resulting in AssetA_COL-1.png, AssetA_COL-2.png, and AssetB_COL-1.png. - -## Desired Behavior / Goals -The numeric suffix for map types in `RESPECT_VARIANT_MAP_TYPES` should be determined and applied independently for each distinct asset detected within a multi-asset input. The numbering should start from -1 for each asset group. - -## Implementation Notes (Optional) -- The logic for assigning suffixes is primarily within `AssetProcessor._inventory_and_classify_files`. -- This method currently classifies all files from the input together before determining asset groups. -- The classification logic needs to be adjusted to perform suffix assignment *after* files have been grouped by their determined asset name. -- This might require modifying the output of `_inventory_and_classify_files` or adding a new step after `_determine_base_metadata` to re-process or re-structure the classified files per asset for suffix assignment. - -## Acceptance Criteria (Optional) -* [ ] Processing a multi-asset input containing multiple "COL" variants for different assets results in correct COL-# suffixes starting from -1 for each asset's output files. -* [ ] The GUI preview accurately reflects the correct COL-# numbering for each file based on its predicted asset name. -* [ ] The CLI processing output confirms the correct numbering in the generated filenames. \ No newline at end of file diff --git a/Tickets/Resolved/ISSUE-007-respect-variant-single-map.md b/Tickets/Resolved/ISSUE-007-respect-variant-single-map.md deleted file mode 100644 index e1d6aa5..0000000 --- a/Tickets/Resolved/ISSUE-007-respect-variant-single-map.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -ID: ISSUE-007 # e.g., FEAT-001, ISSUE-002 -Type: Issue # Choose one: Issue or Feature -Status: Resolved # Choose one -Priority: Medium # Choose one -Labels: [core, bug, map_processing, suffix, regression] # Add relevant labels from the list or define new ones -Created: 2025-04-22 -Updated: 2025-04-22 -Related: #ISSUE-006-col-increment-multi-asset.md # Links to other tickets (e.g., #ISSUE-YYY), relevant files, or external URLs ---- - -# [ISSUE-007]: Suffix Not Applied to Single Maps in RESPECT_VARIANT_MAP_TYPES - -## Description -Map types listed in `config.py`'s `RESPECT_VARIANT_MAP_TYPES` (e.g., "COL") are expected to always receive a numeric suffix (e.g., "-1"), even if only one map of that type exists for a given asset. Following the fix for #ISSUE-006, this behavior is no longer occurring. Single maps of these types are now output without a suffix. - -## Current Behavior -An asset containing only one map file designated as "COL" (e.g., `AssetA_COL.png`) results in processed output files named like `AssetA_COL_4K.png`, without the `-1` suffix. - -## Desired Behavior / Goals -An asset containing only one map file designated as "COL" (or any other type listed in `RESPECT_VARIANT_MAP_TYPES`) should result in processed output files named like `AssetA_COL-1_4K.png`, correctly applying the numeric suffix even when it's the sole variant. - -## Implementation Notes (Optional) -- The per-asset suffix assignment logic added in `AssetProcessor.process` (around line 233 in the previous diff) likely needs adjustment. -- The condition `if respect_variants:` might need to be modified, or the loop/enumeration logic needs to explicitly handle the case where `len(maps_in_group) == 1` for types listed in `RESPECT_VARIANT_MAP_TYPES`. - -## Acceptance Criteria (Optional) -* [ ] Processing an asset with a single "COL" map results in output files with the `COL-1` suffix. -* [ ] Processing an asset with multiple "COL" maps still results in correctly incremented suffixes (`COL-1`, `COL-2`, etc.). -* [ ] Map types *not* listed in `RESPECT_VARIANT_MAP_TYPES` continue to receive no suffix if only one exists. \ No newline at end of file diff --git a/Tickets/Resolved/ISSUE-GUI-DisableDetailedPreview-Regression.md b/Tickets/Resolved/ISSUE-GUI-DisableDetailedPreview-Regression.md deleted file mode 100644 index 444d6ba..0000000 --- a/Tickets/Resolved/ISSUE-GUI-DisableDetailedPreview-Regression.md +++ /dev/null @@ -1,24 +0,0 @@ -# ISSUE: GUI - "Disable Detailed Preview" feature regression - -**Ticket Type:** Issue -**Priority:** Medium - -**Description:** -The "Disable Detailed Preview" feature in the GUI is currently not functioning correctly. When attempting to disable the detailed file preview (via the View menu), the GUI does not switch to the simpler asset list view. This regression prevents users from using the less detailed preview mode, which may impact performance or usability, especially when dealing with inputs containing a large number of files. - -**Steps to Reproduce:** -1. Launch the GUI (`python -m gui.main_window`). -2. Load an asset (ZIP or folder) into the drag and drop area. Observe the detailed preview table populating. -3. Go to the "View" menu. -4. Select/Deselect the "Detailed File Preview" option. - -**Expected Result:** -The preview table should switch between the detailed file list view and the simple asset list view when the menu option is toggled. - -**Actual Result:** -The preview table remains in the detailed file list view regardless of the "Detailed File Preview" menu option state. - -**Relevant Files/Components:** -* `gui/main_window.py`: Likely contains the logic for the View menu and handling the toggle state. -* `gui/prediction_handler.py`: Manages the background process that generates the detailed preview data. The GUI needs to be able to stop or not request this process when detailed preview is disabled. -* `gui/preview_table_model.py`: Manages the data and display logic for the preview table. It should adapt its display based on whether detailed preview is enabled or disabled. \ No newline at end of file diff --git a/Tickets/Ticket-README.md b/Tickets/Ticket-README.md deleted file mode 100644 index 30fe79e..0000000 --- a/Tickets/Ticket-README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Issue and Feature Tracking - -This directory is used to track issues and feature ideas for the Asset Processor Tool using Markdown files. - -## Structure - -All ticket files are stored directly within the `Tickets/` directory. - -```mermaid -graph TD - A[Asset_processor_tool] --> B(Tickets); - A --> C(...other files/dirs...); - B --> D(ISSUE-XXX-....md); - B --> E(FEAT-XXX-....md); - B --> F(_template.md); -``` - -## File Naming Convention - -Ticket files should follow the convention: `TYPE-ID-short-description.md` - -* `TYPE`: `ISSUE` for bug reports or problems, `FEAT` for new features or enhancements. -* `ID`: A sequential three-digit number (e.g., `001`, `002`). -* `short-description`: A brief, hyphenated summary of the ticket's content. - -Examples: -* `ISSUE-001-gui-preview-bug.md` -* `FEAT-002-add-dark-mode.md` - -## Ticket Template (`_template.md`) - -Use the `_template.md` file as a starting point for creating new tickets. It includes YAML front matter for structured metadata and standard Markdown headings for the ticket content. - -```markdown ---- -ID: TYPE-XXX # e.g., FEAT-001, ISSUE-002 -Type: Issue | Feature # Choose one: Issue or Feature -Status: Backlog | Planned | In Progress | Blocked | Needs Review | Done | Won't Fix # Choose one -Priority: Low | Medium | High # Choose one -Labels: [gui, cli, core, blender, bug, feature, enhancement, docs, config] # Add relevant labels from the list or define new ones -Created: YYYY-MM-DD -Updated: YYYY-MM-DD -Related: # Links to other tickets (e.g., #ISSUE-YYY), relevant files, or external URLs ---- - -# [TYPE-XXX]: Brief Title of Issue/Feature - -## Description -(Provide a detailed explanation of the issue or feature request. What is the problem you are trying to solve, or what is the new functionality you are proposing?) - -## Current Behavior -(Describe what happens currently. If reporting a bug, explain the steps to reproduce it. If proposing a feature, describe the current state without the feature.) - -## Desired Behavior / Goals -(Describe what *should* happen if the issue is resolved, or what the feature aims to achieve. Be specific about the desired outcome.) - -## Implementation Notes (Optional) -(Add any thoughts on how this could be implemented, potential technical challenges, relevant code sections, or ideas for a solution.) - -## Acceptance Criteria (Optional) -(Define clear, testable criteria that must be met for the ticket to be considered complete.) -* [ ] Criterion 1: The first condition that must be satisfied. -* [ ] Criterion 2: The second condition that must be satisfied. -* [ ] Add more criteria as needed. - -``` - -## How to Use - -1. Create a new Markdown file in the `Tickets/` directory following the naming convention (`TYPE-ID-short-description.md`). -2. Copy the content from `_template.md` into your new file. -3. Fill in the YAML front matter and the Markdown sections with details about the issue or feature. -4. Update the `Status` and `Updated` fields as you work on the ticket. -5. Use the `Related` field to link to other relevant tickets or project files. -``` diff --git a/Tickets/_template.md b/Tickets/_template.md deleted file mode 100644 index 2ce9536..0000000 --- a/Tickets/_template.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -ID: TYPE-XXX # e.g., FEAT-001, ISSUE-002 -Type: Issue | Feature # Choose one: Issue or Feature -Status: Backlog | Planned | In Progress | Blocked | Needs Review | Done | Won't Fix # Choose one -Priority: Low | Medium | High # Choose one -Labels: [gui, cli, core, blender, bug, feature, enhancement, docs, config] # Add relevant labels from the list or define new ones -Created: YYYY-MM-DD -Updated: YYYY-MM-DD -Related: # Links to other tickets (e.g., #ISSUE-YYY), relevant files, or external URLs ---- - -# [TYPE-XXX]: Brief Title of Issue/Feature - -## Description -(Provide a detailed explanation of the issue or feature request. What is the problem you are trying to solve, or what is the new functionality you are proposing?) - -## Current Behavior -(Describe what happens currently. If reporting a bug, explain the steps to reproduce it. If proposing a feature, describe the current state without the feature.) - -## Desired Behavior / Goals -(Describe what *should* happen if the issue is resolved, or what the feature aims to achieve. Be specific about the desired outcome.) - -## Implementation Notes (Optional) -(Add any thoughts on how this could be implemented, potential technical challenges, relevant code sections, or ideas for a solution.) - -## Acceptance Criteria (Optional) -(Define clear, testable criteria that must be met for the ticket to be considered complete.) -* [ ] Criterion 1: The first condition that must be satisfied. -* [ ] Criterion 2: The second condition that must be satisfied. -* [ ] Add more criteria as needed. \ No newline at end of file diff --git a/__pycache__/asset_processor.cpython-310.pyc b/__pycache__/asset_processor.cpython-310.pyc deleted file mode 100644 index 9d713f8..0000000 Binary files a/__pycache__/asset_processor.cpython-310.pyc and /dev/null differ diff --git a/__pycache__/asset_processor.cpython-313.pyc b/__pycache__/asset_processor.cpython-313.pyc deleted file mode 100644 index 1ad8257..0000000 Binary files a/__pycache__/asset_processor.cpython-313.pyc and /dev/null differ diff --git a/__pycache__/config.cpython-310.pyc b/__pycache__/config.cpython-310.pyc deleted file mode 100644 index 63d61b1..0000000 Binary files a/__pycache__/config.cpython-310.pyc and /dev/null differ diff --git a/__pycache__/config.cpython-313.pyc b/__pycache__/config.cpython-313.pyc deleted file mode 100644 index ab1c078..0000000 Binary files a/__pycache__/config.cpython-313.pyc and /dev/null differ diff --git a/__pycache__/configuration.cpython-310.pyc b/__pycache__/configuration.cpython-310.pyc deleted file mode 100644 index 93333de..0000000 Binary files a/__pycache__/configuration.cpython-310.pyc and /dev/null differ diff --git a/__pycache__/configuration.cpython-313.pyc b/__pycache__/configuration.cpython-313.pyc deleted file mode 100644 index d6e3369..0000000 Binary files a/__pycache__/configuration.cpython-313.pyc and /dev/null differ diff --git a/__pycache__/main.cpython-310.pyc b/__pycache__/main.cpython-310.pyc deleted file mode 100644 index 4582835..0000000 Binary files a/__pycache__/main.cpython-310.pyc and /dev/null differ diff --git a/__pycache__/main.cpython-313.pyc b/__pycache__/main.cpython-313.pyc deleted file mode 100644 index 15bf5b0..0000000 Binary files a/__pycache__/main.cpython-313.pyc and /dev/null differ diff --git a/asset_processor.py b/asset_processor.py deleted file mode 100644 index d7ebb75..0000000 --- a/asset_processor.py +++ /dev/null @@ -1,2514 +0,0 @@ -# asset_processor.py - -import os -import math -import shutil -import tempfile -import zipfile -import logging -import json -import re -import time -from pathlib import Path -from fnmatch import fnmatch # For pattern matching like *.fbx, *_Preview* -from typing import List, Dict, Tuple, Optional # Added for type hinting -from collections import defaultdict # Added for grouping - -# Attempt to import archive libraries -try: - import rarfile - import py7zr -except ImportError as e: - print(f"ERROR: Missing required archive libraries: {e}") - print("Please install them using:") - print("pip install rarfile py7zr") - # Do not exit here, allow the script to run but extraction will fail for these types - rarfile = None # Set to None so checks can still be made - py7zr = None # Set to None - -# Attempt to import image processing libraries -try: - import cv2 - import numpy as np -except ImportError: - print("ERROR: Missing required image processing libraries. Please install opencv-python and numpy:") - print("pip install opencv-python numpy") - exit(1) # Exit if essential libraries are missing - -# Attempt to import OpenEXR - Check if needed for advanced EXR flags/types -try: - import OpenEXR - import Imath - _HAS_OPENEXR = True -except ImportError: - _HAS_OPENEXR = False - # Log this information - basic EXR might still work via OpenCV - logging.debug("Optional 'OpenEXR' python package not found. EXR saving relies on OpenCV's built-in support.") - - -# Assuming Configuration class is in configuration.py -try: - from configuration import Configuration, ConfigurationError -except ImportError: - print("ERROR: Cannot import Configuration class from configuration.py.") - print("Ensure configuration.py is in the same directory or Python path.") - exit(1) - -# Use logger defined in main.py (or configure one here if run standalone) -log = logging.getLogger(__name__) -# Basic config if logger hasn't been set up elsewhere (e.g., during testing) -if not log.hasHandlers(): - logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') # Reverted basicConfig level - - -# --- Custom Exception --- -class AssetProcessingError(Exception): - """Custom exception for errors during asset processing.""" - pass - -# --- Helper Functions --- -def calculate_target_dimensions(orig_w, orig_h, target_max_dim) -> tuple[int, int]: - """ - Calculates target dimensions by first scaling to fit target_max_dim - while maintaining aspect ratio, then finding the nearest power-of-two - value for each resulting dimension (Stretch/Squash to POT). - """ - if orig_w <= 0 or orig_h <= 0: - # Fallback to target_max_dim if original dimensions are invalid - pot_dim = get_nearest_pot(target_max_dim) - log.warning(f"Invalid original dimensions ({orig_w}x{orig_h}). Falling back to nearest POT of target_max_dim: {pot_dim}x{pot_dim}") - return (pot_dim, pot_dim) - - # Step 1: Calculate intermediate dimensions maintaining aspect ratio - ratio = orig_w / orig_h - if ratio > 1: # Width is dominant - scaled_w = target_max_dim - scaled_h = max(1, round(scaled_w / ratio)) - else: # Height is dominant or square - scaled_h = target_max_dim - scaled_w = max(1, round(scaled_h * ratio)) - - # Step 2: Find the nearest power of two for each scaled dimension - pot_w = get_nearest_pot(scaled_w) - pot_h = get_nearest_pot(scaled_h) - - log.debug(f"POT Calc: Orig=({orig_w}x{orig_h}), MaxDim={target_max_dim} -> Scaled=({scaled_w}x{scaled_h}) -> POT=({pot_w}x{pot_h})") - - return int(pot_w), int(pot_h) - -def _calculate_image_stats(image_data: np.ndarray) -> dict | None: - """ - Calculates min, max, mean for a given numpy image array. - Handles grayscale and multi-channel images. Converts to float64 for calculation. - """ - if image_data is None: - log.warning("Attempted to calculate stats on None image data.") - return None - try: - # Use float64 for calculations to avoid potential overflow/precision issues - data_float = image_data.astype(np.float64) - - # Normalize data_float based on original dtype before calculating stats - if image_data.dtype == np.uint16: - log.debug("Stats calculation: Normalizing uint16 data to 0-1 range.") - data_float /= 65535.0 - elif image_data.dtype == np.uint8: - log.debug("Stats calculation: Normalizing uint8 data to 0-1 range.") - data_float /= 255.0 - # Assuming float inputs are already in 0-1 range or similar - - log.debug(f"Stats calculation: data_float dtype: {data_float.dtype}, shape: {data_float.shape}") - # Log a few sample values to check range after normalization - if data_float.size > 0: - sample_values = data_float.flatten()[:10] # Get first 10 values - log.debug(f"Stats calculation: Sample values (first 10) after normalization: {sample_values.tolist()}") - - - if len(data_float.shape) == 2: # Grayscale (H, W) - min_val = float(np.min(data_float)) - max_val = float(np.max(data_float)) - mean_val = float(np.mean(data_float)) - stats = {"min": min_val, "max": max_val, "mean": mean_val} - log.debug(f"Calculated Grayscale Stats: Min={min_val:.4f}, Max={max_val:.4f}, Mean={mean_val:.4f}") - elif len(data_float.shape) == 3: # Color (H, W, C) - channels = data_float.shape[2] - min_val = [float(v) for v in np.min(data_float, axis=(0, 1))] - max_val = [float(v) for v in np.max(data_float, axis=(0, 1))] - mean_val = [float(v) for v in np.mean(data_float, axis=(0, 1))] - # The input data_float is now expected to be in RGB order after conversion in _process_maps - stats = {"min": min_val, "max": max_val, "mean": mean_val} - log.debug(f"Calculated {channels}-Channel Stats (RGB order): Min={min_val}, Max={max_val}, Mean={mean_val}") - else: - log.warning(f"Cannot calculate stats for image with unsupported shape {data_float.shape}") - return None - return stats - except Exception as e: - log.error(f"Error calculating image stats: {e}", exc_info=True) # Log exception info - return {"error": str(e)} - - -# --- Helper function --- -def _get_base_map_type(target_map_string: str) -> str: - """Extracts the base map type (e.g., 'COL') from a potentially numbered string ('COL-1').""" - match = re.match(r"([a-zA-Z]+)", target_map_string) - if match: - return match.group(1).upper() - return target_map_string.upper() # Fallback if no number suffix - - -def _is_power_of_two(n: int) -> bool: - """Checks if a number is a power of two.""" - return (n > 0) and (n & (n - 1) == 0) - -def get_nearest_pot(value: int) -> int: - """Finds the nearest power of two to the given value.""" - if value <= 0: - return 1 # Or raise error, POT must be positive - if _is_power_of_two(value): - return value - - # Calculate the powers of two below and above the value - lower_pot = 1 << (value.bit_length() - 1) - upper_pot = 1 << value.bit_length() - - # Determine which power of two is closer - if (value - lower_pot) < (upper_pot - value): - return lower_pot - else: - return upper_pot - -# --- Asset Processor Class --- -class AssetProcessor: - """ - Handles the processing pipeline for a single asset (ZIP or folder). - """ - # Define the list of known grayscale map types (adjust as needed) - GRAYSCALE_MAP_TYPES = ['HEIGHT', 'ROUGH', 'METAL', 'AO', 'OPC', 'MASK'] - - def __init__(self, input_path: Path, config: Configuration, output_base_path: Path, overwrite: bool = False): - """ - Initializes the processor for a given input asset. - - Args: - input_path: Path to the input ZIP file or folder. - config: The loaded Configuration object. - output_base_path: The base directory where processed output will be saved. - overwrite: If True, forces reprocessing even if output exists. - """ - if not isinstance(input_path, Path): input_path = Path(input_path) - if not isinstance(output_base_path, Path): output_base_path = Path(output_base_path) - if not isinstance(config, Configuration): raise TypeError("config must be a Configuration object.") - - if not input_path.exists(): - raise AssetProcessingError(f"Input path does not exist: {input_path}") - supported_suffixes = ['.zip', '.rar', '.7z'] - if not (input_path.is_dir() or (input_path.is_file() and input_path.suffix.lower() in supported_suffixes)): - raise AssetProcessingError(f"Input path must be a directory or a supported archive file (.zip, .rar, .7z): {input_path}") - - self.input_path: Path = input_path - self.config: Configuration = config - self.output_base_path: Path = output_base_path - self.overwrite: bool = overwrite # Store the overwrite flag - - self.temp_dir: Path | None = None # Path to the temporary working directory - self.classified_files: dict[str, list[dict]] = { - "maps": [], "models": [], "extra": [], "ignored": [] - } - # These will no longer store instance-wide results, but are kept for potential future use or refactoring - # self.processed_maps_details: dict[str, dict[str, dict]] = {} - # self.merged_maps_details: dict[str, dict[str, dict]] = {} - # self.metadata_file_path_temp: Path | None = None - # self.metadata: dict = {} # Metadata is now handled per-asset within the process loop - - log.debug(f"AssetProcessor initialized for: {self.input_path.name}") - - - # --- New Helper Function: Load and Transform Source --- - def _load_and_transform_source(self, source_path_rel: Path, map_type: str, target_resolution_key: str, is_gloss_source: bool, cache: dict) -> Tuple[Optional[np.ndarray], Optional[np.dtype]]: - """ - Loads a source image file, performs initial prep (BGR->RGB, Gloss->Rough), - resizes it to the target resolution, and caches the result. - - Args: - source_path_rel: Relative path to the source file within the temp directory. - map_type: The standard map type (e.g., "NRM", "ROUGH"). - target_resolution_key: The key for the target resolution (e.g., "4K"). - is_gloss_source: Boolean indicating if this source should be treated as gloss for inversion. - cache: The dictionary used for caching loaded/resized data. - - Returns: - Tuple containing: - - Resized NumPy array (float32) or None if loading/processing fails. - - Original source NumPy dtype or None if loading fails. - """ - if not self.temp_dir: - log.error("Temporary directory not set in _load_and_transform_source.") - return None, None - - cache_key = (source_path_rel, target_resolution_key) - if cache_key in cache: - log.debug(f"CACHE HIT: Returning cached data for {source_path_rel} at {target_resolution_key}") - return cache[cache_key] # Return tuple (image_data, source_dtype) - - log.debug(f"CACHE MISS: Loading and transforming {source_path_rel} for {target_resolution_key}") - full_source_path = self.temp_dir / source_path_rel - img_prepared = None - source_dtype = None - - try: - # --- 1. Load Source Image --- - # Determine read flag (Grayscale for specific types, unchanged otherwise) - read_flag = cv2.IMREAD_GRAYSCALE if map_type.upper() in self.GRAYSCALE_MAP_TYPES else cv2.IMREAD_UNCHANGED - # Special case for MASK: always load unchanged first to check alpha - if map_type.upper() == 'MASK': read_flag = cv2.IMREAD_UNCHANGED - - log.debug(f"Loading source {full_source_path.name} with flag: {'GRAYSCALE' if read_flag == cv2.IMREAD_GRAYSCALE else 'UNCHANGED'}") - img_loaded = cv2.imread(str(full_source_path), read_flag) - if img_loaded is None: - raise AssetProcessingError(f"Failed to load image file: {full_source_path.name} with flag {read_flag}") - source_dtype = img_loaded.dtype - log.debug(f"Loaded source {full_source_path.name}, dtype: {source_dtype}, shape: {img_loaded.shape}") - - # MASK Handling (Extract alpha or convert) - Do this BEFORE general color conversions - if _get_base_map_type(map_type) == 'MASK': - log.debug(f"Processing as MASK type for {source_path_rel.name}.") - shape = img_loaded.shape # Use img_loaded - if len(shape) == 3 and shape[2] == 4: - log.debug("MASK processing: Extracting alpha channel (4-channel source).") - img_prepared = img_loaded[:, :, 3] # Extract alpha from img_loaded - elif len(shape) == 3 and shape[2] == 3: - log.debug("MASK processing: Converting BGR to Grayscale (3-channel source).") # OpenCV loads as BGR - img_prepared = cv2.cvtColor(img_loaded, cv2.COLOR_BGR2GRAY) # Convert BGR to Gray - elif len(shape) == 2: - log.debug("MASK processing: Source is already grayscale.") - img_prepared = img_loaded # Keep as is - else: - log.warning(f"MASK processing: Unexpected source shape {shape}. Cannot reliably extract mask.") - img_prepared = None # Cannot process - # MASK should ideally be uint8 for saving later, but keep float for now if inverted? - # Let _save_image handle final conversion based on format rules. - else: - # For non-MASK types, start with the loaded image - img_prepared = img_loaded - # --- 2. Initial Preparation (BGR->RGB, Gloss Inversion, MASK handling) --- - img_prepared = img_loaded # Start with loaded image - - # BGR -> RGB conversion (only for 3-channel images) - if len(img_prepared.shape) == 3 and img_prepared.shape[2] >= 3: # Check for 3 or 4 channels - # Ensure it's not already grayscale before attempting conversion - if read_flag != cv2.IMREAD_GRAYSCALE: - log.debug(f"Converting loaded image from BGR to RGB for {source_path_rel.name}.") - # Handle 4-channel (BGRA) by converting to RGB first - if img_prepared.shape[2] == 4: - img_prepared = cv2.cvtColor(img_prepared, cv2.COLOR_BGRA2RGB) - else: # 3-channel (BGR) - img_prepared = cv2.cvtColor(img_prepared, cv2.COLOR_BGR2RGB) - else: - log.debug(f"Skipping BGR->RGB conversion for {source_path_rel.name} as it was loaded grayscale.") - elif len(img_prepared.shape) == 2: - log.debug(f"Image {source_path_rel.name} is grayscale, no BGR->RGB conversion needed.") - else: - log.warning(f"Unexpected image shape {img_prepared.shape} for {source_path_rel.name} after loading.") - - - # Gloss -> Roughness Inversion - if map_type == 'ROUGH' and is_gloss_source: - log.info(f"Performing Gloss->Roughness inversion for {source_path_rel.name}") - # Ensure grayscale before inversion - if len(img_prepared.shape) == 3: - img_prepared = cv2.cvtColor(img_prepared, cv2.COLOR_RGB2GRAY) # Use RGB2GRAY as it's already converted - - # Normalize based on original source dtype before inversion - if source_dtype == np.uint16: - img_float = 1.0 - (img_prepared.astype(np.float32) / 65535.0) - elif source_dtype == np.uint8: - img_float = 1.0 - (img_prepared.astype(np.float32) / 255.0) - else: # Assuming float input is already 0-1 range - img_float = 1.0 - img_prepared.astype(np.float32) - img_prepared = np.clip(img_float, 0.0, 1.0) # Result is float32 - log.debug(f"Inverted gloss map stored as float32 for ROUGH, original dtype: {source_dtype}") - - - # Ensure data is float32 for resizing if it came from gloss inversion - if isinstance(img_prepared, np.ndarray) and img_prepared.dtype != np.float32 and map_type == 'ROUGH' and is_gloss_source: - img_prepared = img_prepared.astype(np.float32) - elif isinstance(img_prepared, np.ndarray) and img_prepared.dtype not in [np.uint8, np.uint16, np.float32, np.float16]: - # Convert other potential types (like bool) to float32 for resizing compatibility - log.warning(f"Converting unexpected dtype {img_prepared.dtype} to float32 before resizing.") - img_prepared = img_prepared.astype(np.float32) - - - # --- 3. Resize --- - if img_prepared is None: raise AssetProcessingError("Image data is None after initial prep.") - orig_h, orig_w = img_prepared.shape[:2] - target_dim_px = self.config.image_resolutions.get(target_resolution_key) - if not target_dim_px: - raise AssetProcessingError(f"Target resolution key '{target_resolution_key}' not found in config.") - - # Avoid upscaling check - max_original_dimension = max(orig_w, orig_h) - if target_dim_px > max_original_dimension: - log.warning(f"Target dimension {target_dim_px}px is larger than original {max_original_dimension}px for {source_path_rel}. Skipping resize for {target_resolution_key}.") - # Store None in cache for this specific resolution to avoid retrying - cache[cache_key] = (None, source_dtype) - return None, source_dtype # Indicate resize was skipped - - if orig_w <= 0 or orig_h <= 0: - raise AssetProcessingError(f"Invalid original dimensions ({orig_w}x{orig_h}) for {source_path_rel}.") - - target_w, target_h = calculate_target_dimensions(orig_w, orig_h, target_dim_px) - interpolation = cv2.INTER_LANCZOS4 if (target_w * target_h) < (orig_w * orig_h) else cv2.INTER_CUBIC - log.debug(f"Resizing {source_path_rel.name} from ({orig_w}x{orig_h}) to ({target_w}x{target_h}) for {target_resolution_key}") - img_resized = cv2.resize(img_prepared, (target_w, target_h), interpolation=interpolation) - - # --- 4. Cache and Return --- - # Ensure result is float32 if it came from gloss inversion, otherwise keep resized dtype - final_data_to_cache = img_resized - if map_type == 'ROUGH' and is_gloss_source and final_data_to_cache.dtype != np.float32: - final_data_to_cache = final_data_to_cache.astype(np.float32) - - log.debug(f"CACHING result for {cache_key}. Shape: {final_data_to_cache.shape}, Dtype: {final_data_to_cache.dtype}") - cache[cache_key] = (final_data_to_cache, source_dtype) - return final_data_to_cache, source_dtype - - except Exception as e: - log.error(f"Error in _load_and_transform_source for {source_path_rel} at {target_resolution_key}: {e}", exc_info=True) - # Cache None to prevent retrying on error for this specific key - cache[cache_key] = (None, None) - return None, None - - - # --- New Helper Function: Save Image --- - def _save_image(self, image_data: np.ndarray, map_type: str, resolution_key: str, asset_base_name: str, source_info: dict, output_bit_depth_rule: str, temp_dir: Path) -> Optional[Dict]: - """ - Handles saving an image NumPy array to a temporary file, including determining - format, bit depth, performing final conversions, and fallback logic. - - Args: - image_data: NumPy array containing the image data to save. - map_type: The standard map type being saved (e.g., "COL", "NRMRGH"). - resolution_key: The resolution key (e.g., "4K"). - asset_base_name: The sanitized base name of the asset. - source_info: Dictionary containing details about the source(s), e.g., - {'original_extension': '.tif', 'source_bit_depth': 16, 'involved_extensions': {'.tif', '.png'}} - output_bit_depth_rule: Rule for determining output bit depth ('respect', 'force_8bit', 'force_16bit', 'respect_inputs'). - temp_dir: The temporary directory path to save the file in. - - Returns: - A dictionary containing details of the saved file (path, width, height, - bit_depth, format) or None if saving failed. - """ - if image_data is None: - log.error(f"Cannot save image for {map_type} ({resolution_key}): image_data is None.") - return None - if not temp_dir or not temp_dir.exists(): - log.error(f"Cannot save image for {map_type} ({resolution_key}): temp_dir is invalid.") - return None - - try: - h, w = image_data.shape[:2] - current_dtype = image_data.dtype - log.debug(f"Saving {map_type} ({resolution_key}) for asset '{asset_base_name}'. Input shape: {image_data.shape}, dtype: {current_dtype}") - - # --- 1. Determine Output Bit Depth --- - source_bpc = source_info.get('source_bit_depth', 8) # Default to 8 if missing - max_input_bpc = source_info.get('max_input_bit_depth', source_bpc) # For 'respect_inputs' merge rule - output_dtype_target, output_bit_depth = np.uint8, 8 # Default - - if output_bit_depth_rule == 'force_8bit': - output_dtype_target, output_bit_depth = np.uint8, 8 - elif output_bit_depth_rule == 'force_16bit': - output_dtype_target, output_bit_depth = np.uint16, 16 - elif output_bit_depth_rule == 'respect': # For individual maps - if source_bpc == 16: output_dtype_target, output_bit_depth = np.uint16, 16 - # Handle float source? Assume 16-bit output if source was float? Needs clarification. - # For now, stick to uint8/16 based on source_bpc. - elif output_bit_depth_rule == 'respect_inputs': # For merged maps - if max_input_bpc == 16: output_dtype_target, output_bit_depth = np.uint16, 16 - else: # Default to 8-bit if rule is unknown - log.warning(f"Unknown output_bit_depth_rule '{output_bit_depth_rule}'. Defaulting to 8-bit.") - output_dtype_target, output_bit_depth = np.uint8, 8 - - log.debug(f"Target output bit depth: {output_bit_depth}-bit (dtype: {output_dtype_target.__name__}) based on rule '{output_bit_depth_rule}'") - - # --- 2. Determine Output Format --- - output_format, output_ext, save_params, needs_float16 = "", "", [], False - primary_fmt_16, fallback_fmt_16 = self.config.get_16bit_output_formats() - fmt_8bit_config = self.config.get_8bit_output_format() - threshold = self.config.resolution_threshold_for_jpg - force_lossless = map_type in self.config.force_lossless_map_types - original_extension = source_info.get('original_extension', '.png') # Primary source ext - involved_extensions = source_info.get('involved_extensions', {original_extension}) # For merges - target_dim_px = self.config.image_resolutions.get(resolution_key, 0) # Get target dimension size - - # Apply format determination logic (similar to old _process_maps/_merge_maps) - if force_lossless: - log.debug(f"Format forced to lossless for map type '{map_type}'.") - if output_bit_depth == 16: - output_format = primary_fmt_16 - if output_format.startswith("exr"): - output_ext, needs_float16 = ".exr", True - save_params.extend([cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_HALF]) - else: # Assume PNG if primary 16-bit isn't EXR - if output_format != "png": log.warning(f"Primary 16-bit format '{output_format}' not PNG/EXR for forced lossless. Using fallback '{fallback_fmt_16}'.") - output_format = fallback_fmt_16 if fallback_fmt_16 == "png" else "png" # Ensure PNG - output_ext = ".png" - png_level = self.config._core_settings.get('PNG_COMPRESSION_LEVEL', 6) - save_params.extend([cv2.IMWRITE_PNG_COMPRESSION, png_level]) - else: # 8-bit lossless -> PNG - output_format = "png"; output_ext = ".png" - png_level = self.config._core_settings.get('PNG_COMPRESSION_LEVEL', 6) - save_params = [cv2.IMWRITE_PNG_COMPRESSION, png_level] - - elif output_bit_depth == 8 and target_dim_px >= threshold: - output_format = 'jpg'; output_ext = '.jpg' - jpg_quality = self.config.jpg_quality - save_params.extend([cv2.IMWRITE_JPEG_QUALITY, jpg_quality]) - log.debug(f"Using JPG format (Quality: {jpg_quality}) for {map_type} at {resolution_key} due to resolution threshold ({target_dim_px} >= {threshold}).") - else: - # Determine highest format involved (for merges) or use original (for individuals) - highest_format_str = 'jpg' # Default lowest - relevant_extensions = involved_extensions if map_type in self.config.map_merge_rules else {original_extension} - if '.exr' in relevant_extensions: highest_format_str = 'exr' - elif '.tif' in relevant_extensions: highest_format_str = 'tif' - elif '.png' in relevant_extensions: highest_format_str = 'png' - - if highest_format_str == 'exr': - if output_bit_depth == 16: output_format, output_ext, needs_float16 = "exr", ".exr", True; save_params.extend([cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_HALF]) - else: output_format, output_ext = "png", ".png"; save_params.extend([cv2.IMWRITE_PNG_COMPRESSION, self.config._core_settings.get('PNG_COMPRESSION_LEVEL', 6)]) - elif highest_format_str == 'tif': - if output_bit_depth == 16: - output_format = primary_fmt_16 - if output_format.startswith("exr"): output_ext, needs_float16 = ".exr", True; save_params.extend([cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_HALF]) - else: output_format = "png"; output_ext = ".png"; save_params.extend([cv2.IMWRITE_PNG_COMPRESSION, self.config._core_settings.get('PNG_COMPRESSION_LEVEL', 6)]) - else: output_format, output_ext = "png", ".png"; save_params.extend([cv2.IMWRITE_PNG_COMPRESSION, self.config._core_settings.get('PNG_COMPRESSION_LEVEL', 6)]) - elif highest_format_str == 'png': - if output_bit_depth == 16: - output_format = primary_fmt_16 - if output_format.startswith("exr"): output_ext, needs_float16 = ".exr", True; save_params.extend([cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_HALF]) - else: output_format = "png"; output_ext = ".png"; save_params.extend([cv2.IMWRITE_PNG_COMPRESSION, self.config._core_settings.get('PNG_COMPRESSION_LEVEL', 6)]) - else: output_format, output_ext = "png", ".png"; save_params.extend([cv2.IMWRITE_PNG_COMPRESSION, self.config._core_settings.get('PNG_COMPRESSION_LEVEL', 6)]) - else: # Default to configured 8-bit format if highest was JPG or unknown - output_format = fmt_8bit_config; output_ext = f".{output_format}" - if output_format == "png": save_params.extend([cv2.IMWRITE_PNG_COMPRESSION, self.config._core_settings.get('PNG_COMPRESSION_LEVEL', 6)]) - elif output_format == "jpg": save_params.extend([cv2.IMWRITE_JPEG_QUALITY, self.config.jpg_quality]) - - # Final check: JPG must be 8-bit - if output_format == "jpg" and output_bit_depth == 16: - log.warning(f"Output format is JPG, but target bit depth is 16. Forcing 8-bit for {map_type} ({resolution_key}).") - output_dtype_target, output_bit_depth = np.uint8, 8 - - log.debug(f"Determined save format: {output_format}, ext: {output_ext}, bit_depth: {output_bit_depth}, needs_float16: {needs_float16}") - - # --- 3. Final Data Type Conversion --- - img_to_save = image_data.copy() # Work on a copy - if output_dtype_target == np.uint8 and img_to_save.dtype != np.uint8: - log.debug(f"Converting image data from {img_to_save.dtype} to uint8 for saving.") - if img_to_save.dtype == np.uint16: img_to_save = (img_to_save.astype(np.float32) / 65535.0 * 255.0).astype(np.uint8) - elif img_to_save.dtype in [np.float16, np.float32]: img_to_save = (np.clip(img_to_save, 0.0, 1.0) * 255.0).astype(np.uint8) - else: img_to_save = img_to_save.astype(np.uint8) # Direct cast for other types (e.g., bool) - elif output_dtype_target == np.uint16 and img_to_save.dtype != np.uint16: - log.debug(f"Converting image data from {img_to_save.dtype} to uint16 for saving.") - if img_to_save.dtype == np.uint8: img_to_save = img_to_save.astype(np.uint16) * 257 # Proper 8->16 bit scaling - elif img_to_save.dtype in [np.float16, np.float32]: img_to_save = (np.clip(img_to_save, 0.0, 1.0) * 65535.0).astype(np.uint16) - else: img_to_save = img_to_save.astype(np.uint16) - if needs_float16 and img_to_save.dtype != np.float16: - log.debug(f"Converting image data from {img_to_save.dtype} to float16 for EXR saving.") - if img_to_save.dtype == np.uint16: img_to_save = (img_to_save.astype(np.float32) / 65535.0).astype(np.float16) - elif img_to_save.dtype == np.uint8: img_to_save = (img_to_save.astype(np.float32) / 255.0).astype(np.float16) - elif img_to_save.dtype == np.float32: img_to_save = img_to_save.astype(np.float16) - else: log.warning(f"Cannot convert {img_to_save.dtype} to float16 for EXR save."); return None - - # --- 4. Final Color Space Conversion (RGB -> BGR for non-EXR) --- - img_save_final = img_to_save - is_3_channel = len(img_to_save.shape) == 3 and img_to_save.shape[2] == 3 - if is_3_channel and not output_format.startswith("exr"): - log.debug(f"Converting RGB to BGR for saving {map_type} ({resolution_key}) as {output_format}") - try: - img_save_final = cv2.cvtColor(img_to_save, cv2.COLOR_RGB2BGR) - except Exception as cvt_err: - log.error(f"Failed RGB->BGR conversion before save for {map_type} ({resolution_key}): {cvt_err}. Saving original RGB.") - img_save_final = img_to_save # Fallback - - # --- 5. Construct Filename & Save --- - filename = self.config.target_filename_pattern.format( - base_name=asset_base_name, - map_type=map_type, - resolution=resolution_key, - ext=output_ext.lstrip('.') - ) - output_path_temp = temp_dir / filename - log.debug(f"Attempting to save: {output_path_temp.name} (Format: {output_format}, Dtype: {img_save_final.dtype})") - - saved_successfully = False - actual_format_saved = output_format - try: - cv2.imwrite(str(output_path_temp), img_save_final, save_params) - saved_successfully = True - log.info(f" > Saved {map_type} ({resolution_key}, {output_bit_depth}-bit) as {output_format}") - except Exception as save_err: - log.error(f"Save failed ({output_format}) for {map_type} {resolution_key}: {save_err}") - # --- Try Fallback --- - if output_bit_depth == 16 and output_format.startswith("exr") and fallback_fmt_16 != output_format and fallback_fmt_16 == "png": - log.warning(f"Attempting fallback PNG save for {map_type} {resolution_key}") - actual_format_saved = "png"; output_ext = ".png"; - filename = self.config.target_filename_pattern.format(base_name=asset_base_name, map_type=map_type, resolution=resolution_key, ext="png") - output_path_temp = temp_dir / filename - save_params_fallback = [cv2.IMWRITE_PNG_COMPRESSION, self.config._core_settings.get('PNG_COMPRESSION_LEVEL', 6)] - img_fallback = None; target_fallback_dtype = np.uint16 - - # Convert original data (before float16 conversion) to uint16 for PNG fallback - if img_to_save.dtype == np.float16: # This means original was likely float or uint16/8 converted to float16 - # Need to get back to uint16 - use the pre-float16 converted data if possible? - # Safest is to convert the float16 back to uint16 - img_scaled = np.clip(img_to_save.astype(np.float32) * 65535.0, 0, 65535) - img_fallback = img_scaled.astype(target_fallback_dtype) - elif img_to_save.dtype == target_fallback_dtype: img_fallback = img_to_save # Already uint16 - else: log.error(f"Cannot convert {img_to_save.dtype} for PNG fallback."); return None - - # --- Conditional RGB -> BGR Conversion for fallback --- - img_fallback_save_final = img_fallback - is_3_channel_fallback = len(img_fallback.shape) == 3 and img_fallback.shape[2] == 3 - if is_3_channel_fallback: # PNG is non-EXR - log.debug(f"Converting RGB to BGR for fallback PNG save {map_type} ({resolution_key})") - try: img_fallback_save_final = cv2.cvtColor(img_fallback, cv2.COLOR_RGB2BGR) - except Exception as cvt_err_fb: log.error(f"Failed RGB->BGR conversion for fallback PNG: {cvt_err_fb}. Saving original.") - - try: - cv2.imwrite(str(output_path_temp), img_fallback_save_final, save_params_fallback) - saved_successfully = True - log.info(f" > Saved {map_type} ({resolution_key}) using fallback PNG") - except Exception as fallback_err: - log.error(f"Fallback PNG save failed for {map_type} {resolution_key}: {fallback_err}", exc_info=True) - else: - log.error(f"No suitable fallback available or applicable for failed save of {map_type} ({resolution_key}) as {output_format}.") - - - # --- 6. Return Result --- - if saved_successfully: - return { - "path": output_path_temp.relative_to(self.temp_dir), # Store relative path - "resolution": resolution_key, - "width": w, "height": h, - "bit_depth": output_bit_depth, - "format": actual_format_saved - } - else: - return None # Indicate save failure - - except Exception as e: - log.error(f"Unexpected error in _save_image for {map_type} ({resolution_key}): {e}", exc_info=True) - return None - - def process(self) -> Dict[str, List[str]]: - """ - Executes the full processing pipeline for the input path, handling - multiple assets within a single input if detected. - - Returns: - Dict[str, List[str]]: A dictionary summarizing the status of each - detected asset within the input: - {"processed": [asset_name1, ...], - "skipped": [asset_name2, ...], - "failed": [asset_name3, ...]} - """ - log.info(f"Starting processing for input: {self.input_path.name}") - overall_status = {"processed": [], "skipped": [], "failed": []} - supplier_name = self.config.supplier_name # Get once - loaded_data_cache = {} # Initialize cache for this process call - - try: - self._setup_workspace() - self._extract_input() - self._inventory_and_classify_files() # Classifies all files in self.classified_files - - # Determine distinct assets and file mapping - distinct_base_names, file_to_base_name_map = self._determine_base_metadata() - unmatched_files_paths = [p for p, name in file_to_base_name_map.items() if name is None] - if unmatched_files_paths: - log.warning(f"Found {len(unmatched_files_paths)} files not matched to any specific asset base name. They will be copied to each asset's Extra folder.") - log.debug(f"Unmatched files: {[str(p) for p in unmatched_files_paths]}") - - - # --- Loop through each detected asset --- - for current_asset_name in distinct_base_names: - log.info(f"--- Processing detected asset: '{current_asset_name}' ---") - asset_processed = False - asset_skipped = False - asset_failed = False - temp_metadata_path_asset = None # Track metadata file for this asset - map_details_asset = {} # Store map details for this asset - - try: - # --- Filter classified files for the current asset --- - filtered_classified_files_asset = defaultdict(list) - for category, file_list in self.classified_files.items(): - for file_info in file_list: - file_path = file_info.get('source_path') - if file_path and file_to_base_name_map.get(file_path) == current_asset_name: - filtered_classified_files_asset[category].append(file_info) - log.debug(f"Asset '{current_asset_name}': Filtered files - Maps: {len(filtered_classified_files_asset.get('maps',[]))}, Models: {len(filtered_classified_files_asset.get('models',[]))}, Extra: {len(filtered_classified_files_asset.get('extra',[]))}, Ignored: {len(filtered_classified_files_asset.get('ignored',[]))}") - - # --- Assign Suffixes Per-Asset --- - log.debug(f"Asset '{current_asset_name}': Assigning map type suffixes...") - asset_maps = filtered_classified_files_asset.get('maps', []) - grouped_asset_maps = defaultdict(list) - for map_info in asset_maps: - # Group by the base map type stored earlier - grouped_asset_maps[map_info['map_type']].append(map_info) - - for base_map_type, maps_in_group in grouped_asset_maps.items(): - log.debug(f" Assigning suffixes for base type '{base_map_type}' within asset '{current_asset_name}' ({len(maps_in_group)} maps)") - # Sorting is already done by _inventory_and_classify_files, just need to assign suffix - respect_variants = base_map_type in self.config.respect_variant_map_types - for i, map_info in enumerate(maps_in_group): - if respect_variants: - final_map_type = f"{base_map_type}-{i + 1}" - else: - final_map_type = base_map_type - log.debug(f" Updating '{map_info['source_path']}' map_type from '{map_info['map_type']}' to '{final_map_type}'") - map_info['map_type'] = final_map_type # Update the map_type in the dictionary - - # --- Determine Metadata for this specific asset --- - asset_specific_metadata = self._determine_single_asset_metadata(current_asset_name, filtered_classified_files_asset) - current_asset_metadata = { - "asset_name": current_asset_name, - "supplier_name": supplier_name, - "asset_category": asset_specific_metadata.get("asset_category", self.config.default_asset_category), - "archetype": asset_specific_metadata.get("archetype", "Unknown"), - # Initialize fields that will be populated by processing steps - "maps_present": [], - "merged_maps": [], - "shader_features": [], - "source_files_in_extra": [], # Will be populated in _generate_metadata - "image_stats_1k": {}, - "map_details": {}, # Will be populated by _process_maps - "aspect_ratio_change_string": "N/A" - } - - # --- Skip Check for this specific asset --- - if not self.overwrite: - supplier_sanitized = self._sanitize_filename(supplier_name) - asset_name_sanitized = self._sanitize_filename(current_asset_name) - final_dir = self.output_base_path / supplier_sanitized / asset_name_sanitized - metadata_file_path = final_dir / self.config.metadata_filename - if final_dir.exists() and metadata_file_path.is_file(): - log.info(f"Output directory and metadata found for asset '{asset_name_sanitized}' and overwrite is False. Skipping this asset.") - overall_status["skipped"].append(current_asset_name) - asset_skipped = True - continue # Skip to the next asset in the loop - elif self.overwrite: - log.info(f"Overwrite flag is set. Processing asset '{current_asset_name}' even if output exists.") - - # --- Process Individual Maps for this asset --- - processed_maps_details_asset, image_stats_asset, aspect_ratio_change_string_asset, ignored_rough_maps = self._process_individual_maps( - filtered_maps_list=filtered_classified_files_asset.get('maps', []), - current_asset_metadata=current_asset_metadata, # Pass base metadata - loaded_data_cache=loaded_data_cache # Pass cache - ) - # Update current metadata with results - current_asset_metadata["image_stats_1k"] = image_stats_asset - current_asset_metadata["aspect_ratio_change_string"] = aspect_ratio_change_string_asset - # Add newly ignored rough maps to the asset's specific ignored list - if ignored_rough_maps: - filtered_classified_files_asset['ignored'].extend(ignored_rough_maps) - # Store map details (like source bit depth) collected during processing - # This was previously stored in self.metadata["map_details"] - map_details_asset = {k: v for k, v in current_asset_metadata.pop("map_details", {}).items() if k in processed_maps_details_asset} - - - # --- Merge Maps from Source for this asset --- - merged_maps_details_asset = self._merge_maps_from_source( - processed_maps_details_asset=processed_maps_details_asset, # Still needed for source info lookup? Or pass classified files? Check impl. - filtered_classified_files=filtered_classified_files_asset, - current_asset_metadata=current_asset_metadata, - loaded_data_cache=loaded_data_cache # Pass cache - ) - - # --- Generate Metadata for this asset --- - temp_metadata_path_asset = self._generate_metadata_file( - current_asset_metadata=current_asset_metadata, # Pass the populated dict - processed_maps_details_asset=processed_maps_details_asset, - merged_maps_details_asset=merged_maps_details_asset, - filtered_classified_files_asset=filtered_classified_files_asset, - unmatched_files_paths=unmatched_files_paths, # Pass the list of unmatched files - map_details_asset=map_details_asset # Pass the filtered map details - ) - - # --- Organize Output Files for this asset --- - self._organize_output_files( - current_asset_name=current_asset_name, - processed_maps_details_asset=processed_maps_details_asset, - merged_maps_details_asset=merged_maps_details_asset, - filtered_classified_files_asset=filtered_classified_files_asset, - unmatched_files_paths=unmatched_files_paths, # Pass unmatched files for copying - temp_metadata_path=temp_metadata_path_asset - ) - - log.info(f"--- Asset '{current_asset_name}' processed successfully. ---") - overall_status["processed"].append(current_asset_name) - asset_processed = True - - except Exception as asset_err: - log.error(f"--- Failed processing asset '{current_asset_name}': {asset_err} ---", exc_info=True) - overall_status["failed"].append(current_asset_name) - asset_failed = True - # Continue to the next asset even if one fails - - # --- Determine Final Consolidated Status --- - # This logic remains the same, interpreting the overall_status dict - final_status = "failed" # Default if nothing else matches - if overall_status["processed"] and not overall_status["failed"]: - final_status = "processed" - elif overall_status["skipped"] and not overall_status["processed"] and not overall_status["failed"]: - final_status = "skipped" - elif overall_status["processed"] and overall_status["failed"]: - final_status = "partial_success" # Indicate some succeeded, some failed - elif overall_status["processed"] and overall_status["skipped"] and not overall_status["failed"]: - final_status = "processed" # Consider processed+skipped as processed overall - elif overall_status["skipped"] and overall_status["failed"] and not overall_status["processed"]: - final_status = "failed" # If only skips and fails, report as failed - # Add any other combinations if needed - - log.info(f"Finished processing input '{self.input_path.name}'. Overall Status: {final_status}. Summary: {overall_status}") - # Return the detailed status dictionary instead of just a string - # The wrapper function in main.py will interpret this - return overall_status - - except Exception as e: - # Catch errors during initial setup (before asset loop) - if not isinstance(e, (AssetProcessingError, ConfigurationError)): - log.exception(f"Asset processing failed unexpectedly for {self.input_path.name} during setup: {e}") - if not isinstance(e, AssetProcessingError): - raise AssetProcessingError(f"Failed processing {self.input_path.name}: {e}") from e - else: - raise - finally: - # Ensure cleanup always happens - self._cleanup_workspace() - - def _setup_workspace(self): - """Creates a temporary directory for processing.""" - try: - self.temp_dir = Path(tempfile.mkdtemp(prefix=self.config.temp_dir_prefix)) - log.debug(f"Created temporary workspace: {self.temp_dir}") - except Exception as e: - raise AssetProcessingError(f"Failed to create temporary workspace: {e}") from e - - def _extract_input(self): - """Extracts ZIP or copies folder contents to the temporary workspace.""" - if not self.temp_dir: - raise AssetProcessingError("Temporary workspace not setup before extraction.") - - log.info(f"Preparing source files from {self.input_path.name}...") - try: - if self.input_path.is_file(): - suffix = self.input_path.suffix.lower() - if suffix == '.zip': - log.debug(f"Extracting ZIP file: {self.input_path}") - with zipfile.ZipFile(self.input_path, 'r') as zip_ref: - zip_ref.extractall(self.temp_dir) - log.info(f"ZIP extracted to {self.temp_dir}") - elif suffix == '.rar': - log.debug(f"Extracting RAR file: {self.input_path}") - # rarfile requires unrar to be installed and in the system's PATH - # We assume this is handled by the user's environment setup. - # Basic error handling for common rarfile exceptions. - try: - with rarfile.RarFile(self.input_path, 'r') as rar_ref: - rar_ref.extractall(self.temp_dir) - log.info(f"RAR extracted to {self.temp_dir}") - except rarfile.BadRarFile: - raise AssetProcessingError(f"Input file is not a valid RAR archive: {self.input_path.name}") - except rarfile.NeedFirstVolume: - raise AssetProcessingError(f"RAR archive is part of a multi-volume set, but the first volume is missing: {self.input_path.name}") - except rarfile.PasswordRequired: - # As per plan, we don't handle passwords at this stage - raise AssetProcessingError(f"RAR archive is password protected. Skipping: {self.input_path.name}") - except rarfile.NoRarEntry: - raise AssetProcessingError(f"RAR archive is empty or corrupted: {self.input_path.name}") - except Exception as rar_err: - # Catch any other unexpected rarfile errors - raise AssetProcessingError(f"Failed to extract RAR archive {self.input_path.name}: {rar_err}") from rar_err - - elif suffix == '.7z': - log.debug(f"Extracting 7z file: {self.input_path}") - # py7zr handles extraction directly - try: - with py7zr.SevenZipFile(self.input_path, mode='r') as sz_ref: - sz_ref.extractall(path=self.temp_dir) - log.info(f"7z extracted to {self.temp_dir}") - except py7zr.Bad7zFile: - raise AssetProcessingError(f"Input file is not a valid 7z archive: {self.input_path.name}") - except py7zr.PasswordRequired: - # As per plan, we don't handle passwords at this stage - raise AssetProcessingError(f"7z archive is password protected. Skipping: {self.input_path.name}") - except Exception as sz_err: - # Catch any other unexpected py7zr errors - raise AssetProcessingError(f"Failed to extract 7z archive {self.input_path.name}: {sz_err}") from sz_err - - else: - # If it's a file but not zip, rar, or 7z, treat it as an error for now - # Or could add logic to copy single files? Plan says zip or folder. - raise AssetProcessingError(f"Input file is not a supported archive type (.zip, .rar, .7z): {self.input_path.name}") - - elif self.input_path.is_dir(): - log.debug(f"Copying directory contents: {self.input_path}") - for item in self.input_path.iterdir(): - destination = self.temp_dir / item.name - if item.is_dir(): - # Use dirs_exist_ok=True for robustness if Python version supports it (3.8+) - try: - shutil.copytree(item, destination, dirs_exist_ok=True) - except TypeError: # Fallback for older Python - if not destination.exists(): - shutil.copytree(item, destination) - else: - log.warning(f"Subdirectory '{item.name}' already exists in temp dir, skipping copytree (potential issue on older Python).") - - else: - shutil.copy2(item, destination) - log.info(f"Directory contents copied to {self.temp_dir}") - else: - # This case should be caught by __init__ but included for robustness - raise AssetProcessingError(f"Input path must be a directory or a supported archive file (.zip, .rar, .7z): {self.input_path}") - - except AssetProcessingError: - # Re-raise our custom exception directly - raise - except Exception as e: - # Wrap any other unexpected exceptions - raise AssetProcessingError(f"An unexpected error occurred during input extraction for {self.input_path.name}: {e}") from e - - def _inventory_and_classify_files(self): - """ - Scans workspace, classifies files according to preset rules, handling - 16-bit prioritization and multiple variants of the same base map type. - """ - if not self.temp_dir: - raise AssetProcessingError("Temporary workspace not setup before inventory.") - - log.info("Scanning and classifying files...") - log.debug("--- Starting File Inventory and Classification (v2) ---") - all_files_rel = [] - for root, _, files in os.walk(self.temp_dir): - root_path = Path(root) - for file in files: - full_path = root_path / file - relative_path = full_path.relative_to(self.temp_dir) - all_files_rel.append(relative_path) - - log.debug(f"Found {len(all_files_rel)} files in workspace: {[str(p) for p in all_files_rel]}") - - # --- Initialization --- - processed_files = set() # Track relative paths handled (Extra, Models, Ignored, Final Maps) - potential_map_candidates = [] # List to store potential map file info - # Reset classified files (important if this method is ever called multiple times) - self.classified_files = {"maps": [], "models": [], "extra": [], "ignored": []} - - - # --- Step 1: Identify Explicit 'Extra' Files --- - log.debug("Step 1: Checking for files to move to 'Extra' (using regex)...") - compiled_extra_regex = getattr(self.config, 'compiled_extra_regex', []) - log.debug(f" Compiled 'Extra' regex patterns: {[r.pattern for r in compiled_extra_regex]}") - for file_rel_path in all_files_rel: - if file_rel_path in processed_files: continue - for compiled_regex in compiled_extra_regex: - if compiled_regex.search(file_rel_path.name): - log.debug(f" REGEX MATCH FOUND: Marking '{file_rel_path}' for 'Extra' folder based on pattern '{compiled_regex.pattern}'.") - self.classified_files["extra"].append({'source_path': file_rel_path, 'reason': f'Regex match: {compiled_regex.pattern}'}) - processed_files.add(file_rel_path) - log.debug(f" Added '{file_rel_path}' to processed files.") - break # Stop checking extra patterns for this file - - # --- Step 2: Identify Model Files --- - log.debug("Step 2: Identifying model files (using regex)...") - compiled_model_regex = getattr(self.config, 'compiled_model_regex', []) - log.debug(f" Compiled 'Model' regex patterns: {[r.pattern for r in compiled_model_regex]}") - for file_rel_path in all_files_rel: - if file_rel_path in processed_files: continue - for compiled_regex in compiled_model_regex: - if compiled_regex.search(file_rel_path.name): - log.debug(f" REGEX MATCH FOUND: Identified '{file_rel_path}' as model file based on pattern '{compiled_regex.pattern}'.") - self.classified_files["models"].append({'source_path': file_rel_path}) - processed_files.add(file_rel_path) - log.debug(f" Added '{file_rel_path}' to processed files.") - break # Stop checking model patterns for this file - - # --- Step 3: Gather Potential Map Candidates (Refactored) --- - log.debug("Step 3: Gathering potential map candidates (iterating files first)...") - # Compiled map keyword regex now maps: base_type -> [(regex, keyword, rule_index), ...] - compiled_map_keyword_regex_tuples = getattr(self.config, 'compiled_map_keyword_regex', {}) - - for file_rel_path in all_files_rel: - # Skip files already classified as Extra or Model - if file_rel_path in processed_files: - continue - - file_stem = file_rel_path.stem - match_found = False - - # Iterate through base types and their associated regex tuples - for base_map_type, regex_tuples in compiled_map_keyword_regex_tuples.items(): - if match_found: break # Stop checking types for this file once matched - - # Get the original keywords list for the current rule index - # Assuming self.config.map_type_mapping holds the original list of dicts from JSON - original_rule = None - # Find the rule based on the first tuple's rule_index (they should all be the same for this base_map_type) - if regex_tuples: - current_rule_index = regex_tuples[0][2] # Get rule_index from the first tuple - if hasattr(self.config, 'map_type_mapping') and current_rule_index < len(self.config.map_type_mapping): - rule_candidate = self.config.map_type_mapping[current_rule_index] - # Verify it's the correct rule by checking target_type - if rule_candidate.get("target_type") == base_map_type: - original_rule = rule_candidate - else: - log.warning(f"Rule index mismatch for {base_map_type} at index {current_rule_index}. Searching...") - # Fallback search if index doesn't match (shouldn't happen ideally) - for idx, rule in enumerate(self.config.map_type_mapping): - if rule.get("target_type") == base_map_type: - original_rule = rule - log.warning(f"Found rule for {base_map_type} at index {idx} instead.") - break - - original_keywords_list = [] - if original_rule and 'keywords' in original_rule: - original_keywords_list = original_rule['keywords'] - else: - log.warning(f"Could not find original keywords list for rule matching base_map_type '{base_map_type}'. Keyword indexing may fail.") - - for kw_regex, original_keyword, rule_index in regex_tuples: - if kw_regex.search(file_stem): - log.debug(f" Match found: '{file_rel_path}' matches keyword '{original_keyword}' (rule {rule_index}, pattern: '{kw_regex.pattern}') for base type '{base_map_type}'") - - # Find the index of the matched keyword within its rule's list - keyword_index_in_rule = -1 # Default if not found - if original_keywords_list: - try: - # Use the original_keyword string directly - keyword_index_in_rule = original_keywords_list.index(original_keyword) - except ValueError: - log.warning(f"Keyword '{original_keyword}' not found in its original rule list? {original_keywords_list}") - else: - log.warning(f"Original keywords list empty for rule {rule_index}, cannot find index for '{original_keyword}'.") - - # Add candidate only if not already added - if not any(c['source_path'] == file_rel_path for c in potential_map_candidates): - potential_map_candidates.append({ - 'source_path': file_rel_path, - 'matched_keyword': original_keyword, - 'base_map_type': base_map_type, - 'preset_rule_index': rule_index, - 'keyword_index_in_rule': keyword_index_in_rule, # <<< STORE THE KEYWORD INDEX - 'is_16bit_source': False - }) - else: - log.warning(f" '{file_rel_path}' was already added as a candidate? Skipping duplicate add.") - - match_found = True - break # Stop checking regex tuples for this base_type once matched - - log.debug(f"Gathered {len(potential_map_candidates)} potential map candidates based on keywords.") - - # --- Step 3.5: Identify Standalone 16-bit Variants (Not caught by keywords) --- - log.debug("Step 3.5: Checking for standalone 16-bit variants...") - compiled_bit_depth_regex = getattr(self.config, 'compiled_bit_depth_regex_map', {}) - for file_rel_path in all_files_rel: - # Skip if already processed or already identified as a candidate - if file_rel_path in processed_files or any(c['source_path'] == file_rel_path for c in potential_map_candidates): - continue - - for base_type, compiled_regex in compiled_bit_depth_regex.items(): - log.debug(f" Step 3.5: Checking file '{file_rel_path.name}' against 16-bit pattern for '{base_type}': {compiled_regex.pattern}") # ADDED LOG - match = compiled_regex.search(file_rel_path.name) # Store result - if match: - log.debug(f" --> MATCH FOUND for standalone 16-bit variant: '{file_rel_path}' for base type '{base_type}'") # MODIFIED LOG - potential_map_candidates.append({ - 'source_path': file_rel_path, - 'matched_keyword': 'N/A (16bit variant)', # Placeholder keyword - 'base_map_type': base_type, - 'preset_rule_index': 9999, # Assign high index to avoid interfering with keyword priority - 'is_16bit_source': True # Mark as 16-bit immediately - }) - log.debug(f" Added candidate: {potential_map_candidates[-1]}") - # Don't add to processed_files yet, let Step 4 handle filtering - break # Stop checking bit depth patterns for this file - - log.debug(f"Total potential map candidates after checking standalone 16-bit: {len(potential_map_candidates)}") - - # --- Step 4: Prioritize 16-bit Variants & Filter Candidates --- - log.debug("Step 4: Prioritizing 16-bit variants and filtering candidates...") - compiled_bit_depth_regex = getattr(self.config, 'compiled_bit_depth_regex_map', {}) - candidates_to_keep = [] - candidates_to_ignore = [] # Store 8-bit versions superseded by 16-bit - - # Mark 16-bit candidates - for candidate in potential_map_candidates: - base_type = candidate['base_map_type'] - # Check if the base type exists in the bit depth map AND the filename matches the regex - if base_type in compiled_bit_depth_regex: - if compiled_bit_depth_regex[base_type].search(candidate['source_path'].name): - candidate['is_16bit_source'] = True - log.debug(f" Marked '{candidate['source_path']}' as 16-bit source for base type '{base_type}'.") - - - # Identify base types that have a 16-bit version present - prioritized_16bit_bases = { - candidate['base_map_type'] for candidate in potential_map_candidates if candidate['is_16bit_source'] - } - log.debug(f" Base map types with 16-bit variants found: {prioritized_16bit_bases}") - - # Filter: Keep 16-bit versions, or 8-bit versions if no 16-bit exists for that base type - for candidate in potential_map_candidates: - if candidate['is_16bit_source']: - candidates_to_keep.append(candidate) - log.debug(f" Keeping 16-bit candidate: {candidate['source_path']} ({candidate['base_map_type']})") - elif candidate['base_map_type'] not in prioritized_16bit_bases: - candidates_to_keep.append(candidate) - log.debug(f" Keeping 8-bit candidate (no 16-bit found): {candidate['source_path']} ({candidate['base_map_type']})") - else: - # This is an 8-bit candidate whose 16-bit counterpart exists - candidates_to_ignore.append(candidate) - log.debug(f" Ignoring 8-bit candidate (16-bit found): {candidate['source_path']} ({candidate['base_map_type']})") - - # Add ignored 8-bit files to the main ignored list - for ignored_candidate in candidates_to_ignore: - self.classified_files["ignored"].append({ - 'source_path': ignored_candidate['source_path'], - 'reason': f'Superseded by 16bit variant for {ignored_candidate["base_map_type"]}' - }) - processed_files.add(ignored_candidate['source_path']) # Mark as processed - - log.debug(f"Filtered candidates. Keeping: {len(candidates_to_keep)}, Ignored: {len(candidates_to_ignore)}") - - # --- Step 5: Group, Sort, Assign Suffixes, and Finalize Maps --- - log.debug("Step 5: Grouping, sorting, assigning suffixes, and finalizing maps...") - # from collections import defaultdict # Moved import to top of file - grouped_by_base_type = defaultdict(list) - for candidate in candidates_to_keep: - grouped_by_base_type[candidate['base_map_type']].append(candidate) - - final_map_list = [] - for base_map_type, candidates in grouped_by_base_type.items(): - # --- DIAGNOSTIC LOGGING START --- - candidate_paths_str = [str(c['source_path']) for c in candidates] - log.debug(f" [DIAGNOSIS] Processing base_map_type: '{base_map_type}'. Candidates before sort: {candidate_paths_str}") - # --- DIAGNOSTIC LOGGING END --- - log.debug(f" Processing final candidates for base type: '{base_map_type}' ({len(candidates)} candidates)") - - # --- NEW SORTING LOGIC --- - # Sort candidates based on: - # 1. The index of the rule object in the preset's map_type_mapping list. - # 2. The index of the matched keyword within that rule object's 'keywords' list. - # 3. Alphabetical order of the source file path as a tie-breaker. - candidates.sort(key=lambda c: ( - c.get('preset_rule_index', 9999), # Use get with fallback for safety - c.get('keyword_index_in_rule', 9999), # Use get with fallback for safety - str(c['source_path']) - )) - # --- END NEW SORTING LOGIC --- - - # Removed diagnostic log - - # Add sorted candidates to the final list, but without assigning the suffix yet. - # Suffix assignment will happen per-asset later. - for final_candidate in candidates: # Use the directly sorted list - # Store the base map type for now. - final_map_list.append({ - "map_type": base_map_type, # Store BASE type only - "source_path": final_candidate["source_path"], - "source_keyword": final_candidate["matched_keyword"], - "is_16bit_source": final_candidate["is_16bit_source"], - "original_extension": final_candidate["source_path"].suffix.lower() # Store original extension - }) - processed_files.add(final_candidate["source_path"]) # Mark final map source as processed - - self.classified_files["maps"] = final_map_list - - # --- Step 6: Classify Remaining Files as 'Unrecognised' (in 'Extra') --- - log.debug("Step 6: Classifying remaining files as 'Unrecognised'...") - remaining_count = 0 - for file_rel_path in all_files_rel: - if file_rel_path not in processed_files: - log.debug(f" Marking remaining file '{file_rel_path}' for 'Extra' folder (Unrecognised).") - self.classified_files["extra"].append({'source_path': file_rel_path, 'reason': 'Unrecognised'}) - remaining_count += 1 - # No need to add to processed_files here, it's the final step - log.debug(f" Marked {remaining_count} remaining files as 'Unrecognised'.") - - # --- Final Summary --- - # Note: self.metadata["source_files_in_extra"] is now populated per-asset in _generate_metadata_file - log.info(f"File classification complete.") - log.debug("--- Final Classification Summary (v2) ---") - map_details_log = [f"{m['map_type']}:{m['source_path']}" for m in self.classified_files["maps"]] - model_details_log = [str(f['source_path']) for f in self.classified_files["models"]] - extra_details_log = [f"{str(f['source_path'])} ({f['reason']})" for f in self.classified_files["extra"]] - ignored_details_log = [f"{str(f['source_path'])} ({f['reason']})" for f in self.classified_files["ignored"]] - log.debug(f" Identified Maps ({len(self.classified_files['maps'])}): {map_details_log}") - log.debug(f" Model Files ({len(self.classified_files['models'])}): {model_details_log}") - log.debug(f" Extra/Unrecognised Files ({len(self.classified_files['extra'])}): {extra_details_log}") - log.debug(f" Ignored Files ({len(self.classified_files['ignored'])}): {ignored_details_log}") - log.debug("--- End File Inventory and Classification (v2) ---") - - - def _determine_base_metadata(self) -> Tuple[List[str], Dict[Path, Optional[str]]]: - """ - Determines distinct asset base names within the input based on preset rules - and maps each relevant source file to its determined base name. - - Returns: - Tuple[List[str], Dict[Path, Optional[str]]]: - - A list of unique, sanitized base names found. - - A dictionary mapping source file relative paths to their determined - base name string (or None if no base name could be determined for that file). - """ - if not self.temp_dir: raise AssetProcessingError("Workspace not setup.") - log.info("Determining distinct base names and file mapping...") - - # Combine map and model files for base name determination - relevant_files = self.classified_files.get('maps', []) + self.classified_files.get('models', []) - if not relevant_files: - log.warning("No map or model files found to determine base name(s).") - # Fallback: Use input path name as a single asset - input_name = self.input_path.stem if self.input_path.is_file() else self.input_path.name - sanitized_input_name = self._sanitize_filename(input_name or "UnknownInput") - # Map all files (maps, models, extra, ignored) to this fallback name - all_files_paths = [f['source_path'] for cat in self.classified_files.values() for f in cat if 'source_path' in f] - file_to_base_name_map = {f_path: sanitized_input_name for f_path in all_files_paths} - log.info(f"Using input path name '{sanitized_input_name}' as the single asset name.") - return [sanitized_input_name], file_to_base_name_map - - # --- Determine Base Names from Files --- - separator = self.config.source_naming_separator - indices_dict = self.config.source_naming_indices - base_index_raw = indices_dict.get('base_name') - base_index = None - if base_index_raw is not None: - try: - base_index = int(base_index_raw) - except (ValueError, TypeError): - log.warning(f"Could not convert base_name index '{base_index_raw}' to integer. Base name determination might be inaccurate.") - - file_to_base_name_map: Dict[Path, Optional[str]] = {} - potential_base_names_per_file: Dict[Path, str] = {} # Store potential name for each file path - - if isinstance(base_index, int): - log.debug(f"Attempting base name extraction using separator '{separator}' and index {base_index}.") - for file_info in relevant_files: - file_path = file_info['source_path'] - stem = file_path.stem - parts = stem.split(separator) - if len(parts) > base_index: - extracted_name = parts[base_index] - sanitized_name = self._sanitize_filename(extracted_name) - if sanitized_name: # Ensure we don't add empty names - potential_base_names_per_file[file_path] = sanitized_name - log.debug(f" File '{file_path.name}' -> Potential Base Name: '{sanitized_name}'") - else: - log.debug(f" File '{file_path.name}' -> Extracted empty name at index {base_index}. Marking as None.") - file_to_base_name_map[file_path] = None # Explicitly mark as None if extraction yields empty - else: - log.debug(f" File '{file_path.name}' -> Stem '{stem}' has too few parts ({len(parts)}) for index {base_index}. Marking as None.") - file_to_base_name_map[file_path] = None # Mark as None if index is invalid for this file - else: - log.warning("Base name index not configured or invalid. Cannot determine distinct assets based on index. Treating as single asset.") - # Fallback to common prefix if no valid index - stems = [f['source_path'].stem for f in relevant_files] - common_prefix_name = os.path.commonprefix(stems) if stems else "" - sanitized_common_name = self._sanitize_filename(common_prefix_name or self.input_path.stem or "UnknownAsset") - log.info(f"Using common prefix '{sanitized_common_name}' as the single asset name.") - # Map all relevant files to this single name - for file_info in relevant_files: - potential_base_names_per_file[file_info['source_path']] = sanitized_common_name - - # --- Consolidate Distinct Names and Final Mapping --- - distinct_base_names_set = set(potential_base_names_per_file.values()) - distinct_base_names = sorted(list(distinct_base_names_set)) # Sort for consistent processing order - - # Populate the final map, including files that didn't match the index rule (marked as None earlier) - for file_info in relevant_files: - file_path = file_info['source_path'] - if file_path not in file_to_base_name_map: # If not already marked as None - file_to_base_name_map[file_path] = potential_base_names_per_file.get(file_path) # Assign determined name or None if somehow missed - - # Add files from 'extra' and 'ignored' to the map, marking them as None for base name - for category in ['extra', 'ignored']: - for file_info in self.classified_files.get(category, []): - file_path = file_info['source_path'] - if file_path not in file_to_base_name_map: # Avoid overwriting if somehow already mapped - file_to_base_name_map[file_path] = None - log.debug(f" File '{file_path.name}' (Category: {category}) -> Marked as None (No Base Name).") - - - if not distinct_base_names: - # This case should be rare due to fallbacks, but handle it. - log.warning("No distinct base names could be determined. Using input name as fallback.") - input_name = self.input_path.stem if self.input_path.is_file() else self.input_path.name - fallback_name = self._sanitize_filename(input_name or "FallbackAsset") - distinct_base_names = [fallback_name] - # Remap all files to this single fallback name - file_to_base_name_map = {f_path: fallback_name for f_path in file_to_base_name_map.keys()} - - - log.info(f"Determined {len(distinct_base_names)} distinct asset base name(s): {distinct_base_names}") - log.debug(f"File-to-BaseName Map ({len(file_to_base_name_map)} entries): { {str(k): v for k, v in file_to_base_name_map.items()} }") # Log string paths for readability - - return distinct_base_names, file_to_base_name_map - - def _determine_single_asset_metadata(self, asset_base_name: str, filtered_classified_files: Dict[str, List[Dict]]) -> Dict[str, str]: - """ - Determines the asset_category and archetype for a single, specific asset - based on its filtered list of classified files. - - Args: - asset_base_name: The determined base name for this specific asset. - filtered_classified_files: A dictionary containing only the classified - files (maps, models, etc.) belonging to this asset. - - Returns: - A dictionary containing {"asset_category": str, "archetype": str}. - """ - log.debug(f"Determining category and archetype for asset: '{asset_base_name}'") - determined_category = self.config.default_asset_category # Start with default - determined_archetype = "Unknown" - - # --- Determine Asset Category --- - if filtered_classified_files.get("models"): - determined_category = "Asset" - log.debug(f" Category set to 'Asset' for '{asset_base_name}' due to model file presence.") - else: - # Check for Decal keywords only if not an Asset - decal_keywords = self.config.asset_category_rules.get('decal_keywords', []) - found_decal = False - # Check map names first for decal keywords - candidate_files = [f['source_path'] for f in filtered_classified_files.get('maps', [])] - # Fallback to checking extra files if no maps found for this asset - if not candidate_files: - candidate_files = [f['source_path'] for f in filtered_classified_files.get('extra', [])] - - if decal_keywords: - for file_path in candidate_files: - # Check against the specific file's name within this asset's context - for keyword in decal_keywords: - if keyword.lower() in file_path.name.lower(): - determined_category = "Decal" - found_decal = True; break - if found_decal: break - if found_decal: log.debug(f" Category set to 'Decal' for '{asset_base_name}' due to keyword match.") - # If not Asset or Decal, it remains the default (e.g., "Texture") - - log.debug(f" Determined Category for '{asset_base_name}': {determined_category}") - - # --- Determine Archetype (Usage) --- - archetype_rules = self.config.archetype_rules - # Use stems from maps and models belonging *only* to this asset - check_stems = [f['source_path'].stem.lower() for f in filtered_classified_files.get('maps', [])] - check_stems.extend([f['source_path'].stem.lower() for f in filtered_classified_files.get('models', [])]) - # Also check the determined base name itself - check_stems.append(asset_base_name.lower()) - - if check_stems: - best_match_archetype = "Unknown" - # Using simple "first match wins" logic as before - for rule in archetype_rules: - if len(rule) != 2 or not isinstance(rule[1], dict): continue - arch_name, rules_dict = rule - match_any = rules_dict.get("match_any", []) - matched_any_keyword = False - if match_any: - for keyword in match_any: - kw_lower = keyword.lower() - for stem in check_stems: - if kw_lower in stem: # Simple substring check - matched_any_keyword = True - break # Found a match for this keyword - if matched_any_keyword: break # Found a match for this rule's keywords - - if matched_any_keyword: - best_match_archetype = arch_name - log.debug(f" Archetype match '{arch_name}' for '{asset_base_name}' based on keywords: {match_any}") - break # First rule match wins - - determined_archetype = best_match_archetype - - log.debug(f" Determined Archetype for '{asset_base_name}': {determined_archetype}") - - return {"asset_category": determined_category, "archetype": determined_archetype} - - - def _process_individual_maps(self, filtered_maps_list: List[Dict], current_asset_metadata: Dict, loaded_data_cache: dict) -> Tuple[Dict[str, Dict[str, Dict]], Dict[str, Dict], str, List[Dict]]: - """ - Processes, resizes, and saves classified map files for a specific asset - that are NOT used as inputs for merge rules. Uses helper functions. - - Args: - filtered_maps_list: List of map dictionaries belonging to the current asset. - current_asset_metadata: Metadata dictionary for the current asset. - loaded_data_cache: Cache dictionary for loaded/resized source data. - - Returns: - Tuple containing: - - processed_maps_details_asset: Dict mapping map_type to resolution details. - - image_stats_asset: Dict mapping map_type to calculated image statistics. - - aspect_ratio_change_string_asset: String indicating aspect ratio change. - - ignored_rough_maps: List of map dictionaries for native rough maps ignored due to gloss priority. - - """ - if not self.temp_dir: raise AssetProcessingError("Workspace not setup.") - asset_name = current_asset_metadata.get("asset_name", "UnknownAsset") - log.info(f"Processing individual map files for asset '{asset_name}'...") - - # Initialize results specific to this asset - processed_maps_details_asset: Dict[str, Dict[str, Dict]] = defaultdict(dict) - image_stats_asset: Dict[str, Dict] = {} - map_details_asset: Dict[str, Dict] = {} # Store details like source bit depth, gloss inversion - aspect_ratio_change_string_asset: str = "N/A" - ignored_rough_maps: List[Dict] = [] # Store ignored native rough maps - - # --- Settings retrieval --- - resolutions = self.config.image_resolutions - stats_res_key = self.config.calculate_stats_resolution - stats_target_dim = resolutions.get(stats_res_key) - if not stats_target_dim: log.warning(f"Stats resolution key '{stats_res_key}' not found. Stats skipped for '{asset_name}'.") - gloss_keywords = self.config.source_glossiness_keywords - # target_pattern = self.config.target_filename_pattern # Not needed here, handled by _save_image - base_name = asset_name # Use the asset name passed in - - # --- Pre-process Glossiness -> Roughness --- - # This logic needs to stay here to determine which ROUGH source to use - # and potentially ignore the native one. - derived_from_gloss_flag = {} - gloss_map_info_for_rough, native_rough_map_info = None, None - for map_info in filtered_maps_list: - # Use the final assigned map_type (e.g., ROUGH, ROUGH-1) - if map_info['map_type'].startswith('ROUGH'): - is_gloss = any(kw.lower() in map_info['source_path'].stem.lower() for kw in gloss_keywords) - if is_gloss: - # If multiple gloss sources map to ROUGH variants, prioritize the first one? - # For now, assume only one gloss source maps to ROUGH variants. - if gloss_map_info_for_rough is None: gloss_map_info_for_rough = map_info - else: - # If multiple native rough sources map to ROUGH variants, prioritize the first one? - if native_rough_map_info is None: native_rough_map_info = map_info - - rough_source_to_use_info = None # Store the map_info dict of the source to use - if gloss_map_info_for_rough: - rough_source_to_use_info = gloss_map_info_for_rough - derived_from_gloss_flag['ROUGH'] = True # Apply to all ROUGH variants if derived from gloss - if native_rough_map_info: - log.warning(f"Asset '{asset_name}': Both Gloss source ('{gloss_map_info_for_rough['source_path']}') and Rough source ('{native_rough_map_info['source_path']}') found for ROUGH maps. Prioritizing Gloss.") - ignored_rough_maps.append({'source_path': native_rough_map_info['source_path'], 'reason': 'Superseded by Gloss->Rough'}) - elif native_rough_map_info: - rough_source_to_use_info = native_rough_map_info - derived_from_gloss_flag['ROUGH'] = False - - # --- Identify maps used in merge rules --- - merge_input_map_types = set() - for rule in self.config.map_merge_rules: - inputs_mapping = rule.get("inputs", {}) - for source_map_type in inputs_mapping.values(): - # Use the base type for checking against merge rules - base_type = _get_base_map_type(source_map_type) - merge_input_map_types.add(base_type) - log.debug(f"Map types used as input for merge rules: {merge_input_map_types}") - - # --- Filter maps to process individually --- - maps_to_process_individually = [] - for map_info in filtered_maps_list: - base_map_type = _get_base_map_type(map_info['map_type']) - # Skip if this base map type is used in *any* merge rule input - if base_map_type in merge_input_map_types: - log.debug(f"Skipping individual processing for {map_info['map_type']} ({map_info['source_path']}) as its base type '{base_map_type}' is used in merge rules.") - continue - # Skip native rough map if gloss was prioritized - if map_info['map_type'].startswith('ROUGH') and any(ignored['source_path'] == map_info['source_path'] for ignored in ignored_rough_maps): - log.debug(f"Skipping individual processing of native rough map '{map_info['source_path']}' as gloss version was prioritized.") - continue - maps_to_process_individually.append(map_info) - - log.info(f"Processing {len(maps_to_process_individually)} maps individually for asset '{asset_name}'...") - - # --- Aspect Ratio Calculation Setup --- - # We need original dimensions once per asset for aspect ratio. - # Find the first map to process to get its dimensions. - first_map_info_for_aspect = next((m for m in maps_to_process_individually), None) - orig_w_aspect, orig_h_aspect = None, None - if first_map_info_for_aspect: - # Load just to get dimensions (might hit cache if used later) - # Use the first resolution key as a representative target for loading - first_res_key = next(iter(resolutions)) - temp_img_for_dims, _ = self._load_and_transform_source( - first_map_info_for_aspect['source_path'], - first_map_info_for_aspect['map_type'], - first_res_key, - False, # is_gloss_source doesn't matter for dims - loaded_data_cache # Use the main cache - ) - if temp_img_for_dims is not None: - orig_h_aspect, orig_w_aspect = temp_img_for_dims.shape[:2] - log.debug(f"Got original dimensions ({orig_w_aspect}x{orig_h_aspect}) for aspect ratio calculation from {first_map_info_for_aspect['source_path']}") - else: - log.warning(f"Could not load image {first_map_info_for_aspect['source_path']} to get original dimensions for aspect ratio.") - else: - log.warning("No maps found to process individually, cannot calculate aspect ratio string.") - - - # --- Process Each Individual Map --- - for map_info in maps_to_process_individually: - map_type = map_info['map_type'] # Final type (e.g., COL-1) - source_path_rel = map_info['source_path'] - original_extension = map_info.get('original_extension', '.png') - # Determine if this specific map type should use gloss inversion logic - # If ROUGH-1, ROUGH-2 etc derive from gloss, they all use inversion - is_gloss_source_for_this_map = map_type.startswith('ROUGH') and derived_from_gloss_flag.get('ROUGH', False) - - log.info(f"-- Asset '{asset_name}': Processing Individual Map: {map_type} (Source: {source_path_rel.name}) --") - current_map_details = {"derived_from_gloss": is_gloss_source_for_this_map} - source_bit_depth_found = None # Track if we've found the bit depth for this map type - - try: - # --- Loop through target resolutions --- - for res_key, target_dim_px in resolutions.items(): - log.debug(f"Processing {map_type} for resolution: {res_key}...") - - # --- 1. Load and Transform Source (using helper + cache) --- - img_resized, source_dtype = self._load_and_transform_source( - source_path_rel=source_path_rel, - map_type=map_type, # Pass the specific map type (e.g., ROUGH-1) - target_resolution_key=res_key, - is_gloss_source=is_gloss_source_for_this_map, - cache=loaded_data_cache - ) - - if img_resized is None: - log.warning(f"Failed to load/transform source {source_path_rel} for {res_key}. Skipping resolution.") - continue # Skip this resolution - - # Store source bit depth once found - if source_dtype is not None and source_bit_depth_found is None: - source_bit_depth_found = 16 if source_dtype == np.uint16 else (8 if source_dtype == np.uint8 else 8) # Default non-uint to 8 - current_map_details["source_bit_depth"] = source_bit_depth_found - log.debug(f"Stored source bit depth for {map_type}: {source_bit_depth_found}") - - # --- 2. Calculate Stats (if applicable) --- - if res_key == stats_res_key and stats_target_dim: - log.debug(f"Calculating stats for {map_type} using {res_key} image...") - stats = _calculate_image_stats(img_resized) - if stats: image_stats_asset[map_type] = stats - else: log.warning(f"Stats calculation failed for {map_type} at {res_key}.") - - # --- 3. Calculate Aspect Ratio Change String (once per asset, using pre-calculated dims) --- - if aspect_ratio_change_string_asset == "N/A" and orig_w_aspect is not None and orig_h_aspect is not None: - target_w_aspect, target_h_aspect = img_resized.shape[1], img_resized.shape[0] # Use current resized dims - try: - aspect_string = self._normalize_aspect_ratio_change(orig_w_aspect, orig_h_aspect, target_w_aspect, target_h_aspect) - aspect_ratio_change_string_asset = aspect_string - log.debug(f"Stored aspect ratio change string using {res_key}: '{aspect_string}'") - except Exception as aspect_err: - log.error(f"Failed to calculate aspect ratio change string using {res_key}: {aspect_err}", exc_info=True) - aspect_ratio_change_string_asset = "Error" - elif aspect_ratio_change_string_asset == "N/A": - # This case happens if we couldn't get original dims - aspect_ratio_change_string_asset = "Unknown" # Set to unknown instead of recalculating - - - # --- 4. Save Image (using helper) --- - source_info = { - 'original_extension': original_extension, - 'source_bit_depth': source_bit_depth_found or 8, # Use found depth or default - 'involved_extensions': {original_extension} # Only self for individual maps - } - bit_depth_rule = self.config.get_bit_depth_rule(map_type) # Get rule for this specific map type - - save_result = self._save_image( - image_data=img_resized, - map_type=map_type, - resolution_key=res_key, - asset_base_name=base_name, - source_info=source_info, - output_bit_depth_rule=bit_depth_rule, - temp_dir=self.temp_dir - ) - - # --- 5. Store Result --- - if save_result: - processed_maps_details_asset.setdefault(map_type, {})[res_key] = save_result - # Update overall map detail (e.g., final format) if needed - current_map_details["output_format"] = save_result.get("format") - else: - log.error(f"Failed to save {map_type} at {res_key}.") - processed_maps_details_asset.setdefault(map_type, {})[f'error_{res_key}'] = "Save failed" - - - except Exception as map_proc_err: - log.error(f"Failed processing map {map_type} from {source_path_rel.name}: {map_proc_err}", exc_info=True) - processed_maps_details_asset.setdefault(map_type, {})['error'] = str(map_proc_err) - - # Store collected details for this map type - map_details_asset[map_type] = current_map_details - - # --- Final Metadata Updates (Handled in main process loop) --- - # Update the passed-in current_asset_metadata dictionary directly with map_details - # This avoids returning it and merging later. - current_asset_metadata["map_details"] = map_details_asset - - log.info(f"Finished processing individual map files for asset '{asset_name}'.") - return processed_maps_details_asset, image_stats_asset, aspect_ratio_change_string_asset, ignored_rough_maps - - - def _merge_maps_from_source(self, processed_maps_details_asset: Dict[str, Dict[str, Dict]], filtered_classified_files: Dict[str, List[Dict]], current_asset_metadata: Dict, loaded_data_cache: dict) -> Dict[str, Dict[str, Dict]]: - """ - Merges channels from different SOURCE maps for a specific asset based on rules - in configuration, using helper functions for loading and saving. - - Args: - processed_maps_details_asset: Details of processed maps (used to find common resolutions). - filtered_classified_files: Classified files dictionary filtered for this asset (used to find source paths). - current_asset_metadata: Metadata dictionary for the current asset. - loaded_data_cache: Cache dictionary for loaded/resized source data. - - Returns: - Dict[str, Dict[str, Dict]]: Details of the merged maps created for this asset. - """ - if not self.temp_dir: raise AssetProcessingError("Workspace not setup.") - asset_name = current_asset_metadata.get("asset_name", "UnknownAsset") - gloss_keywords = self.config.source_glossiness_keywords # Get gloss keywords - - merge_rules = self.config.map_merge_rules - log.info(f"Asset '{asset_name}': Applying {len(merge_rules)} map merging rule(s) from source...") - - # Initialize results for this asset - merged_maps_details_asset: Dict[str, Dict[str, Dict]] = defaultdict(dict) - - for rule_index, rule in enumerate(merge_rules): - output_map_type = rule.get("output_map_type") - inputs_mapping = rule.get("inputs") # e.g., {"R": "AO", "G": "ROUGH", "B": "METAL"} - defaults = rule.get("defaults", {}) - rule_bit_depth = rule.get("output_bit_depth", "respect_inputs") - - if not output_map_type or not inputs_mapping: - log.warning(f"Asset '{asset_name}': Skipping merge rule #{rule_index+1}: Missing 'output_map_type' or 'inputs'. Rule: {rule}") - continue - - log.info(f"-- Asset '{asset_name}': Applying merge rule for '{output_map_type}' --") - - # --- Find required SOURCE files and their details for this asset --- - required_input_sources = {} # map_type -> {'source_path': Path, 'original_extension': str, 'is_gloss_source': bool} - possible_to_find_sources = True - for input_type in set(inputs_mapping.values()): # e.g., {"AO", "ROUGH", "METAL"} - found_source_for_type = False - # Search in the filtered classified maps for this asset - for classified_map in filtered_classified_files.get("maps", []): - # Check if the classified map's type matches the required input type - # This needs to handle variants (e.g., ROUGH-1 should match ROUGH) - if classified_map['map_type'].startswith(input_type): - source_path_rel = classified_map.get('source_path') - if not source_path_rel: continue # Skip if path is missing - - # Determine if this source is gloss (only relevant if input_type is ROUGH) - is_gloss = False - if input_type == 'ROUGH': - is_gloss = any(kw.lower() in source_path_rel.stem.lower() for kw in gloss_keywords) - # Prioritize gloss source if both exist (logic from _process_individual_maps) - native_rough_exists = any(m['map_type'].startswith('ROUGH') and not any(gk.lower() in m['source_path'].stem.lower() for gk in gloss_keywords) for m in filtered_classified_files.get("maps", [])) - if is_gloss and native_rough_exists: - log.debug(f"Merge input '{input_type}': Prioritizing gloss source '{source_path_rel}' over native rough.") - elif not is_gloss and native_rough_exists and any(m['map_type'].startswith('ROUGH') and any(gk.lower() in m['source_path'].stem.lower() for gk in gloss_keywords) for m in filtered_classified_files.get("maps", [])): - log.debug(f"Merge input '{input_type}': Skipping native rough source '{source_path_rel}' because gloss source exists.") - continue # Skip this native rough source - - required_input_sources[input_type] = { - 'source_path': source_path_rel, - 'original_extension': classified_map.get('original_extension', '.png'), - 'is_gloss_source': is_gloss - } - found_source_for_type = True - log.debug(f"Found source for merge input '{input_type}': {source_path_rel} (Gloss: {is_gloss})") - break # Found the first matching source for this input type - if not found_source_for_type: - log.warning(f"Asset '{asset_name}': Required source file for input map type '{input_type}' not found in classified files. Cannot perform merge for '{output_map_type}'.") - possible_to_find_sources = False - break - - if not possible_to_find_sources: - continue # Skip this merge rule - - # --- Determine common resolutions based on *processed* maps (as a proxy for available sizes) --- - # This assumes _process_individual_maps ran first and populated processed_maps_details_asset - possible_resolutions_per_input = [] - for input_type in set(inputs_mapping.values()): - if input_type in processed_maps_details_asset: - res_keys = {res for res, details in processed_maps_details_asset[input_type].items() if isinstance(details, dict) and 'error' not in details} - if not res_keys: - log.warning(f"Asset '{asset_name}': Input map type '{input_type}' for merge rule '{output_map_type}' has no successfully processed resolutions (needed for size check).") - possible_resolutions_per_input = [] - break - possible_resolutions_per_input.append(res_keys) - else: - # This case might happen if the input map is *only* used for merging - # We need a way to determine available resolutions without relying on prior processing. - # For now, we'll rely on the check above ensuring the source exists. - # We'll load the source at *all* target resolutions and let _load_and_transform_source handle skipping if upscale is needed. - log.debug(f"Input map type '{input_type}' for merge rule '{output_map_type}' might not have been processed individually. Will attempt loading source for all target resolutions.") - # Add all configured resolutions as possibilities for this input - possible_resolutions_per_input.append(set(self.config.image_resolutions.keys())) - - - if not possible_resolutions_per_input: - log.warning(f"Asset '{asset_name}': Cannot determine common resolutions for '{output_map_type}'. Skipping rule.") - continue - - common_resolutions = set.intersection(*possible_resolutions_per_input) - - if not common_resolutions: - log.warning(f"Asset '{asset_name}': No common resolutions found among required inputs {set(inputs_mapping.values())} for merge rule '{output_map_type}'. Skipping rule.") - continue - log.debug(f"Asset '{asset_name}': Common resolutions for '{output_map_type}': {common_resolutions}") - - # --- Loop through common resolutions --- - res_order = {k: self.config.image_resolutions[k] for k in common_resolutions if k in self.config.image_resolutions} - if not res_order: - log.warning(f"Asset '{asset_name}': Common resolutions {common_resolutions} do not match config. Skipping merge for '{output_map_type}'.") - continue - - sorted_res_keys = sorted(res_order.keys(), key=lambda k: res_order[k], reverse=True) - base_name = asset_name # Use current asset's name - - for current_res_key in sorted_res_keys: - log.debug(f"Asset '{asset_name}': Merging '{output_map_type}' for resolution: {current_res_key}") - try: - loaded_inputs_data = {} # map_type -> loaded numpy array - source_info_for_save = {'involved_extensions': set(), 'max_input_bit_depth': 8} - - # --- Load required SOURCE maps using helper --- - possible_to_load = True - target_channels = list(inputs_mapping.keys()) # e.g., ['R', 'G', 'B'] - - for map_type in set(inputs_mapping.values()): # e.g., {"AO", "ROUGH", "METAL"} - source_details = required_input_sources.get(map_type) - if not source_details: - log.error(f"Internal Error: Source details missing for '{map_type}' during merge load.") - possible_to_load = False; break - - source_path_rel = source_details['source_path'] - is_gloss = source_details['is_gloss_source'] - original_ext = source_details['original_extension'] - source_info_for_save['involved_extensions'].add(original_ext) - - log.debug(f"Loading source '{source_path_rel}' for merge input '{map_type}' at {current_res_key} (Gloss: {is_gloss})") - img_resized, source_dtype = self._load_and_transform_source( - source_path_rel=source_path_rel, - map_type=map_type, # Pass the base map type (e.g., ROUGH) - target_resolution_key=current_res_key, - is_gloss_source=is_gloss, - cache=loaded_data_cache - ) - - if img_resized is None: - log.warning(f"Asset '{asset_name}': Failed to load/transform source '{source_path_rel}' for merge input '{map_type}' at {current_res_key}. Skipping resolution.") - possible_to_load = False; break - - loaded_inputs_data[map_type] = img_resized - - # Track max source bit depth - if source_dtype == np.uint16: - source_info_for_save['max_input_bit_depth'] = max(source_info_for_save['max_input_bit_depth'], 16) - # Add other dtype checks if needed (e.g., float32 -> 16?) - - if not possible_to_load: continue - - # --- Calculate Stats for ROUGH source if used and at stats resolution --- - stats_res_key = self.config.calculate_stats_resolution - if current_res_key == stats_res_key: - log.debug(f"Asset '{asset_name}': Checking for ROUGH source stats for '{output_map_type}' at {stats_res_key}") - for target_channel, source_map_type in inputs_mapping.items(): - if source_map_type == 'ROUGH' and source_map_type in loaded_inputs_data: - log.debug(f"Asset '{asset_name}': Calculating stats for ROUGH source (mapped to channel '{target_channel}') for '{output_map_type}' at {stats_res_key}") - rough_image_data = loaded_inputs_data[source_map_type] - rough_stats = _calculate_image_stats(rough_image_data) - if rough_stats: - # Ensure the nested dictionary structure exists - if "merged_map_channel_stats" not in current_asset_metadata: - current_asset_metadata["merged_map_channel_stats"] = {} - if output_map_type not in current_asset_metadata["merged_map_channel_stats"]: - current_asset_metadata["merged_map_channel_stats"][output_map_type] = {} - if target_channel not in current_asset_metadata["merged_map_channel_stats"][output_map_type]: - current_asset_metadata["merged_map_channel_stats"][output_map_type][target_channel] = {} - - current_asset_metadata["merged_map_channel_stats"][output_map_type][target_channel][stats_res_key] = rough_stats - log.debug(f"Asset '{asset_name}': Stored ROUGH stats for '{output_map_type}' channel '{target_channel}' at {stats_res_key}: {rough_stats}") - else: - log.warning(f"Asset '{asset_name}': Failed to calculate ROUGH stats for '{output_map_type}' channel '{target_channel}' at {stats_res_key}.") - - # --- Determine dimensions --- - # All loaded inputs should have the same dimensions for this resolution - first_map_type = next(iter(loaded_inputs_data)) - h, w = loaded_inputs_data[first_map_type].shape[:2] - num_target_channels = len(target_channels) - - # --- Prepare and Merge Channels --- - merged_channels_float32 = [] - for target_channel in target_channels: # e.g., 'R', 'G', 'B' - source_map_type = inputs_mapping.get(target_channel) # e.g., "AO", "ROUGH", "METAL" - channel_data_float32 = None - - if source_map_type and source_map_type in loaded_inputs_data: - img_input = loaded_inputs_data[source_map_type] # Get the loaded NumPy array - - # Ensure input is float32 0-1 range for merging - if img_input.dtype == np.uint16: img_float = img_input.astype(np.float32) / 65535.0 - elif img_input.dtype == np.uint8: img_float = img_input.astype(np.float32) / 255.0 - elif img_input.dtype == np.float16: img_float = img_input.astype(np.float32) # Assume float16 is 0-1 - else: img_float = img_input.astype(np.float32) # Assume other floats are 0-1 - - num_source_channels = img_float.shape[2] if len(img_float.shape) == 3 else 1 - - # Extract the correct channel - if num_source_channels >= 3: - if target_channel == 'R': channel_data_float32 = img_float[:, :, 0] - elif target_channel == 'G': channel_data_float32 = img_float[:, :, 1] - elif target_channel == 'B': channel_data_float32 = img_float[:, :, 2] - elif target_channel == 'A' and num_source_channels == 4: channel_data_float32 = img_float[:, :, 3] - else: log.warning(f"Target channel '{target_channel}' invalid for 3/4 channel source '{source_map_type}'.") - elif num_source_channels == 1 or len(img_float.shape) == 2: - # If source is grayscale, use it for R, G, B, or A target channels - channel_data_float32 = img_float.reshape(h, w) - else: - log.warning(f"Unexpected shape {img_float.shape} for source '{source_map_type}'.") - - # Apply default if channel data couldn't be extracted - if channel_data_float32 is None: - default_val = defaults.get(target_channel) - if default_val is None: - raise AssetProcessingError(f"Missing input/default for target channel '{target_channel}' in merge rule '{output_map_type}'.") - log.debug(f"Using default value {default_val} for target channel '{target_channel}' in '{output_map_type}'.") - channel_data_float32 = np.full((h, w), float(default_val), dtype=np.float32) - - merged_channels_float32.append(channel_data_float32) - - if not merged_channels_float32 or len(merged_channels_float32) != num_target_channels: - raise AssetProcessingError(f"Channel count mismatch during merge for '{output_map_type}'. Expected {num_target_channels}, got {len(merged_channels_float32)}.") - - merged_image_float32 = cv2.merge(merged_channels_float32) - log.debug(f"Merged channels for '{output_map_type}' ({current_res_key}). Result shape: {merged_image_float32.shape}, dtype: {merged_image_float32.dtype}") - - # --- Save Merged Map using Helper --- - save_result = self._save_image( - image_data=merged_image_float32, # Pass the merged float32 data - map_type=output_map_type, - resolution_key=current_res_key, - asset_base_name=base_name, - source_info=source_info_for_save, # Pass collected source info - output_bit_depth_rule=rule_bit_depth, # Pass the rule's requirement - temp_dir=self.temp_dir - ) - - # --- Record details locally --- - if save_result: - merged_maps_details_asset[output_map_type][current_res_key] = save_result - else: - log.error(f"Asset '{asset_name}': Failed to save merged map '{output_map_type}' at resolution '{current_res_key}'.") - merged_maps_details_asset.setdefault(output_map_type, {})[f'error_{current_res_key}'] = "Save failed via helper" - - - except Exception as merge_res_err: - log.error(f"Asset '{asset_name}': Failed merging '{output_map_type}' at resolution '{current_res_key}': {merge_res_err}", exc_info=True) - # Store error locally for this asset - merged_maps_details_asset.setdefault(output_map_type, {})[f'error_{current_res_key}'] = str(merge_res_err) - - log.info(f"Asset '{asset_name}': Finished applying map merging rules.") - # Return the details for this asset - return merged_maps_details_asset - - - def _generate_metadata_file(self, current_asset_metadata: Dict, processed_maps_details_asset: Dict[str, Dict[str, Dict]], merged_maps_details_asset: Dict[str, Dict[str, Dict]], filtered_classified_files_asset: Dict[str, List[Dict]], unmatched_files_paths: List[Path], map_details_asset: Dict[str, Dict]) -> Path: - """ - Gathers metadata for a specific asset and writes it to a temporary JSON file. - - Args: - current_asset_metadata: Base metadata for this asset (name, category, archetype, etc.). - processed_maps_details_asset: Details of processed maps for this asset. - merged_maps_details_asset: Details of merged maps for this asset. - filtered_classified_files_asset: Classified files belonging only to this asset. - unmatched_files_paths: List of relative paths for files not matched to any base name. - map_details_asset: Dictionary containing details like source bit depth, gloss inversion per map type. - - - Returns: - Path: The path to the generated temporary metadata file. - """ - if not self.temp_dir: raise AssetProcessingError("Workspace not setup.") - asset_name = current_asset_metadata.get("asset_name") - if not asset_name or asset_name == "UnknownAssetName": - log.warning("Asset name unknown during metadata generation, file may be incomplete or incorrectly named.") - asset_name = "UnknownAsset_Metadata" # Fallback for filename - - log.info(f"Generating metadata file for asset '{asset_name}'...") - # Start with the base metadata passed in for this asset - final_metadata = current_asset_metadata.copy() - - # Populate map details from the specific asset's processing results - # Add merged map channel stats - final_metadata["merged_map_channel_stats"] = current_asset_metadata.get("merged_map_channel_stats", {}) # Get from passed metadata - - final_metadata["processed_map_resolutions"] = {} - for map_type, res_dict in processed_maps_details_asset.items(): - keys = [res for res, d in res_dict.items() if isinstance(d, dict) and 'error' not in d] - if keys: final_metadata["processed_map_resolutions"][map_type] = sorted(keys) - - final_metadata["merged_map_resolutions"] = {} - for map_type, res_dict in merged_maps_details_asset.items(): - keys = [res for res, d in res_dict.items() if isinstance(d, dict) and 'error' not in d] - if keys: final_metadata["merged_map_resolutions"][map_type] = sorted(keys) - - # Determine maps present based on successful processing for this asset - final_metadata["maps_present"] = sorted(list(processed_maps_details_asset.keys())) - final_metadata["merged_maps"] = sorted(list(merged_maps_details_asset.keys())) - - # Determine shader features based on this asset's maps - features = set() - for map_type, details in map_details_asset.items(): # Use map_details_asset passed in - if map_type in ["SSS", "FUZZ", "MASK"]: features.add(map_type) - if details.get("derived_from_gloss"): features.add("InvertedGloss") - res_details = processed_maps_details_asset.get(map_type, {}) - if any(res_info.get("bit_depth") == 16 for res_info in res_details.values() if isinstance(res_info, dict)): features.add(f"16bit_{map_type}") - final_metadata["shader_features"] = sorted(list(features)) - - # Determine source files in this asset's Extra folder - # Includes: - # - Files originally classified as 'Extra' or 'Unrecognised' belonging to this asset. - # - Files originally classified as 'Ignored' belonging to this asset. - # - All 'unmatched' files (belonging to no specific asset). - source_files_in_extra_set = set() - for category in ['extra', 'ignored']: - for file_info in filtered_classified_files_asset.get(category, []): - source_files_in_extra_set.add(str(file_info['source_path'])) - # Add all unmatched files - for file_path in unmatched_files_paths: - source_files_in_extra_set.add(str(file_path)) - final_metadata["source_files_in_extra"] = sorted(list(source_files_in_extra_set)) - - # Add image stats and map details specific to this asset - final_metadata["image_stats_1k"] = current_asset_metadata.get("image_stats_1k", {}) # Get from passed metadata - final_metadata["map_details"] = map_details_asset # Use map_details_asset passed in - final_metadata["aspect_ratio_change_string"] = current_asset_metadata.get("aspect_ratio_change_string", "N/A") # Get from passed metadata - - - # Add processing info - final_metadata["_processing_info"] = { - "preset_used": self.config.preset_name, - "timestamp_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - "input_source": str(self.input_path.name), # Add original input source - } - - # Sort lists just before writing - for key in ["maps_present", "merged_maps", "shader_features", "source_files_in_extra"]: - if key in final_metadata and isinstance(final_metadata[key], list): final_metadata[key].sort() - - # Use asset name in temporary filename to avoid conflicts - metadata_filename = f"{asset_name}_{self.config.metadata_filename}" - output_path = self.temp_dir / metadata_filename - log.debug(f"Writing metadata for asset '{asset_name}' to temporary file: {output_path}") - try: - with open(output_path, 'w', encoding='utf-8') as f: - json.dump(final_metadata, f, indent=4, ensure_ascii=False, sort_keys=True) - log.info(f"Metadata file '{metadata_filename}' generated successfully for asset '{asset_name}'.") - return output_path # Return the path to the temporary file - except Exception as e: - raise AssetProcessingError(f"Failed to write metadata file {output_path} for asset '{asset_name}': {e}") from e - - - def _normalize_aspect_ratio_change(self, original_width, original_height, resized_width, resized_height, decimals=2): - """ - Calculates the aspect ratio change string (e.g., "EVEN", "X133") based on original prototype logic. - Returns the string representation. - """ - if original_width <= 0 or original_height <= 0: - log.warning("Cannot calculate aspect ratio change with zero original dimensions.") - return "InvalidInput" - - # Avoid division by zero if resize resulted in zero dimensions (shouldn't happen with checks) - if resized_width <= 0 or resized_height <= 0: - log.warning("Cannot calculate aspect ratio change with zero resized dimensions.") - return "InvalidResize" - - # Original logic from user feedback - width_change_percentage = ((resized_width - original_width) / original_width) * 100 - height_change_percentage = ((resized_height - original_height) / original_height) * 100 - - normalized_width_change = width_change_percentage / 100 - normalized_height_change = height_change_percentage / 100 - - normalized_width_change = min(max(normalized_width_change + 1, 0), 2) - normalized_height_change = min(max(normalized_height_change + 1, 0), 2) - - # Handle potential zero division if one dimension change is exactly -100% (normalized to 0) - # If both are 0, aspect ratio is maintained. If one is 0, the other dominates. - if normalized_width_change == 0 and normalized_height_change == 0: - closest_value_to_one = 1.0 # Avoid division by zero, effectively scale_factor = 1 - elif normalized_width_change == 0: - closest_value_to_one = abs(normalized_height_change) - elif normalized_height_change == 0: - closest_value_to_one = abs(normalized_width_change) - else: - closest_value_to_one = min(abs(normalized_width_change), abs(normalized_height_change)) - - # Add a small epsilon to avoid division by zero if closest_value_to_one is extremely close to 0 - epsilon = 1e-9 - scale_factor = 1 / (closest_value_to_one + epsilon) if abs(closest_value_to_one) < epsilon else 1 / closest_value_to_one - - scaled_normalized_width_change = scale_factor * normalized_width_change - scaled_normalized_height_change = scale_factor * normalized_height_change - - output_width = round(scaled_normalized_width_change, decimals) - output_height = round(scaled_normalized_height_change, decimals) - - # Convert to int if exactly 1.0 after rounding - if abs(output_width - 1.0) < epsilon: output_width = 1 - if abs(output_height - 1.0) < epsilon: output_height = 1 - - # Determine output string - if original_width == original_height or abs(output_width - output_height) < epsilon: - output = "EVEN" - elif output_width != 1 and output_height == 1: - output = f"X{str(output_width).replace('.', '')}" - elif output_height != 1 and output_width == 1: - output = f"Y{str(output_height).replace('.', '')}" - else: - # Both changed relative to each other - output = f"X{str(output_width).replace('.', '')}Y{str(output_height).replace('.', '')}" - - log.debug(f"Aspect ratio change calculated: Orig=({original_width}x{original_height}), Resized=({resized_width}x{resized_height}) -> String='{output}'") - return output - - def _sanitize_filename(self, name: str) -> str: - """Removes or replaces characters invalid for filenames/directory names.""" - # ... (Implementation from Response #51) ... - if not isinstance(name, str): name = str(name) - name = re.sub(r'[^\w.\-]+', '_', name) # Allow alphanumeric, underscore, hyphen, dot - name = re.sub(r'_+', '_', name) - name = name.strip('_') - if not name: name = "invalid_name" - return name - - def _organize_output_files(self, current_asset_name: str, processed_maps_details_asset: Dict[str, Dict[str, Dict]], merged_maps_details_asset: Dict[str, Dict[str, Dict]], filtered_classified_files_asset: Dict[str, List[Dict]], unmatched_files_paths: List[Path], temp_metadata_path: Path): - """ - Moves/copies processed files for a specific asset from the temp dir to the final output structure. - - Args: - current_asset_name: The sanitized name of the asset being organized. - processed_maps_details_asset: Details of processed maps for this asset. - merged_maps_details_asset: Details of merged maps for this asset. - filtered_classified_files_asset: Classified files dictionary filtered for this asset. - unmatched_files_paths: List of relative paths for files not matched to any base name. - temp_metadata_path: Path to the temporary metadata file for this asset. - """ - if not self.temp_dir or not self.temp_dir.exists(): raise AssetProcessingError("Temp workspace missing.") - if not current_asset_name or current_asset_name == "UnknownAssetName": raise AssetProcessingError("Asset name missing for organization.") - supplier_name = self.config.supplier_name # Get supplier name from config - if not supplier_name: raise AssetProcessingError("Supplier name missing from config.") - - supplier_sanitized = self._sanitize_filename(supplier_name) - asset_name_sanitized = self._sanitize_filename(current_asset_name) # Already sanitized, but ensure consistency - final_dir = self.output_base_path / supplier_sanitized / asset_name_sanitized - log.info(f"Organizing output files for asset '{asset_name_sanitized}' into: {final_dir}") - - try: - # Handle overwrite logic specifically for this asset's directory - if final_dir.exists() and self.overwrite: - log.warning(f"Output directory exists for '{asset_name_sanitized}' and overwrite is True. Removing existing directory: {final_dir}") - try: - shutil.rmtree(final_dir) - except Exception as rm_err: - raise AssetProcessingError(f"Failed to remove existing output directory {final_dir} for asset '{asset_name_sanitized}' during overwrite: {rm_err}") from rm_err - # Note: Skip check should prevent this if overwrite is False, but mkdir handles exist_ok=True - - final_dir.mkdir(parents=True, exist_ok=True) - except Exception as e: - if not isinstance(e, AssetProcessingError): - raise AssetProcessingError(f"Failed to create final dir {final_dir} for asset '{asset_name_sanitized}': {e}") from e - else: - raise - - # --- Helper for moving files --- - # Keep track of files successfully moved to avoid copying them later as 'unmatched' - moved_source_files = set() - def _safe_move(src_rel_path: Path | None, dest_dir: Path, file_desc: str): - if not src_rel_path: log.warning(f"Asset '{asset_name_sanitized}': Missing src path for {file_desc}."); return - source_abs = self.temp_dir / src_rel_path - # Use the original filename from the source path for the destination - dest_abs = dest_dir / src_rel_path.name - try: - if source_abs.exists(): - log.debug(f"Asset '{asset_name_sanitized}': Moving {file_desc}: {source_abs.name} -> {dest_dir.relative_to(self.output_base_path)}/") - dest_dir.mkdir(parents=True, exist_ok=True) - shutil.move(str(source_abs), str(dest_abs)) - moved_source_files.add(src_rel_path) # Track successfully moved source files - else: log.warning(f"Asset '{asset_name_sanitized}': Source file missing for {file_desc}: {source_abs}") - except Exception as e: log.error(f"Asset '{asset_name_sanitized}': Failed moving {file_desc} '{source_abs.name}': {e}", exc_info=True) - - # --- Helper for copying files (for unmatched extras) --- - def _safe_copy(src_rel_path: Path | None, dest_dir: Path, file_desc: str): - if not src_rel_path: log.warning(f"Asset '{asset_name_sanitized}': Missing src path for {file_desc} copy."); return - # Skip copying if this source file was already moved (e.g., it was an 'Extra' for this specific asset) - if src_rel_path in moved_source_files: - log.debug(f"Asset '{asset_name_sanitized}': Skipping copy of {file_desc} '{src_rel_path.name}' as it was already moved.") - return - source_abs = self.temp_dir / src_rel_path - dest_abs = dest_dir / src_rel_path.name - try: - if source_abs.exists(): - # Avoid copying if the exact destination file already exists (e.g., from a previous asset's copy) - if dest_abs.exists(): - log.debug(f"Asset '{asset_name_sanitized}': Destination file already exists for {file_desc} copy: {dest_abs.name}. Skipping copy.") - return - log.debug(f"Asset '{asset_name_sanitized}': Copying {file_desc}: {source_abs.name} -> {dest_dir.relative_to(self.output_base_path)}/") - dest_dir.mkdir(parents=True, exist_ok=True) - shutil.copy2(str(source_abs), str(dest_abs)) # Use copy2 to preserve metadata - else: log.warning(f"Asset '{asset_name_sanitized}': Source file missing for {file_desc} copy: {source_abs}") - except Exception as e: log.error(f"Asset '{asset_name_sanitized}': Failed copying {file_desc} '{source_abs.name}': {e}", exc_info=True) - - - # --- Move Processed/Merged Maps --- - for details_dict in [processed_maps_details_asset, merged_maps_details_asset]: - for map_type, res_dict in details_dict.items(): - if 'error' in res_dict: continue - for res_key, details in res_dict.items(): - if isinstance(details, dict) and 'path' in details: - _safe_move(details['path'], final_dir, f"{map_type} ({res_key})") - - # --- Move Models specific to this asset --- - for model_info in filtered_classified_files_asset.get('models', []): - _safe_move(model_info.get('source_path'), final_dir, "model file") - - # --- Move Metadata File --- - if temp_metadata_path and temp_metadata_path.exists(): - final_metadata_path = final_dir / self.config.metadata_filename # Use standard name - try: - log.debug(f"Asset '{asset_name_sanitized}': Moving metadata file: {temp_metadata_path.name} -> {final_metadata_path.relative_to(self.output_base_path)}") - shutil.move(str(temp_metadata_path), str(final_metadata_path)) - # No need to add metadata path to moved_source_files as it's uniquely generated - except Exception as e: - log.error(f"Asset '{asset_name_sanitized}': Failed moving metadata file '{temp_metadata_path.name}': {e}", exc_info=True) - else: - log.warning(f"Asset '{asset_name_sanitized}': Temporary metadata file path missing or file does not exist: {temp_metadata_path}") - - - # --- Handle Extra/Ignored/Unmatched Files --- - extra_subdir_name = self.config.extra_files_subdir - extra_dir = final_dir / extra_subdir_name - if filtered_classified_files_asset.get('extra') or filtered_classified_files_asset.get('ignored') or unmatched_files_paths: - try: - extra_dir.mkdir(parents=True, exist_ok=True) - - # Move asset-specific Extra/Ignored files - files_to_move_extra = filtered_classified_files_asset.get('extra', []) + filtered_classified_files_asset.get('ignored', []) - if files_to_move_extra: - log.debug(f"Asset '{asset_name_sanitized}': Moving {len(files_to_move_extra)} asset-specific files to '{extra_subdir_name}/'...") - for file_info in files_to_move_extra: - _safe_move(file_info.get('source_path'), extra_dir, f"extra/ignored file ({file_info.get('reason', 'Unknown')})") - - # Copy unmatched files - if unmatched_files_paths: - log.debug(f"Asset '{asset_name_sanitized}': Copying {len(unmatched_files_paths)} unmatched files to '{extra_subdir_name}/'...") - for file_path in unmatched_files_paths: - _safe_copy(file_path, extra_dir, "unmatched file") - - except Exception as e: log.error(f"Asset '{asset_name_sanitized}': Failed creating/moving/copying to Extra dir {extra_dir}: {e}", exc_info=True) - - log.info(f"Finished organizing output for asset '{asset_name_sanitized}'.") - - - def _cleanup_workspace(self): - """Removes the temporary workspace directory if it exists.""" - # ... (Implementation from Response #45) ... - if self.temp_dir and self.temp_dir.exists(): - try: - log.debug(f"Cleaning up temporary workspace: {self.temp_dir}") - shutil.rmtree(self.temp_dir) - self.temp_dir = None - log.debug("Temporary workspace cleaned up successfully.") - except Exception as e: - log.error(f"Failed to remove temporary workspace {self.temp_dir}: {e}", exc_info=True) - - # --- Prediction Method --- - def predict_output_structure(self) -> tuple[str | None, str | None, dict[str, str] | None] | None: - """ - Predicts the final output structure (supplier, asset name) and attempts - to predict output filenames for potential map files based on naming conventions. - Does not perform full processing or image loading. - - Returns: - tuple[str | None, str | None, dict[str, str] | None]: - (sanitized_supplier_name, sanitized_asset_name, file_predictions_dict) - where file_predictions_dict maps input filename -> predicted output filename. - Returns None if prediction fails critically. - """ - log.debug(f"Predicting output structure and filenames for: {self.input_path.name}") - try: - # 1. Get Supplier Name - supplier_name = self.config.supplier_name - if not supplier_name: - log.warning("Supplier name not found in configuration during prediction.") - return None - - # 2. List Input Filenames/Stems - candidate_stems = set() # Use set for unique stems - filenames = [] - if self.input_path.is_file() and self.input_path.suffix.lower() == '.zip': - try: - with zipfile.ZipFile(self.input_path, 'r') as zip_ref: - # Get only filenames, ignore directories - filenames = [Path(f).name for f in zip_ref.namelist() if not f.endswith('/')] - except zipfile.BadZipFile: - log.error(f"Bad ZIP file during prediction: {self.input_path.name}") - return None - except Exception as zip_err: - log.error(f"Error reading ZIP file list during prediction for {self.input_path.name}: {zip_err}") - return None # Cannot proceed if we can't list files - elif self.input_path.is_dir(): - try: - for item in self.input_path.iterdir(): - if item.is_file(): # Only consider files directly in the folder for prediction simplicity - filenames.append(item.name) - # Note: Not walking subdirs for prediction to keep it fast - except Exception as dir_err: - log.error(f"Error listing directory contents during prediction for {self.input_path.name}: {dir_err}") - return None - - if not filenames: - log.warning(f"No files found in input for prediction: {self.input_path.name}") - return None # Return None if no files found - - # 3. Lightweight Classification for Stems and Potential Maps - map_type_mapping = self.config.map_type_mapping - model_patterns = self.config.asset_category_rules.get('model_patterns', []) - separator = self.config.source_naming_separator - processed_filenames = set() # Track full filenames processed - potential_map_files = {} # Store fname -> potential map_type - - for fname in filenames: - if fname in processed_filenames: continue - - fstem = Path(fname).stem - fstem_lower = fstem.lower() - name_parts = fstem_lower.split(separator) - - # Check map rules first - map_matched = False - for mapping_rule in map_type_mapping: - source_keywords, standard_map_type = mapping_rule - if standard_map_type not in self.config.standard_map_types: continue - for keyword in source_keywords: - kw_lower = keyword.lower().strip('*') - if kw_lower in name_parts: - is_exact_match = any(part == kw_lower for part in name_parts) - if is_exact_match: - candidate_stems.add(fstem) # Add unique stem - potential_map_files[fname] = standard_map_type # Store potential type - processed_filenames.add(fname) - map_matched = True - break # Found keyword match for this rule - if map_matched: break # Found a rule match for this file - if map_matched: continue # Move to next filename if identified as map - - # Check model patterns if not a map - for pattern in model_patterns: - if fnmatch(fname.lower(), pattern.lower()): - candidate_stems.add(fstem) # Still add stem for base name determination - processed_filenames.add(fname) - # Don't add models to potential_map_files - break # Found model match - - # Note: Files matching neither maps nor models are ignored for prediction details - - log.debug(f"[PREDICTION] Potential map files identified: {potential_map_files}") # DEBUG PREDICTION - candidate_stems_list = list(candidate_stems) # Convert set to list for commonprefix - log.debug(f"[PREDICTION] Candidate stems identified: {candidate_stems_list}") # DEBUG PREDICTION - if not candidate_stems_list: - log.warning(f"Prediction: No relevant map/model stems found in {self.input_path.name}. Using input name as fallback.") - # Fallback: Use the input path's name itself if no stems found - base_name_fallback = self.input_path.stem if self.input_path.is_file() else self.input_path.name - determined_base_name = base_name_fallback - else: - # 4. Replicate _determine_base_metadata logic for base name - determined_base_name = "UnknownAssetName" - separator = self.config.source_naming_separator - indices_dict = self.config.source_naming_indices - base_index_raw = indices_dict.get('base_name') - log.debug(f"[PREDICTION] Base Name Determination: Separator='{separator}', Indices Dict={indices_dict}, Raw Base Index='{base_index_raw}'") # DEBUG PREDICTION - - base_index = None - if base_index_raw is not None: - try: - base_index = int(base_index_raw) # Use explicit conversion like in main logic - except (ValueError, TypeError): - log.warning(f"[PREDICTION] Could not convert base_name index '{base_index_raw}' to integer.") - - if isinstance(base_index, int): - potential_base_names = set() - for stem in candidate_stems_list: # Iterate over the list - parts = stem.split(separator) - log.debug(f"[PREDICTION] Processing stem: '{stem}', Parts: {parts}") # DEBUG PREDICTION - if len(parts) > base_index: - extracted_name = parts[base_index] - potential_base_names.add(extracted_name) - log.debug(f"[PREDICTION] Extracted potential base name: '{extracted_name}' using index {base_index}") # DEBUG PREDICTION - else: - log.debug(f"[PREDICTION] Stem '{stem}' has too few parts ({len(parts)}) for index {base_index}.") # DEBUG PREDICTION - if len(potential_base_names) == 1: - determined_base_name = potential_base_names.pop() - log.debug(f"[PREDICTION] Determined base name '{determined_base_name}' from structured parts (index {base_index}).") # DEBUG PREDICTION - elif len(potential_base_names) > 1: - log.debug(f"[PREDICTION] Multiple potential base names found from index {base_index}: {potential_base_names}. Falling back to common prefix.") # DEBUG PREDICTION - determined_base_name = os.path.commonprefix(candidate_stems_list) # Use list here - determined_base_name = determined_base_name.strip(separator + ' _').rstrip(separator + ' _') - # else: Use common prefix below - - if determined_base_name == "UnknownAssetName" or not determined_base_name: - log.debug("[PREDICTION] Falling back to common prefix for base name determination (structured parts failed or no index).") # DEBUG PREDICTION - determined_base_name = os.path.commonprefix(candidate_stems_list) # Use list here - determined_base_name = determined_base_name.strip(separator + ' _').rstrip(separator + ' _') - - # 5. Sanitize Names - final_base_name = self._sanitize_filename(determined_base_name or "UnknownAssetName") - log.debug(f"[PREDICTION] Final determined base name for prediction: '{final_base_name}'") # DEBUG PREDICTION - final_supplier_name = self._sanitize_filename(supplier_name) - - # 6. Predict Output Filenames - file_predictions = {} - target_pattern = self.config.target_filename_pattern - # Use highest resolution key as a placeholder for prediction - highest_res_key = "Res?" # Fallback - if self.config.image_resolutions: - highest_res_key = max(self.config.image_resolutions, key=self.config.image_resolutions.get) - - for input_fname, map_type in potential_map_files.items(): - # Assume PNG for prediction, extension might change based on bit depth rules later - # but this gives a good idea of the renaming. - # A more complex prediction could check bit depth rules. - predicted_ext = "png" # Simple assumption for preview - try: - predicted_fname = target_pattern.format( - base_name=final_base_name, - map_type=map_type, - resolution=highest_res_key, # Use placeholder resolution - ext=predicted_ext - ) - file_predictions[input_fname] = predicted_fname - except KeyError as fmt_err: - log.warning(f"Prediction: Error formatting filename for {input_fname} (KeyError: {fmt_err}). Skipping file prediction.") - file_predictions[input_fname] = "[Filename Format Error]" - - - log.debug(f"Predicted structure: Supplier='{final_supplier_name}', Asset='{final_base_name}', Files={len(file_predictions)}") - return final_supplier_name, final_base_name, file_predictions - - except Exception as e: - log.error(f"Error during output structure prediction for {self.input_path.name}: {e}", exc_info=True) - return None - - - # --- New Detailed Prediction Method --- - def get_detailed_file_predictions(self) -> list[dict] | None: - """ - Performs extraction and classification to provide a detailed list of all - files found within the input and their predicted status/output name, - handling multiple potential assets within the input. - - Returns: - list[dict] | None: A list of dictionaries, each representing a file: - {'original_path': str, - 'predicted_asset_name': str | None, - 'predicted_output_name': str | None, - 'status': str, - 'details': str | None} - Returns None if a critical error occurs during setup/classification. - """ - log.info(f"Getting detailed file predictions for input: {self.input_path.name}") - results = [] - all_files_in_workspace = [] # Keep track of all files found - - try: - # --- Perform necessary setup and classification --- - self._setup_workspace() - self._extract_input() - # Run classification - this populates self.classified_files - self._inventory_and_classify_files() - - # --- Determine distinct assets and file mapping --- - # This uses the results from _inventory_and_classify_files - distinct_base_names, file_to_base_name_map = self._determine_base_metadata() - log.debug(f"Prediction: Determined base names: {distinct_base_names}") - log.debug(f"Prediction: File to base name map: { {str(k):v for k,v in file_to_base_name_map.items()} }") - - # --- Apply Suffixes for Prediction Preview --- - # This logic is similar to the main process method but applied to the classified_files list - log.debug("Prediction: Applying map type suffixes for preview...") - grouped_classified_maps = defaultdict(list) - for map_info in self.classified_files.get('maps', []): - # Group by the base map type - grouped_classified_maps[map_info['map_type']].append(map_info) - - # Create a new list for maps with updated types for prediction - maps_with_predicted_types = [] - for base_map_type, maps_in_group in grouped_classified_maps.items(): - respect_variants = base_map_type in self.config.respect_variant_map_types - # Sort maps within the group for consistent suffixing (using the same key as in _inventory_and_classify_files) - maps_in_group.sort(key=lambda c: ( - c.get('preset_rule_index', 9999), - c.get('keyword_index_in_rule', 9999) if 'keyword_index_in_rule' in c else 9999, # Handle potential missing key - str(c['source_path']) - )) - - for i, map_info in enumerate(maps_in_group): - predicted_map_type = f"{base_map_type}-{i + 1}" if respect_variants else base_map_type - # Create a copy to avoid modifying the original classified_files list in place - map_info_copy = map_info.copy() - map_info_copy['predicted_map_type'] = predicted_map_type # Store the predicted type - maps_with_predicted_types.append(map_info_copy) - - # Replace the original maps list with the one containing predicted types for the next step - # Note: This is a temporary list for prediction generation, not modifying the instance's classified_files permanently - # self.classified_files["maps"] = maps_with_predicted_types # Avoid modifying instance state - - # --- Prepare for filename prediction --- - target_pattern = self.config.target_filename_pattern - highest_res_key = "Res?" # Placeholder resolution for prediction - if self.config.image_resolutions: - highest_res_key = max(self.config.image_resolutions, key=self.config.image_resolutions.get) - - # --- Process all classified files (including maps with predicted types) --- - all_classified_files_with_category = [] - # Add maps with predicted types first - for map_info in maps_with_predicted_types: - map_info['category'] = 'maps' # Ensure category is set - all_classified_files_with_category.append(map_info) - if 'source_path' in map_info: - all_files_in_workspace.append(map_info['source_path']) - - # Add other categories (models, extra, ignored) - for category in ['models', 'extra', 'ignored']: - for file_info in self.classified_files.get(category, []): - file_info['category'] = category - all_classified_files_with_category.append(file_info) - if 'source_path' in file_info: - all_files_in_workspace.append(file_info['source_path']) - - - # --- Generate results for each file --- - processed_paths = set() # Track paths already added to results - for file_info in all_classified_files_with_category: - original_path = file_info.get("source_path") - if not original_path or original_path in processed_paths: - continue # Skip if path missing or already processed - - original_path_str = str(original_path) - processed_paths.add(original_path) # Mark as processed - - # Determine predicted asset name and status - predicted_asset_name = file_to_base_name_map.get(original_path) # Can be None - category = file_info['category'] # maps, models, extra, ignored - reason = file_info.get('reason') # Specific reason for extra/ignored - status = "Unknown" - details = None - predicted_output_name = None # Usually original name, except for maps - - if category == "maps": - status = "Mapped" - # Use the predicted_map_type for the preview display - map_type_for_preview = file_info.get("predicted_map_type", file_info.get("map_type", "UnknownType")) - details = f"[{map_type_for_preview}]" - if file_info.get("is_16bit_source"): details += " (16-bit)" - # Predict map output name using its determined asset name and predicted map type - if predicted_asset_name: - try: - predicted_ext = "png" # Assume PNG for prediction simplicity - predicted_output_name = target_pattern.format( - base_name=predicted_asset_name, - map_type=map_type_for_preview, # Use the predicted type here - resolution=highest_res_key, - ext=predicted_ext - ) - except KeyError as fmt_err: - log.warning(f"Prediction format error for map {original_path_str}: {fmt_err}") - predicted_output_name = "[Format Error]" - details += f" (Format Key Error: {fmt_err})" - except Exception as pred_err: - log.warning(f"Prediction error for map {original_path_str}: {pred_err}") - predicted_output_name = "[Prediction Error]" - details += f" (Error: {pred_err})" - else: - # Should not happen for maps if _determine_base_metadata worked correctly - log.warning(f"Map file '{original_path_str}' has no predicted asset name.") - predicted_output_name = "[No Asset Name]" - - elif category == "models": - status = "Model" - details = "[Model]" - predicted_output_name = original_path.name # Models keep original name - - elif category == "ignored": - status = "Ignored" - details = f"Ignored ({reason or 'Unknown reason'})" - predicted_output_name = None # Ignored files have no output - - elif category == "extra": - if predicted_asset_name is None: - # This is an "Unmatched Extra" file (includes Unrecognised and explicit Extras without a base name) - status = "Unmatched Extra" - details = f"[Unmatched Extra ({reason or 'N/A'})]" # Include original reason if available - elif reason == 'Unrecognised': - # Unrecognised but belongs to a specific asset - status = "Unrecognised" - details = "[Unrecognised]" - else: - # Explicitly matched an 'extra' pattern and belongs to an asset - status = "Extra" - details = f"Extra ({reason})" - predicted_output_name = original_path.name # Extra files keep original name - - else: - log.warning(f"Unknown category '{category}' encountered during prediction for {original_path_str}") - status = "Error" - details = f"[Unknown Category: {category}]" - predicted_output_name = original_path.name - - - results.append({ - "original_path": original_path_str, - "predicted_asset_name": predicted_asset_name, # May be None - "predicted_output_name": predicted_output_name, - "status": status, - "details": details - }) - - # Add any files found during walk but missed by classification (should be rare) - # These are likely unmatched as well. - for file_path in all_files_in_workspace: - if file_path not in processed_paths: - log.warning(f"File found in workspace but not classified: {file_path}. Adding as Unmatched Extra.") - results.append({ - "original_path": str(file_path), - "predicted_asset_name": None, # Explicitly None as it wasn't mapped - "predicted_output_name": file_path.name, - "status": "Unmatched Extra", - "details": "[Missed Classification]" - }) - - - log.info(f"Detailed prediction complete for input '{self.input_path.name}'. Found {len(results)} files.") - # Sort results by original path for consistent display - results.sort(key=lambda x: x.get("original_path", "")) - return results - - except (AssetProcessingError, ConfigurationError, Exception) as e: - log.error(f"Critical error during detailed prediction for {self.input_path.name}: {e}", exc_info=True) - return None # Indicate critical failure - finally: - # Ensure cleanup always happens - self._cleanup_workspace() - - -# --- End of AssetProcessor Class --- \ No newline at end of file diff --git a/autotest.py b/autotest.py new file mode 100644 index 0000000..1782ef2 --- /dev/null +++ b/autotest.py @@ -0,0 +1,863 @@ +import argparse +import sys +import logging +import logging.handlers +import time +import json +import shutil # Import shutil for directory operations +from pathlib import Path +from typing import List, Dict, Any + +from PySide6.QtCore import QCoreApplication, QTimer, Slot, QEventLoop, QObject, Signal +from PySide6.QtWidgets import QApplication, QListWidgetItem + +# Add project root to sys.path +project_root = Path(__file__).resolve().parent +if str(project_root) not in sys.path: + sys.path.insert(0, str(project_root)) + +try: + from main import App + from gui.main_window import MainWindow + from rule_structure import SourceRule # Assuming SourceRule is in rule_structure.py +except ImportError as e: + print(f"Error importing project modules: {e}") + print(f"Ensure that the script is run from the project root or that the project root is in PYTHONPATH.") + print(f"Current sys.path: {sys.path}") + sys.exit(1) + +# Global variable for the memory log handler +autotest_memory_handler = None + +# Custom Log Filter for Concise Output +class InfoSummaryFilter(logging.Filter): + # Keywords that identify INFO messages to *allow* for concise output + SUMMARY_KEYWORDS_PRECISE = [ + "Test run completed", + "Test succeeded", + "Test failed", + "Rule comparison successful", + "Rule comparison failed", + "ProcessingEngine finished. Summary:", + "Autotest Context:", + "Parsed CLI arguments:", + "Prediction completed successfully.", + "Processing completed.", + "Signal 'all_tasks_finished' received", + "final status:", # To catch "Asset '...' final status:" + "User settings file not found:", + "MainPanelWidget: Default output directory set to:", + # Search related (as per original filter) + "Searching logs for term", + "Search term ", + "Found ", + "No tracebacks found in the logs.", + "--- End Log Analysis ---", + "Log analysis completed.", + ] + # Patterns for case-insensitive rejection + REJECT_PATTERNS_LOWER = [ + # Original debug prefixes (ensure these are still relevant or merge if needed) + "debug:", "orchestrator_trace:", "configuration_debug:", "app_debug:", "output_org_debug:", + # Iterative / Per-item / Per-file details / Intermediate steps + ": item ", # Catches "Asset '...', Item X/Y" + "item successfully processed and saved", + ", file '", # Catches "Asset '...', File '...'" + ": processing regular map", + ": found source file:", + ": determined source bit depth:", + "successfully processed regular map", + "successfully created mergetaskdefinition", + ": preparing processing items", + ": finished preparing items. found", + ": starting core item processing loop", + ", task '", + ": processing merge task", + "loaded from context:", + "using dimensions from first loaded input", + "successfully merged inputs into image", + "successfully processed merge task", + "mergedtaskprocessorstage result", + "calling savevariantsstage", + "savevariantsstage result", + "adding final details to context", + ": finished core item processing loop", + ": copied variant", + ": copied extra file", + ": successfully organized", + ": output organization complete.", + ": metadata saved to", + "worker thread: starting processing for rule:", + "preparing workspace for input:", + "input is a supported archive", + "calling processingengine.process with rule", + "calculated sha5 for", + "calculated next incrementing value for", + "verify: processingengine.process called", + ": effective supplier set to", + ": metadata initialized.", + ": file rules queued for processing", + "successfully loaded base application settings", + "successfully loaded and merged asset_type_definitions", + "successfully loaded and merged file_type_definitions", + "starting rule-based prediction for:", + "rule-based prediction finished successfully for", + "finished rule-based prediction run for", + "updating model with rule-based results for source:", + "debug task ", + "worker thread: finished processing for rule:", + "task finished signal received for", + # Autotest step markers (not global summaries) + "step 1: loading zip file:", + "step 2: selecting preset:", + "step 4: retrieving and comparing rules...", + "step 5: starting processing...", + "step 7: checking output path:", + "output path check completed.", + ] + + def filter(self, record): + # Allow CRITICAL, ERROR, WARNING unconditionally + if record.levelno >= logging.WARNING: + return True + + if record.levelno == logging.INFO: + msg = record.getMessage() + msg_lower = msg.lower() # For case-insensitive pattern rejection + + # 1. Explicitly REJECT if message contains verbose patterns (case-insensitive) + for pattern in self.REJECT_PATTERNS_LOWER: # Use the new list + if pattern in msg_lower: + return False # Reject + + # 2. Then, if not rejected, ALLOW only if message contains precise summary keywords + for keyword in self.SUMMARY_KEYWORDS_PRECISE: # Use the new list + if keyword in msg: # Original message for case-sensitive summary keywords if needed + return True # Allow + + # 3. Reject all other INFO messages that don't match precise summary keywords + return False + + # Reject levels below INFO (e.g., DEBUG) by default for this handler + return False + +# --- Root Logger Configuration for Concise Console Output --- +def setup_autotest_logging(): + """ + Configures the root logger for concise console output for autotest.py. + This ensures that only essential summary information, warnings, and errors + are displayed on the console by default. + """ + root_logger = logging.getLogger() + + # 1. Remove all existing handlers from the root logger. + # This prevents interference from other logging configurations. + for handler in root_logger.handlers[:]: + root_logger.removeHandler(handler) + handler.close() # Close handler before removing + + # 2. Set the root logger's level to DEBUG to capture everything for the memory handler. + # The console handler will still filter down to INFO/selected. + root_logger.setLevel(logging.DEBUG) # Changed from INFO to DEBUG + + # 3. Create a new StreamHandler for sys.stdout (for concise console output). + console_handler = logging.StreamHandler(sys.stdout) + + # 4. Set this console handler's level to INFO. + # The filter will then decide which INFO messages to display on console. + console_handler.setLevel(logging.INFO) + + # 5. Apply the enhanced InfoSummaryFilter to the console handler. + info_filter = InfoSummaryFilter() + console_handler.addFilter(info_filter) + + # 6. Set a concise formatter for the console handler. + formatter = logging.Formatter('[%(levelname)s] %(message)s') + console_handler.setFormatter(formatter) + + # 7. Add this newly configured console handler to the root_logger. + root_logger.addHandler(console_handler) + + # 8. Setup the MemoryHandler + global autotest_memory_handler # Declare usage of global + autotest_memory_handler = logging.handlers.MemoryHandler( + capacity=20000, # Increased capacity + flushLevel=logging.CRITICAL + 1, # Prevent automatic flushing + target=None # Does not flush to another handler + ) + autotest_memory_handler.setLevel(logging.DEBUG) # Capture all logs from DEBUG up + # Not adding a formatter here, will format in _process_and_display_logs + + # 9. Add the memory handler to the root logger. + root_logger.addHandler(autotest_memory_handler) + +# Call the setup function early in the script's execution. +setup_autotest_logging() + +# Logger for autotest.py's own messages. +# Messages from this logger will propagate to the root logger and be filtered +# by the console_handler configured above. +# Setting its level to DEBUG allows autotest.py to generate DEBUG messages, +# which won't appear on the concise console (due to handler's INFO level) +# but can be captured by other handlers (e.g., the GUI's log console). +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) # Ensure autotest.py can generate DEBUGs for other handlers + +# Note: The GUI's log console (e.g., self.main_window.log_console.log_console_output) +# is assumed to capture all logs (including DEBUG) from various modules. +# The _process_and_display_logs function then uses these comprehensive logs for the --search feature. +# This root logger setup primarily makes autotest.py's direct console output concise, +# ensuring that only filtered, high-level information appears on stdout by default. +# --- End of Root Logger Configuration --- + +# --- Argument Parsing --- +def parse_arguments(): + """Parses command-line arguments for the autotest script.""" + parser = argparse.ArgumentParser(description="Automated test script for Asset Processor GUI.") + parser.add_argument( + "--zipfile", + type=Path, + default=project_root / "TestFiles" / "BoucleChunky001.zip", + help="Path to the test asset ZIP file. Default: TestFiles/BoucleChunky001.zip" + ) + parser.add_argument( + "--preset", + type=str, + default="Dinesen", # This should match a preset name in the application + help="Name of the preset to use. Default: Dinesen" + ) + parser.add_argument( + "--expectedrules", + type=Path, + default=project_root / "TestFiles" / "Test-BoucleChunky001.json", + help="Path to the JSON file with expected rules. Default: TestFiles/Test-BoucleChunky001.json" + ) + parser.add_argument( + "--outputdir", + type=Path, + default=project_root / "TestFiles" / "TestOutputs" / "BoucleChunkyOutput", + help="Path for processing output. Default: TestFiles/TestOutputs/BoucleChunkyOutput" + ) + parser.add_argument( + "--search", + type=str, + default=None, + help="Optional log search term. Default: None" + ) + parser.add_argument( + "--additional-lines", + type=int, + default=0, + help="Context lines for log search. Default: 0" + ) + return parser.parse_args() + +class AutoTester(QObject): + """ + Handles the automated testing process for the Asset Processor GUI. + """ + # Define signals if needed, e.g., for specific test events + # test_step_completed = Signal(str) + + def __init__(self, app_instance: App, cli_args: argparse.Namespace): + super().__init__() + self.app_instance: App = app_instance + self.main_window: MainWindow = app_instance.main_window + self.cli_args: argparse.Namespace = cli_args + self.event_loop = QEventLoop(self) + self.prediction_poll_timer = QTimer(self) + self.expected_rules_data: Dict[str, Any] = {} + self.test_step: str = "INIT" # Possible values: INIT, LOADING_ZIP, SELECTING_PRESET, AWAITING_PREDICTION, PREDICTION_COMPLETE, COMPARING_RULES, STARTING_PROCESSING, AWAITING_PROCESSING, PROCESSING_COMPLETE, CHECKING_OUTPUT, ANALYZING_LOGS, DONE + + if not self.main_window: + logger.error("MainWindow instance not found in App. Cannot proceed.") + self.cleanup_and_exit(success=False) + return + + # Connect signals + if hasattr(self.app_instance, 'all_tasks_finished') and isinstance(self.app_instance.all_tasks_finished, Signal): + self.app_instance.all_tasks_finished.connect(self._on_all_tasks_finished) + else: + logger.warning("App instance does not have 'all_tasks_finished' signal or it's not a Signal. Processing completion might not be detected.") + + self._load_expected_rules() + + def _load_expected_rules(self) -> None: + """Loads the expected rules from the JSON file specified by cli_args.""" + self.test_step = "LOADING_EXPECTED_RULES" + logger.debug(f"Loading expected rules from: {self.cli_args.expectedrules}") + try: + with open(self.cli_args.expectedrules, 'r') as f: + self.expected_rules_data = json.load(f) + logger.debug("Expected rules loaded successfully.") + except FileNotFoundError: + logger.error(f"Expected rules file not found: {self.cli_args.expectedrules}") + self.cleanup_and_exit(success=False) + except json.JSONDecodeError as e: + logger.error(f"Error decoding expected rules JSON: {e}") + self.cleanup_and_exit(success=False) + except Exception as e: + logger.error(f"An unexpected error occurred while loading expected rules: {e}") + self.cleanup_and_exit(success=False) + + def run_test(self) -> None: + """Orchestrates the test steps.""" + logger.info("Starting test run...") + + if not self.expected_rules_data: # Ensure rules were loaded + logger.error("Expected rules not loaded. Aborting test.") + self.cleanup_and_exit(success=False) + return + + # Add a specific summary log for essential context + logger.info(f"Autotest Context: Input='{self.cli_args.zipfile.name}', Preset='{self.cli_args.preset}', Output='{self.cli_args.outputdir}'") + + # Step 1: Load ZIP + self.test_step = "LOADING_ZIP" + logger.info(f"Step 1: Loading ZIP file: {self.cli_args.zipfile}") # KEEP INFO - Passes filter + if not self.cli_args.zipfile.exists(): + logger.error(f"ZIP file not found: {self.cli_args.zipfile}") + self.cleanup_and_exit(success=False) + return + try: + # Assuming add_input_paths can take a list of strings or Path objects + self.main_window.add_input_paths([str(self.cli_args.zipfile)]) + logger.debug("ZIP file loading initiated.") + except Exception as e: + logger.error(f"Error during ZIP file loading: {e}") + self.cleanup_and_exit(success=False) + return + + # Step 2: Select Preset + self.test_step = "SELECTING_PRESET" + logger.info(f"Step 2: Selecting preset: {self.cli_args.preset}") # KEEP INFO - Passes filter + preset_found = False + preset_list_widget = self.main_window.preset_editor_widget.editor_preset_list + for i in range(preset_list_widget.count()): + item = preset_list_widget.item(i) + if item and item.text() == self.cli_args.preset: + preset_list_widget.setCurrentItem(item) + logger.debug(f"Preset '{self.cli_args.preset}' selected.") + preset_found = True + break + if not preset_found: + logger.error(f"Preset '{self.cli_args.preset}' not found in the list.") + available_presets = [preset_list_widget.item(i).text() for i in range(preset_list_widget.count())] + logger.debug(f"Available presets: {available_presets}") + self.cleanup_and_exit(success=False) + return + + # Step 3: Await Prediction Completion + self.test_step = "AWAITING_PREDICTION" + logger.debug("Step 3: Awaiting prediction completion...") + self.prediction_poll_timer.timeout.connect(self._check_prediction_status) + self.prediction_poll_timer.start(500) # Poll every 500ms + + # Use a QTimer to allow event loop to process while waiting for this step + # This ensures that the _check_prediction_status can be called. + # We will exit this event_loop from _check_prediction_status when prediction is done. + logger.debug("Starting event loop for prediction...") + self.event_loop.exec() # This loop is quit by _check_prediction_status + self.prediction_poll_timer.stop() + logger.debug("Event loop for prediction finished.") + + + if self.test_step != "PREDICTION_COMPLETE": + logger.error(f"Prediction did not complete as expected. Current step: {self.test_step}") + # Check if there were any pending predictions that never cleared + if hasattr(self.main_window, '_pending_predictions'): + logger.error(f"Pending predictions at timeout: {self.main_window._pending_predictions}") + self.cleanup_and_exit(success=False) + return + logger.info("Prediction completed successfully.") # KEEP INFO - Passes filter + + # Step 4: Retrieve & Compare Rulelist + self.test_step = "COMPARING_RULES" + logger.info("Step 4: Retrieving and Comparing Rules...") # KEEP INFO - Passes filter + actual_source_rules_list: List[SourceRule] = self.main_window.unified_model.get_all_source_rules() + actual_rules_obj = actual_source_rules_list # Keep the SourceRule list for processing + + comparable_actual_rules = self._convert_rules_to_comparable(actual_source_rules_list) + + if not self._compare_rules(comparable_actual_rules, self.expected_rules_data): + logger.error("Rule comparison failed. See logs for details.") + self.cleanup_and_exit(success=False) + return + logger.info("Rule comparison successful.") # KEEP INFO - Passes filter + + # Step 5: Start Processing + self.test_step = "START_PROCESSING" + logger.info("Step 5: Starting Processing...") # KEEP INFO - Passes filter + processing_settings = { + "output_dir": str(self.cli_args.outputdir), # Ensure it's a string for JSON/config + "overwrite": True, + "workers": 1, + "blender_enabled": False # Basic test, no Blender + } + try: + Path(self.cli_args.outputdir).mkdir(parents=True, exist_ok=True) + logger.debug(f"Ensured output directory exists: {self.cli_args.outputdir}") + except Exception as e: + logger.error(f"Could not create output directory {self.cli_args.outputdir}: {e}") + self.cleanup_and_exit(success=False) + return + + if hasattr(self.main_window, 'start_backend_processing') and isinstance(self.main_window.start_backend_processing, Signal): + logger.debug(f"Emitting start_backend_processing with rules count: {len(actual_rules_obj)} and settings: {processing_settings}") + self.main_window.start_backend_processing.emit(actual_rules_obj, processing_settings) + else: + logger.error("'start_backend_processing' signal not found on MainWindow. Cannot start processing.") + self.cleanup_and_exit(success=False) + return + + # Step 6: Await Processing Completion + self.test_step = "AWAIT_PROCESSING" + logger.debug("Step 6: Awaiting processing completion...") + self.event_loop.exec() # This loop is quit by _on_all_tasks_finished + + if self.test_step != "PROCESSING_COMPLETE": + logger.error(f"Processing did not complete as expected. Current step: {self.test_step}") + self.cleanup_and_exit(success=False) + return + logger.info("Processing completed.") # KEEP INFO - Passes filter + + # Step 7: Check Output Path + self.test_step = "CHECK_OUTPUT" + logger.info(f"Step 7: Checking output path: {self.cli_args.outputdir}") # KEEP INFO - Passes filter + output_path = Path(self.cli_args.outputdir) + if not output_path.exists() or not output_path.is_dir(): + logger.error(f"Output directory {output_path} does not exist or is not a directory.") + self.cleanup_and_exit(success=False) + return + + output_items = list(output_path.iterdir()) + if not output_items: + logger.warning(f"Output directory {output_path} is empty. This might be a test failure depending on the case.") + # For a more specific check, one might iterate through actual_rules_obj + # and verify if subdirectories matching asset_name exist. + # e.g. for asset_rule in source_rule.assets: + # expected_asset_dir = output_path / asset_rule.asset_name + # if not expected_asset_dir.is_dir(): logger.error(...) + else: + logger.debug(f"Found {len(output_items)} item(s) in output directory:") + for item in output_items: + logger.debug(f" - {item.name} ({'dir' if item.is_dir() else 'file'})") + logger.info("Output path check completed.") # KEEP INFO - Passes filter + + # Step 8: Retrieve & Analyze Logs + self.test_step = "CHECK_LOGS" + logger.debug("Step 8: Retrieving and Analyzing Logs...") + all_logs_text = "" + if self.main_window.log_console and self.main_window.log_console.log_console_output: + all_logs_text = self.main_window.log_console.log_console_output.toPlainText() + else: + logger.warning("Log console or output widget not found. Cannot retrieve logs.") + + self._process_and_display_logs(all_logs_text) + logger.info("Log analysis completed.") + + # Final Step + logger.info("Test run completed successfully.") # KEEP INFO - Passes filter + self.cleanup_and_exit(success=True) + + @Slot() + def _check_prediction_status(self) -> None: + """Polls the main window for pending predictions.""" + # logger.debug(f"Checking prediction status. Pending: {self.main_window._pending_predictions if hasattr(self.main_window, '_pending_predictions') else 'N/A'}") + if hasattr(self.main_window, '_pending_predictions'): + if not self.main_window._pending_predictions: # Assuming _pending_predictions is a list/dict that's empty when done + logger.debug("No pending predictions. Prediction assumed complete.") + self.test_step = "PREDICTION_COMPLETE" + if self.event_loop.isRunning(): + self.event_loop.quit() + # else: + # logger.debug(f"Still awaiting predictions: {len(self.main_window._pending_predictions)} remaining.") + else: + logger.warning("'_pending_predictions' attribute not found on MainWindow. Cannot check prediction status automatically.") + # As a fallback, if the attribute is missing, we might assume prediction is instant or needs manual check. + # For now, let's assume it means it's done if the attribute is missing, but this is risky. + # A better approach would be to have a clear signal from MainWindow when predictions are done. + self.test_step = "PREDICTION_COMPLETE" # Risky assumption + if self.event_loop.isRunning(): + self.event_loop.quit() + + + @Slot(int, int, int) + def _on_all_tasks_finished(self, processed_count: int, skipped_count: int, failed_count: int) -> None: + """Slot for App.all_tasks_finished signal.""" + logger.info(f"Signal 'all_tasks_finished' received: Processed={processed_count}, Skipped={skipped_count}, Failed={failed_count}") # KEEP INFO - Passes filter + + if self.test_step == "AWAIT_PROCESSING": + logger.debug("Processing completion signal received.") # Covered by the summary log above + if failed_count > 0: + logger.error(f"Processing finished with {failed_count} failed task(s).") + # Even if tasks failed, the test might pass based on output checks. + # The error is logged for information. + self.test_step = "PROCESSING_COMPLETE" + if self.event_loop.isRunning(): + self.event_loop.quit() + else: + logger.warning(f"Signal 'all_tasks_finished' received at an unexpected test step: '{self.test_step}'. Counts: P={processed_count}, S={skipped_count}, F={failed_count}") + + + def _convert_rules_to_comparable(self, source_rules_list: List[SourceRule]) -> Dict[str, Any]: + """ + Converts a list of SourceRule objects to a dictionary structure + suitable for comparison with the expected_rules.json. + """ + logger.debug(f"Converting {len(source_rules_list)} SourceRule objects to comparable dictionary...") + comparable_sources_list = [] + for source_rule_obj in source_rules_list: + comparable_asset_list = [] + # source_rule_obj.assets is List[AssetRule] + for asset_rule_obj in source_rule_obj.assets: + comparable_file_list = [] + # asset_rule_obj.files is List[FileRule] + for file_rule_obj in asset_rule_obj.files: + comparable_file_list.append({ + "file_path": file_rule_obj.file_path, + "item_type": file_rule_obj.item_type, + "target_asset_name_override": file_rule_obj.target_asset_name_override + }) + comparable_asset_list.append({ + "asset_name": asset_rule_obj.asset_name, + "asset_type": asset_rule_obj.asset_type, + "files": comparable_file_list + }) + comparable_sources_list.append({ + "input_path": Path(source_rule_obj.input_path).name, # Use only the filename + "supplier_identifier": source_rule_obj.supplier_identifier, + "preset_name": source_rule_obj.preset_name, + "assets": comparable_asset_list + }) + logger.debug("Conversion to comparable dictionary finished.") + return {"source_rules": comparable_sources_list} + + def _compare_rule_item(self, actual_item: Dict[str, Any], expected_item: Dict[str, Any], item_type_name: str, parent_context: str = "") -> bool: + """ + Recursively compares an individual actual rule item dictionary with an expected rule item dictionary. + Logs differences and returns True if they match, False otherwise. + """ + item_match = True + + identifier = "" + if item_type_name == "SourceRule": + identifier = expected_item.get('input_path', f'UnknownSource_at_{parent_context}') + elif item_type_name == "AssetRule": + identifier = expected_item.get('asset_name', f'UnknownAsset_at_{parent_context}') + elif item_type_name == "FileRule": + identifier = expected_item.get('file_path', f'UnknownFile_at_{parent_context}') + + current_context = f"{parent_context}/{identifier}" if parent_context else identifier + + # Log Extra Fields: Iterate through keys in actual_item. + # If a key is in actual_item but not in expected_item (and is not a list container like "assets" or "files"), + # log this as an informational message. + for key in actual_item.keys(): + if key not in expected_item and key not in ["assets", "files"]: + logger.debug(f"Field '{key}' present in actual {item_type_name} ({current_context}) but not specified in expected. Value: '{actual_item[key]}'") + + # Check Expected Fields: Iterate through keys in expected_item. + for key, expected_value in expected_item.items(): + if key not in actual_item: + logger.error(f"Missing expected field '{key}' in actual {item_type_name} ({current_context}).") + item_match = False + continue # Continue to check other fields in the expected_item + + actual_value = actual_item[key] + + if key == "assets": # List of AssetRule dictionaries + if not self._compare_list_of_rules(actual_value, expected_value, "AssetRule", current_context, "asset_name"): + item_match = False + elif key == "files": # List of FileRule dictionaries + if not self._compare_list_of_rules(actual_value, expected_value, "FileRule", current_context, "file_path"): + item_match = False + else: # Regular field comparison + if actual_value != expected_value: + # Handle None vs "None" string for preset_name specifically if it's a common issue + if key == "preset_name" and actual_value is None and expected_value == "None": + logger.debug(f"Field '{key}' in {item_type_name} ({current_context}): Actual is None, Expected is string \"None\". Treating as match for now.") + elif key == "target_asset_name_override" and actual_value is not None and expected_value is None: + # If actual has a value (e.g. parent asset name) and expected is null/None, + # this is a mismatch according to strict comparison. + # For a more lenient check, this logic could be adjusted here. + # Current strict comparison will flag this as error, which is what the logs show. + logger.error(f"Value mismatch for field '{key}' in {item_type_name} ({current_context}): Actual='{actual_value}', Expected='{expected_value}'.") + item_match = False + else: + logger.error(f"Value mismatch for field '{key}' in {item_type_name} ({current_context}): Actual='{actual_value}', Expected='{expected_value}'.") + item_match = False + + return item_match + + def _compare_list_of_rules(self, actual_list: List[Dict[str, Any]], expected_list: List[Dict[str, Any]], item_type_name: str, parent_context: str, item_key_field: str) -> bool: + """ + Compares a list of actual rule items against a list of expected rule items. + Items are matched by a key field (e.g., 'asset_name' or 'file_path'). + Order independent for matching, but logs count mismatches. + """ + list_match = True # Corrected indentation + if not isinstance(actual_list, list) or not isinstance(expected_list, list): + logger.error(f"Type mismatch for list of {item_type_name}s in {parent_context}. Expected lists.") + return False + + if len(actual_list) != len(expected_list): + logger.error(f"Mismatch in number of {item_type_name}s for {parent_context}. Actual: {len(actual_list)}, Expected: {len(expected_list)}.") + list_match = False # Count mismatch is an error + # If counts differ, we still try to match what we can to provide more detailed feedback, + # but the overall list_match will remain False. + + actual_items_map = {item.get(item_key_field): item for item in actual_list if item.get(item_key_field) is not None} + + # Keep track of expected items that found a match to identify missing ones more easily + matched_expected_keys = set() + + for expected_item in expected_list: + expected_key_value = expected_item.get(item_key_field) + if expected_key_value is None: + logger.error(f"Expected {item_type_name} in {parent_context} is missing key field '{item_key_field}'. Cannot compare this item: {expected_item}") + list_match = False # This specific expected item cannot be processed + continue + + actual_item = actual_items_map.get(expected_key_value) + if actual_item: + matched_expected_keys.add(expected_key_value) + if not self._compare_rule_item(actual_item, expected_item, item_type_name, parent_context): + list_match = False # Individual item comparison failed + else: + logger.error(f"Expected {item_type_name} with {item_key_field} '{expected_key_value}' not found in actual items for {parent_context}.") + list_match = False + + # Identify actual items that were not matched by any expected item + # This is useful if len(actual_list) >= len(expected_list) but some actual items are "extra" + for actual_key_value, actual_item_data in actual_items_map.items(): + if actual_key_value not in matched_expected_keys: + logger.debug(f"Extra actual {item_type_name} with {item_key_field} '{actual_key_value}' found in {parent_context} (not in expected list or already matched).") + if len(actual_list) != len(expected_list): # If counts already flagged a mismatch, this is just detail + pass + else: # Counts matched, but content didn't align perfectly by key + list_match = False + + + return list_match # Corrected indentation + + def _compare_rules(self, actual_rules_data: Dict[str, Any], expected_rules_data: Dict[str, Any]) -> bool: # Corrected structure: moved out + item_match = False + + return item_match + + def _compare_rules(self, actual_rules_data: Dict[str, Any], expected_rules_data: Dict[str, Any]) -> bool: + """ + Compares the actual rule data (converted from live SourceRule objects) + with the expected rule data (loaded from JSON). + """ + logger.debug("Comparing actual rules with expected rules...") + + actual_source_rules = actual_rules_data.get("source_rules", []) if actual_rules_data else [] + expected_source_rules = expected_rules_data.get("source_rules", []) if expected_rules_data else [] + + if not isinstance(actual_source_rules, list): + logger.error(f"Actual 'source_rules' is not a list. Found type: {type(actual_source_rules)}. Comparison aborted.") + return False # Cannot compare if actual data is malformed + if not isinstance(expected_source_rules, list): + logger.error(f"Expected 'source_rules' is not a list. Found type: {type(expected_source_rules)}. Test configuration error. Comparison aborted.") + return False # Test setup error + + if not expected_source_rules and not actual_source_rules: + logger.debug("Both expected and actual source rules lists are empty. Considered a match.") + return True + + if len(actual_source_rules) != len(expected_source_rules): + logger.error(f"Mismatch in the number of source rules. Actual: {len(actual_source_rules)}, Expected: {len(expected_source_rules)}.") + # Optionally, log more details about which list is longer/shorter or identifiers if available + return False + + overall_match_status = True + for i in range(len(expected_source_rules)): + actual_sr = actual_source_rules[i] + expected_sr = expected_source_rules[i] + + # For context, use input_path or an index + source_rule_context = expected_sr.get('input_path', f"SourceRule_index_{i}") + + if not self._compare_rule_item(actual_sr, expected_sr, "SourceRule", parent_context=source_rule_context): + overall_match_status = False + # Continue checking other source rules to log all discrepancies + + if overall_match_status: + logger.debug("All rules match the expected criteria.") # Covered by "Rule comparison successful" summary + else: + logger.warning("One or more rules did not match the expected criteria. See logs above for details.") + + return overall_match_status + + def _process_and_display_logs(self, logs_text: str) -> None: # logs_text is no longer the primary source for search + """ + Processes and displays logs, potentially filtering them if --search is used. + Also checks for tracebacks. + Sources logs from the in-memory handler for search and detailed analysis. + """ + logger.debug("--- Log Analysis ---") + global autotest_memory_handler # Access the global handler + log_records = [] + if autotest_memory_handler and autotest_memory_handler.buffer: + log_records = autotest_memory_handler.buffer + + formatted_log_lines = [] + # Define a consistent formatter, similar to what might be expected or useful for search + record_formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s') + # Default asctime format includes milliseconds. + + + for record in log_records: + formatted_log_lines.append(record_formatter.format(record)) + + lines_for_search_and_traceback = formatted_log_lines + + if not lines_for_search_and_traceback: + logger.warning("No log records found in memory handler. No analysis to perform.") + # Still check the console logs_text for tracebacks if it exists, as a fallback + # or if some critical errors didn't make it to the memory handler (unlikely with DEBUG level) + if logs_text: + logger.debug("Checking provided logs_text (from console) for tracebacks as a fallback.") + console_lines = logs_text.splitlines() + traceback_found_console = False + for i, line in enumerate(console_lines): + if line.strip().startswith("Traceback (most recent call last):"): + logger.error(f"!!! TRACEBACK DETECTED in console logs_text around line {i+1} !!!") + traceback_found_console = True + if traceback_found_console: + logger.warning("A traceback was found in the console logs_text.") + else: + logger.info("No tracebacks found in the console logs_text either.") + logger.info("--- End Log Analysis ---") + return + + traceback_found = False + + if self.cli_args.search: + logger.info(f"Searching {len(lines_for_search_and_traceback)} in-memory log lines for term '{self.cli_args.search}' with {self.cli_args.additional_lines} context lines.") + matched_line_indices = [i for i, line in enumerate(lines_for_search_and_traceback) if self.cli_args.search in line] + + if not matched_line_indices: + logger.info(f"Search term '{self.cli_args.search}' not found in in-memory logs.") + else: + logger.info(f"Found {len(matched_line_indices)} match(es) for '{self.cli_args.search}' in in-memory logs:") + collected_lines_to_print = set() + for match_idx in matched_line_indices: + start_idx = max(0, match_idx - self.cli_args.additional_lines) + end_idx = min(len(lines_for_search_and_traceback), match_idx + self.cli_args.additional_lines + 1) + for i in range(start_idx, end_idx): + # Use i directly as index for lines_for_search_and_traceback, line number is for display + collected_lines_to_print.add(f"L{i+1:05d}: {lines_for_search_and_traceback[i]}") + + print("--- Filtered Log Output (from Memory Handler) ---") + for line_to_print in sorted(list(collected_lines_to_print)): + print(line_to_print) + print("--- End Filtered Log Output ---") + # Removed: else block that showed last N lines by default (as per original instruction for this section) + + # Traceback Check (on lines_for_search_and_traceback) + for i, line in enumerate(lines_for_search_and_traceback): + if line.strip().startswith("Traceback (most recent call last):") or "Traceback (most recent call last):" in line : # More robust check + logger.error(f"!!! TRACEBACK DETECTED in in-memory logs around line index {i} !!!") + logger.error(f"Line content: {line}") + traceback_found = True + + if traceback_found: + logger.warning("A traceback was found in the in-memory logs. This usually indicates a significant issue.") + else: + logger.info("No tracebacks found in the in-memory logs.") # This refers to the comprehensive memory logs + + logger.info("--- End Log Analysis ---") + + def cleanup_and_exit(self, success: bool = True) -> None: + """Cleans up and exits the application.""" + global autotest_memory_handler + if autotest_memory_handler: + logger.debug("Clearing memory log handler buffer and removing handler.") + autotest_memory_handler.buffer = [] # Clear buffer + logging.getLogger().removeHandler(autotest_memory_handler) # Remove handler + autotest_memory_handler.close() # MemoryHandler close is a no-op but good practice + autotest_memory_handler = None + + logger.info(f"Test {'succeeded' if success else 'failed'}. Cleaning up and exiting...") # KEEP INFO - Passes filter + q_app = QCoreApplication.instance() + if q_app: + q_app.quit() + sys.exit(0 if success else 1) + +# --- Main Execution --- +def main(): + """Main function to run the autotest script.""" + cli_args = parse_arguments() + # Logger is configured above, this will now use the new filtered setup + logger.info(f"Parsed CLI arguments: {cli_args}") # KEEP INFO - Passes filter + + # Clean and ensure output directory exists + output_dir_path = Path(cli_args.outputdir) + logger.debug(f"Preparing output directory: {output_dir_path}") + try: + if output_dir_path.exists(): + logger.debug(f"Output directory {output_dir_path} exists. Cleaning its contents...") + for item in output_dir_path.iterdir(): + if item.is_dir(): + shutil.rmtree(item) + logger.debug(f"Removed directory: {item}") + else: + item.unlink() + logger.debug(f"Removed file: {item}") + logger.debug(f"Contents of {output_dir_path} cleaned.") + else: + logger.debug(f"Output directory {output_dir_path} does not exist. Creating it.") + + output_dir_path.mkdir(parents=True, exist_ok=True) # Ensure it exists after cleaning/if it didn't exist + logger.debug(f"Output directory {output_dir_path} is ready.") + + except Exception as e: + logger.error(f"Could not prepare output directory {output_dir_path}: {e}", exc_info=True) + sys.exit(1) + + # Initialize QApplication + # Use QCoreApplication if no GUI elements are directly interacted with by the test logic itself, + # but QApplication is needed if MainWindow or its widgets are constructed and used. + # Since MainWindow is instantiated by App, QApplication is appropriate. + q_app = QApplication.instance() + if not q_app: + q_app = QApplication(sys.argv) + if not q_app: # Still no app + logger.error("Failed to initialize QApplication.") + sys.exit(1) + + logger.debug("Initializing main.App()...") + try: + # Instantiate main.App() - this should create MainWindow but not show it by default + # if App is designed to not show GUI unless app.main_window.show() is called. + app_instance = App() + except Exception as e: + logger.error(f"Failed to initialize main.App: {e}", exc_info=True) + sys.exit(1) + + if not app_instance.main_window: + logger.error("main.App initialized, but main_window is None. Cannot proceed with test.") + sys.exit(1) + + logger.debug("Initializing AutoTester...") + try: + tester = AutoTester(app_instance, cli_args) + except Exception as e: + logger.error(f"Failed to initialize AutoTester: {e}", exc_info=True) + sys.exit(1) + + # Use QTimer.singleShot to start the test after the Qt event loop has started. + # This ensures that the Qt environment is fully set up. + logger.debug("Scheduling test run...") + QTimer.singleShot(0, tester.run_test) + + logger.debug("Starting Qt application event loop...") + exit_code = q_app.exec() + logger.debug(f"Qt application event loop finished with exit code: {exit_code}") + sys.exit(exit_code) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/blender_addon/material_merger/__init__.py b/blender_addon/material_merger/__init__.py index 069ab6d..841d774 100644 --- a/blender_addon/material_merger/__init__.py +++ b/blender_addon/material_merger/__init__.py @@ -12,18 +12,15 @@ bl_info = { import bpy -# Import other modules (will be created later) from . import operator from . import panel def register(): - # Register classes from imported modules operator.register() panel.register() print("Material Merger Addon Registered") def unregister(): - # Unregister classes from imported modules panel.unregister() operator.unregister() print("Material Merger Addon Unregistered") diff --git a/blender_addon/material_merger/operator.py b/blender_addon/material_merger/operator.py index 552b835..cbf5f8f 100644 --- a/blender_addon/material_merger/operator.py +++ b/blender_addon/material_merger/operator.py @@ -10,7 +10,6 @@ MATERIAL_MERGE_NODEGROUP_NAME = "MaterialMerge" HANDLER_NODEGROUP_NAME = "PBR_Handler" # Assumption from plan BSDF_NODEGROUP_NAME = "PBR_BSDF" # Assumption from plan -# Helper function to copy nodes and identify outputs def copy_material_nodes(source_mat, target_tree, location_offset=(0, 0)): """ Copies nodes from source_mat's node tree to target_tree, applying an offset. @@ -25,7 +24,7 @@ def copy_material_nodes(source_mat, target_tree, location_offset=(0, 0)): return None, None, None source_tree = source_mat.node_tree - copied_node_map = {} # Map original node to copied node + copied_node_map = {} copied_final_bsdf_node = None copied_final_disp_node = None @@ -50,7 +49,6 @@ def copy_material_nodes(source_mat, target_tree, location_offset=(0, 0)): print(f" Identified top-level '{MATERIAL_MERGE_NODEGROUP_NAME}' in '{source_mat.name}'. Using its outputs.") source_final_bsdf_node = top_merge_node source_final_disp_node = top_merge_node # Both outputs come from the merge node - # Ensure the sockets exist before proceeding if 'BSDF' not in source_final_bsdf_node.outputs or 'Displacement' not in source_final_disp_node.outputs: print(f" Error: Identified merge node in '{source_mat.name}' lacks required BSDF/Displacement outputs.") return None, None, None @@ -65,7 +63,6 @@ def copy_material_nodes(source_mat, target_tree, location_offset=(0, 0)): if not source_final_disp_node: print(f" Error: Could not find base Handler node '{HANDLER_NODEGROUP_NAME}' in '{source_mat.name}'.") return None, None, None - # Ensure sockets exist if 'BSDF' not in source_final_bsdf_node.outputs: print(f" Error: Identified BSDF node '{BSDF_NODEGROUP_NAME}' lacks BSDF output.") return None, None, None @@ -152,7 +149,6 @@ class MATERIAL_OT_merge_materials(Operator): bl_label = "Merge Selected Materials" bl_options = {'REGISTER', 'UNDO'} - # Properties to hold the names of the selected materials # These will be set by the UI panel material_a_name: StringProperty( name="Material A", @@ -195,14 +191,14 @@ class MATERIAL_OT_merge_materials(Operator): # Add Material Output node output_node = new_node_tree.nodes.new(type='ShaderNodeOutputMaterial') - output_node.location = (400, 0) # Basic positioning + output_node.location = (400, 0) # 2. Copy nodes from source materials print("Copying nodes for Material A...") copied_map_a, copied_bsdf_a, copied_disp_a = copy_material_nodes(mat_a, new_node_tree, location_offset=(0, 0)) if not copied_bsdf_a or not copied_disp_a: self.report({'ERROR'}, f"Failed to copy nodes or identify outputs for material '{mat_a.name}'. Check console for details.") - bpy.data.materials.remove(new_mat) # Clean up + bpy.data.materials.remove(new_mat) return {'CANCELLED'} print("Copying nodes for Material B...") @@ -216,7 +212,7 @@ class MATERIAL_OT_merge_materials(Operator): copied_map_b, copied_bsdf_b, copied_disp_b = copy_material_nodes(mat_b, new_node_tree, location_offset=(offset_x, 0)) if not copied_bsdf_b or not copied_disp_b: self.report({'ERROR'}, f"Failed to copy nodes or identify outputs for material '{mat_b.name}'. Check console for details.") - bpy.data.materials.remove(new_mat) # Clean up + bpy.data.materials.remove(new_mat) return {'CANCELLED'} @@ -255,17 +251,14 @@ class MATERIAL_OT_merge_materials(Operator): # Add the linked/appended group to the new material's node tree merge_node = new_node_tree.nodes.new(type='ShaderNodeGroup') merge_node.node_tree = merge_group - merge_node.label = MATERIAL_MERGE_NODEGROUP_NAME # Set label for clarity - merge_node.location = (200, 0) # Basic positioning + merge_node.label = MATERIAL_MERGE_NODEGROUP_NAME + merge_node.location = (200, 0) # 4. Make Connections links = new_node_tree.links # Connect BSDFs to Merge node - # NOTE: Using original nodes here as placeholder. Needs to use *copied* nodes. - # NOTE: Using *copied* nodes now. - # Ensure the sockets exist before linking bsdf_output_socket_a = copied_bsdf_a.outputs.get('BSDF') shader_input_socket_a = merge_node.inputs.get('Shader A') bsdf_output_socket_b = copied_bsdf_b.outputs.get('BSDF') @@ -273,16 +266,13 @@ class MATERIAL_OT_merge_materials(Operator): if not all([bsdf_output_socket_a, shader_input_socket_a, bsdf_output_socket_b, shader_input_socket_b]): self.report({'ERROR'}, "Could not find required BSDF/Shader sockets for linking.") - bpy.data.materials.remove(new_mat) # Clean up + bpy.data.materials.remove(new_mat) return {'CANCELLED'} link_bsdf_a = links.new(bsdf_output_socket_a, shader_input_socket_a) link_bsdf_b = links.new(bsdf_output_socket_b, shader_input_socket_b) # Connect Displacements to Merge node - # NOTE: Using original nodes here as placeholder. Needs to use *copied* nodes. - # NOTE: Using *copied* nodes now. - # Ensure the sockets exist before linking disp_output_socket_a = copied_disp_a.outputs.get('Displacement') disp_input_socket_a = merge_node.inputs.get('Displacement A') disp_output_socket_b = copied_disp_b.outputs.get('Displacement') @@ -290,14 +280,13 @@ class MATERIAL_OT_merge_materials(Operator): if not all([disp_output_socket_a, disp_input_socket_a, disp_output_socket_b, disp_input_socket_b]): self.report({'ERROR'}, "Could not find required Displacement sockets for linking.") - bpy.data.materials.remove(new_mat) # Clean up + bpy.data.materials.remove(new_mat) return {'CANCELLED'} link_disp_a = links.new(disp_output_socket_a, disp_input_socket_a) link_disp_b = links.new(disp_output_socket_b, disp_input_socket_b) # Connect Merge node outputs to Material Output - # Ensure the sockets exist before linking merge_bsdf_output = merge_node.outputs.get('BSDF') output_surface_input = output_node.inputs.get('Surface') merge_disp_output = merge_node.outputs.get('Displacement') @@ -305,7 +294,7 @@ class MATERIAL_OT_merge_materials(Operator): if not all([merge_bsdf_output, output_surface_input, merge_disp_output, output_disp_input]): self.report({'ERROR'}, "Could not find required Merge/Output sockets for linking.") - bpy.data.materials.remove(new_mat) # Clean up + bpy.data.materials.remove(new_mat) return {'CANCELLED'} link_merge_bsdf = links.new(merge_bsdf_output, output_surface_input) @@ -315,7 +304,6 @@ class MATERIAL_OT_merge_materials(Operator): # 5. Layout (Optional) # TODO: Implement better node layout - # Update node tree to apply changes new_node_tree.nodes.update() self.report({'INFO'}, f"Successfully merged '{mat_a.name}' and '{mat_b.name}' into '{new_mat.name}'") @@ -323,10 +311,7 @@ class MATERIAL_OT_merge_materials(Operator): return {'FINISHED'} # Optional: Add invoke method if needed for more complex setup before execute - # def invoke(self, context, event): - # # Example: Open a dialog to select materials if not already selected - # return context.window_manager.invoke_props_dialog(self) - + # Commented-out code moved to Deprecated/Old-Code/blender_addon_material_merger_operator_py_invoke_method_line_326.py def register(): bpy.utils.register_class(MATERIAL_OT_merge_materials) diff --git a/blender_addon/material_merger/panel.py b/blender_addon/material_merger/panel.py index 45e5b78..c0418c7 100644 --- a/blender_addon/material_merger/panel.py +++ b/blender_addon/material_merger/panel.py @@ -1,6 +1,6 @@ import bpy from bpy.types import Panel -from .operator import MATERIAL_OT_merge_materials # Import the operator +from .operator import MATERIAL_OT_merge_materials class MATERIAL_PT_material_merger_panel(Panel): """Creates a Panel in the Shader Editor sidebar""" @@ -13,9 +13,6 @@ class MATERIAL_PT_material_merger_panel(Panel): def draw(self, context): layout = self.layout - # Get the active material in the Shader Editor - # This might be useful for defaulting one of the selectors - # mat = context.material row = layout.row() row.label(text="Select Materials to Merge:") @@ -25,18 +22,14 @@ class MATERIAL_PT_material_merger_panel(Panel): # We'll use StringProperty for simplicity in the UI for now. # A more advanced UI might use PointerProperty to bpy.data.materials - # Material A selection row = layout.row() row.prop(context.scene, "material_merger_mat_a", text="Material A") - # Material B selection row = layout.row() row.prop(context.scene, "material_merger_mat_b", text="Material B") - # Merge button row = layout.row() - # Pass the selected material names to the operator when button is clicked row.operator(MATERIAL_OT_merge_materials.bl_idname, text=MATERIAL_OT_merge_materials.bl_label).material_a_name = context.scene.material_merger_mat_a row.operator(MATERIAL_OT_merge_materials.bl_idname, text=MATERIAL_OT_merge_materials.bl_label).material_b_name = context.scene.material_merger_mat_b diff --git a/blenderscripts/create_materials.py b/blenderscripts/create_materials.py index a5f9642..34d19b9 100644 --- a/blenderscripts/create_materials.py +++ b/blenderscripts/create_materials.py @@ -20,7 +20,7 @@ import json from pathlib import Path import time import base64 # Although not directly used here, keep for consistency if reusing more code later -import sys # <<< ADDED IMPORT +import sys # --- USER CONFIGURATION --- @@ -55,9 +55,9 @@ REFERENCE_MAP_TYPES = ["COL", "COL-1", "COL-2"] REFERENCE_RESOLUTION_ORDER = ["1K", "512", "2K", "4K"] # Adjust as needed # Assumed filename pattern for processed images. -# {asset_name}, {map_type}, {resolution}, {format} will be replaced. -# Check Asset Processor Tool's config.py (TARGET_FILENAME_PATTERN) if this is wrong. -IMAGE_FILENAME_PATTERN = "{asset_name}_{map_type}_{resolution}.{format}" +# [assetname], [maptype], [resolution], [ext] will be replaced. +# This should match OUTPUT_FILENAME_PATTERN from app_settings.json. +IMAGE_FILENAME_PATTERN = "[assetname]_[maptype]_[resolution].[ext]" # Fallback extensions to try if the primary format from metadata is not found # Order matters - first found will be used. @@ -126,16 +126,14 @@ def reconstruct_image_path_with_fallback(asset_dir_path, asset_name, map_type, r if primary_format: try: filename = IMAGE_FILENAME_PATTERN.format( - asset_name=asset_name, - map_type=map_type, - resolution=resolution, - format=primary_format.lower() # Ensure format is lowercase + assetname=asset_name, # Token is 'assetname' + maptype=map_type, # Token is 'maptype' + resolution=resolution, # Token is 'resolution' + ext=primary_format.lower() # Token is 'ext' ) primary_path = asset_dir_path / filename if primary_path.is_file(): - # print(f" Found primary path: {str(primary_path)}") # Verbose return str(primary_path) - # else: print(f" Primary path not found: {str(primary_path)}") # Verbose except KeyError as e: print(f" !!! ERROR: Missing key '{e}' in IMAGE_FILENAME_PATTERN. Cannot reconstruct path.") return None # Cannot proceed without valid pattern @@ -144,17 +142,16 @@ def reconstruct_image_path_with_fallback(asset_dir_path, asset_name, map_type, r # Continue to fallback # 2. Try fallback extensions - # print(f" Trying fallback extensions for {map_type}/{resolution}...") # Verbose for ext in FALLBACK_IMAGE_EXTENSIONS: # Skip if we already tried this extension as primary (and it failed) if primary_format and ext.lower() == primary_format.lower(): continue try: fallback_filename = IMAGE_FILENAME_PATTERN.format( - asset_name=asset_name, - map_type=map_type, - resolution=resolution, - format=ext.lower() + assetname=asset_name, # Token is 'assetname' + maptype=map_type, # Token is 'maptype' + resolution=resolution, # Token is 'resolution' + ext=ext.lower() # Token is 'ext' ) fallback_path = asset_dir_path / fallback_filename if fallback_path.is_file(): @@ -198,9 +195,7 @@ def get_stat_value(stats_dict, map_type_list, stat_key): if isinstance(map_stats, dict) and stat_key in map_stats: return map_stats[stat_key] # Return the value for the first match else: - # print(f" Debug: Stats for '{map_type}' found but key '{stat_key}' or format is invalid.") # Optional debug pass # Continue checking other map types in the list - # else: print(f" Debug: Map type '{map_type}' not found in stats_dict.") # Optional debug return None # Return None if no matching map type or stat key was found @@ -214,11 +209,11 @@ def process_library_for_materials(context, asset_library_root_override=None, nod Scans the library, reads metadata, finds PBRSET node groups in the specified .blend file, and creates/updates materials linking to them. """ - print("DEBUG: Script started.") # DEBUG LOG + print("DEBUG: Script started.") start_time = time.time() print(f"\n--- Starting Material Creation from Node Groups ({time.strftime('%Y-%m-%d %H:%M:%S')}) ---") - print(f" DEBUG: Received asset_library_root_override: {asset_library_root_override}") # DEBUG LOG (Indented) - print(f" DEBUG: Received nodegroup_blend_file_path_override: {nodegroup_blend_file_path_override}") # DEBUG LOG (Indented) + print(f" DEBUG: Received asset_library_root_override: {asset_library_root_override}") + print(f" DEBUG: Received nodegroup_blend_file_path_override: {nodegroup_blend_file_path_override}") # --- Determine Asset Library Root --- @@ -229,7 +224,7 @@ def process_library_for_materials(context, asset_library_root_override=None, nod print("!!! ERROR: Processed asset library root not set in script and not provided via argument.") print("--- Script aborted. ---") return False - print(f" DEBUG: Using final PROCESSED_ASSET_LIBRARY_ROOT: {PROCESSED_ASSET_LIBRARY_ROOT}") # DEBUG LOG (Indented) + print(f" DEBUG: Using final PROCESSED_ASSET_LIBRARY_ROOT: {PROCESSED_ASSET_LIBRARY_ROOT}") # --- Determine Nodegroup Blend File Path --- if nodegroup_blend_file_path_override: @@ -239,7 +234,7 @@ def process_library_for_materials(context, asset_library_root_override=None, nod print("!!! ERROR: Nodegroup blend file path not set in script and not provided via argument.") print("--- Script aborted. ---") return False - print(f" DEBUG: Using final NODEGROUP_BLEND_FILE_PATH: {NODEGROUP_BLEND_FILE_PATH}") # DEBUG LOG (Indented) + print(f" DEBUG: Using final NODEGROUP_BLEND_FILE_PATH: {NODEGROUP_BLEND_FILE_PATH}") # --- Pre-run Checks --- @@ -281,8 +276,8 @@ def process_library_for_materials(context, asset_library_root_override=None, nod else: placeholder_node_found_in_template = True print(f" Found Template Material: '{TEMPLATE_MATERIAL_NAME}' with placeholder '{PLACEHOLDER_NODE_LABEL}'") - print(f" DEBUG: Template Material Found: {template_mat is not None}") # DEBUG LOG (Indented) - print(f" DEBUG: Placeholder Node Found in Template: {placeholder_node_found_in_template}") # DEBUG LOG (Indented) + print(f" DEBUG: Template Material Found: {template_mat is not None}") + print(f" DEBUG: Placeholder Node Found in Template: {placeholder_node_found_in_template}") if not valid_setup: @@ -296,7 +291,6 @@ def process_library_for_materials(context, asset_library_root_override=None, nod assets_processed = 0 assets_skipped = 0 materials_created = 0 - # materials_updated = 0 # Not updating existing materials anymore node_groups_linked = 0 previews_set = 0 viewport_colors_set = 0 @@ -322,7 +316,7 @@ def process_library_for_materials(context, asset_library_root_override=None, nod metadata_files_found = len(metadata_paths) print(f"Found {metadata_files_found} metadata.json files.") - print(f" DEBUG: Metadata paths found: {metadata_paths}") # DEBUG LOG (Indented) + print(f" DEBUG: Metadata paths found: {metadata_paths}") if metadata_files_found == 0: @@ -331,11 +325,11 @@ def process_library_for_materials(context, asset_library_root_override=None, nod return True # No work needed is considered success # --- Process Each Metadata File --- - print(f" DEBUG: Starting metadata file loop. Found {len(metadata_paths)} files.") # DEBUG LOG (Indented) + print(f" DEBUG: Starting metadata file loop. Found {len(metadata_paths)} files.") for metadata_path in metadata_paths: asset_dir_path = metadata_path.parent print(f"\n--- Processing Metadata: {metadata_path.relative_to(root_path)} ---") - print(f" DEBUG: Processing file: {metadata_path}") # DEBUG LOG (Indented) + print(f" DEBUG: Processing file: {metadata_path}") try: with open(metadata_path, 'r', encoding='utf-8') as f: metadata = json.load(f) @@ -355,7 +349,7 @@ def process_library_for_materials(context, asset_library_root_override=None, nod print(f" !!! ERROR: Metadata file is missing 'asset_name'. Skipping.") errors_encountered += 1 continue - print(f" DEBUG: Valid metadata loaded for asset: {asset_name}") # DEBUG LOG (Indented) + print(f" DEBUG: Valid metadata loaded for asset: {asset_name}") print(f" Asset Name: {asset_name}") @@ -363,8 +357,8 @@ def process_library_for_materials(context, asset_library_root_override=None, nod # --- Determine Target Names --- target_material_name = f"{MATERIAL_NAME_PREFIX}{asset_name}" target_pbrset_group_name = f"{PBRSET_GROUP_PREFIX}{asset_name}" - print(f" DEBUG: Target Material Name: {target_material_name}") # DEBUG LOG (Indented) - print(f" DEBUG: Target PBRSET Group Name: {target_pbrset_group_name}") # DEBUG LOG (Indented) + print(f" DEBUG: Target Material Name: {target_material_name}") + print(f" DEBUG: Target PBRSET Group Name: {target_pbrset_group_name}") # --- Check if Material Already Exists (Skip Logic) --- @@ -372,12 +366,12 @@ def process_library_for_materials(context, asset_library_root_override=None, nod print(f" Skipping asset '{asset_name}': Material '{target_material_name}' already exists.") assets_skipped += 1 continue # Move to the next metadata file - print(f" DEBUG: Material '{target_material_name}' does not exist. Proceeding with creation.") # DEBUG LOG (Indented) + print(f" DEBUG: Material '{target_material_name}' does not exist. Proceeding with creation.") # --- Create New Material --- print(f" Creating new material: '{target_material_name}'") - print(f" DEBUG: Copying template material '{TEMPLATE_MATERIAL_NAME}'") # DEBUG LOG (Indented) + print(f" DEBUG: Copying template material '{TEMPLATE_MATERIAL_NAME}'") material = template_mat.copy() if not material: print(f" !!! ERROR: Failed to copy template material '{TEMPLATE_MATERIAL_NAME}'. Skipping asset '{asset_name}'.") @@ -385,7 +379,7 @@ def process_library_for_materials(context, asset_library_root_override=None, nod continue material.name = target_material_name materials_created += 1 - print(f" DEBUG: Material '{material.name}' created.") # DEBUG LOG (Indented) + print(f" DEBUG: Material '{material.name}' created.") # --- Find Placeholder Node --- @@ -400,13 +394,13 @@ def process_library_for_materials(context, asset_library_root_override=None, nod placeholder_node = None # Ensure it's None else: placeholder_node = placeholder_nodes[0] # Assume first is correct - print(f" DEBUG: Found placeholder node '{placeholder_node.label}' in material '{material.name}'.") # DEBUG LOG (Indented) + print(f" DEBUG: Found placeholder node '{placeholder_node.label}' in material '{material.name}'.") # --- Find and Link PBRSET Node Group from Library --- linked_pbrset_group = None if placeholder_node and pbrset_blend_file_path: # Only proceed if placeholder exists and library file is known - print(f" DEBUG: Placeholder node exists and PBRSET library file path is known: {pbrset_blend_file_path}") # DEBUG LOG (Indented) + print(f" DEBUG: Placeholder node exists and PBRSET library file path is known: {pbrset_blend_file_path}") # Check if the group is already linked in the current file existing_linked_group = bpy.data.node_groups.get(target_pbrset_group_name) # Check if the existing group's library filepath matches the target blend file path @@ -440,7 +434,7 @@ def process_library_for_materials(context, asset_library_root_override=None, nod # --- Link Linked Node Group to Placeholder --- if placeholder_node and linked_pbrset_group: - print(f" DEBUG: Attempting to link PBRSET group '{linked_pbrset_group.name}' to placeholder '{placeholder_node.label}'.") # DEBUG LOG (Indented) + print(f" DEBUG: Attempting to link PBRSET group '{linked_pbrset_group.name}' to placeholder '{placeholder_node.label}'.") if placeholder_node.node_tree != linked_pbrset_group: try: placeholder_node.node_tree = linked_pbrset_group @@ -459,7 +453,7 @@ def process_library_for_materials(context, asset_library_root_override=None, nod # --- Mark Material as Asset --- if not material.asset_data: - print(f" DEBUG: Marking material '{material.name}' as asset.") # DEBUG LOG (Indented) + print(f" DEBUG: Marking material '{material.name}' as asset.") try: material.asset_mark() print(f" Marked material '{material.name}' as asset.") @@ -468,7 +462,7 @@ def process_library_for_materials(context, asset_library_root_override=None, nod # --- Copy Asset Tags --- if material.asset_data and linked_pbrset_group and linked_pbrset_group.asset_data: - print(f" DEBUG: Copying asset tags from PBRSET group to material.") # DEBUG LOG (Indented) + print(f" DEBUG: Copying asset tags from PBRSET group to material.") tags_copied_count = 0 if supplier_name: if add_tag_if_new(material.asset_data, supplier_name): tags_copied_count += 1 @@ -477,8 +471,6 @@ def process_library_for_materials(context, asset_library_root_override=None, nod # Copy other tags from PBRSET group for ng_tag in linked_pbrset_group.asset_data.tags: if add_tag_if_new(material.asset_data, ng_tag.name): tags_copied_count += 1 - # if tags_copied_count > 0: print(f" Copied {tags_copied_count} asset tags to material.") # Optional info - # else: print(f" Warn: Cannot copy tags. Material asset_data: {material.asset_data is not None}, Linked Group: {linked_pbrset_group}, Group asset_data: {linked_pbrset_group.asset_data if linked_pbrset_group else None}") # Debug # --- Set Custom Preview --- @@ -525,12 +517,12 @@ def process_library_for_materials(context, asset_library_root_override=None, nod # --- Set Viewport Properties from Stats --- if image_stats_1k and isinstance(image_stats_1k, dict): - print(f" DEBUG: Applying viewport properties from stats.") # DEBUG LOG (Indented) + print(f" DEBUG: Applying viewport properties from stats.") # Viewport Color color_mean = get_stat_value(image_stats_1k, VIEWPORT_COLOR_MAP_TYPES, 'mean') if isinstance(color_mean, list) and len(color_mean) >= 3: color_rgba = (*color_mean[:3], 1.0) - print(f" Debug: Raw color_mean from metadata: {color_mean[:3]}") # Added logging + print(f" Debug: Raw color_mean from metadata: {color_mean[:3]}") if tuple(material.diffuse_color[:3]) != tuple(color_rgba[:3]): material.diffuse_color = color_rgba print(f" Set viewport color: {color_rgba[:3]}") @@ -594,14 +586,13 @@ def process_library_for_materials(context, asset_library_root_override=None, nod print(f"Assets Processed/Attempted: {assets_processed}") print(f"Assets Skipped (Already Exist): {assets_skipped}") print(f"Materials Created: {materials_created}") - # print(f"Materials Updated: {materials_updated}") # Removed as we skip existing print(f"PBRSET Node Groups Linked: {node_groups_linked}") print(f"Material Previews Set: {previews_set}") print(f"Viewport Colors Set: {viewport_colors_set}") print(f"Viewport Roughness Set: {viewport_roughness_set}") print(f"Viewport Metallic Set: {viewport_metallic_set}") if pbrset_groups_missing_in_library > 0: - print(f"!!! PBRSET Node Groups Missing in Library File: {pbrset_groups_missing_in_library} !!!") # Updated message + print(f"!!! PBRSET Node Groups Missing in Library File: {pbrset_groups_missing_in_library} !!!") if library_link_errors > 0: print(f"!!! Library Link Errors: {library_link_errors} !!!") if placeholder_nodes_missing > 0: @@ -611,7 +602,7 @@ def process_library_for_materials(context, asset_library_root_override=None, nod print("---------------------------------------") # --- Explicit Save --- - print(f" DEBUG: Attempting explicit save for file: {bpy.data.filepath}") # DEBUG LOG (Indented) + print(f" DEBUG: Attempting explicit save for file: {bpy.data.filepath}") try: bpy.ops.wm.save_as_mainfile(filepath=bpy.data.filepath) print("\n--- Explicitly saved the .blend file. ---") @@ -649,7 +640,6 @@ if __name__ == "__main__": print(f"Found nodegroup blend file path argument: {nodegroup_blend_file_arg}") else: print("Info: '--' found but not enough arguments after it for nodegroup blend file.") - # else: print("Info: No '--' found in arguments.") # Optional debug except Exception as e: print(f"Error parsing command line arguments: {e}") # --- End Argument Parsing --- diff --git a/blenderscripts/create_nodegroups.py b/blenderscripts/create_nodegroups.py index 8000574..71fee32 100644 --- a/blenderscripts/create_nodegroups.py +++ b/blenderscripts/create_nodegroups.py @@ -28,7 +28,7 @@ from pathlib import Path import time import re # For parsing aspect ratio string import base64 # For encoding node group names -import sys # <<< ADDED IMPORT +import sys # --- USER CONFIGURATION --- @@ -36,7 +36,7 @@ import sys # <<< ADDED IMPORT # Example: r"G:\Assets\Processed" # IMPORTANT: This should point to the base directory containing supplier folders (e.g., Poliigon) # This will be overridden by command-line arguments if provided. -PROCESSED_ASSET_LIBRARY_ROOT = None # Set to None initially +PROCESSED_ASSET_LIBRARY_ROOT = None # Names of the required node group templates in the Blender file PARENT_TEMPLATE_NAME = "Template_PBRSET" @@ -52,9 +52,9 @@ HIGHEST_RESOLUTION_NODE_LABEL = "HighestResolution" # Value node to store highes ENABLE_MANIFEST = False # Disabled based on user feedback in previous run # Assumed filename pattern for processed images. -# {asset_name}, {map_type}, {resolution}, {format} will be replaced. -# Check Asset Processor Tool's config.py (TARGET_FILENAME_PATTERN) if this is wrong. -IMAGE_FILENAME_PATTERN = "{asset_name}_{map_type}_{resolution}.{format}" +# [assetname], [maptype], [resolution], [ext] will be replaced. +# This should match OUTPUT_FILENAME_PATTERN from app_settings.json. +IMAGE_FILENAME_PATTERN = "[assetname]_[maptype]_[resolution].[ext]" # Fallback extensions to try if the primary format from metadata is not found # Order matters - first found will be used. @@ -109,7 +109,6 @@ CATEGORIES_FOR_NODEGROUP_GENERATION = ["Surface", "Decal"] def encode_name_b64(name_str): """Encodes a string using URL-safe Base64 for node group names.""" try: - # Ensure the input is a string name_str = str(name_str) return base64.urlsafe_b64encode(name_str.encode('utf-8')).decode('ascii') except Exception as e: @@ -150,10 +149,31 @@ def add_tag_if_new(asset_data, tag_name): def get_color_space(map_type): """Returns the appropriate Blender color space name for a given map type string.""" - # Handle potential numbered variants like COL-1, COL-2 - base_map_type = map_type.split('-')[0] - return PBR_COLOR_SPACE_MAP.get(map_type.upper(), # Check full name first (e.g., NRMRGH) - PBR_COLOR_SPACE_MAP.get(base_map_type.upper(), DEFAULT_COLOR_SPACE)) # Fallback to base type + # Attempt to map map_type (e.g., "MAP_COL", "COL-1", "NRMRGH") to a standard type for color space lookup. + # PBR_COLOR_SPACE_MAP usually contains standard types like "COL", "NRM". + map_type_upper = map_type.upper() + + # 1. Direct match (e.g., "NRMRGH", "COL") + if map_type_upper in PBR_COLOR_SPACE_MAP: + return PBR_COLOR_SPACE_MAP[map_type_upper] + + # 2. Handle variants like "COL-1", "MAP_ROUGH-2" + # Try to get the part before a hyphen if a hyphen exists + base_type_candidate = map_type_upper.split('-')[0] + if base_type_candidate in PBR_COLOR_SPACE_MAP: + return PBR_COLOR_SPACE_MAP[base_type_candidate] + + # 3. Handle cases like "MAP_COL" -> "COL" + # This is a simple heuristic. A more robust solution would involve access to FILE_TYPE_DEFINITIONS. + # For this script, we assume PBR_COLOR_SPACE_MAP might contain the direct standard_type. + # Example: if map_type is "MAP_DIFFUSE" and PBR_COLOR_SPACE_MAP has "DIFFUSE" + if base_type_candidate.startswith("MAP_") and len(base_type_candidate) > 4: + short_type = base_type_candidate[4:] # Get "COL" from "MAP_COL" + if short_type in PBR_COLOR_SPACE_MAP: + return PBR_COLOR_SPACE_MAP[short_type] + + # Fallback if no specific rule found + return DEFAULT_COLOR_SPACE def calculate_aspect_correction_factor(image_width, image_height, aspect_string): """ @@ -166,13 +186,11 @@ def calculate_aspect_correction_factor(image_width, image_height, aspect_string) print(" Warn: Invalid image dimensions for aspect ratio calculation. Returning 1.0.") return 1.0 - # Calculate the actual aspect ratio of the image file current_aspect_ratio = image_width / image_height if not aspect_string or aspect_string.upper() == "EVEN": # If scaling was even, the correction factor is just the image's aspect ratio # to make UVs match the image proportions. - # print(f" Aspect string is EVEN. Correction factor = current aspect ratio: {current_aspect_ratio:.4f}") return current_aspect_ratio # Handle non-uniform scaling cases ("Xnnn", "Ynnn") @@ -194,7 +212,7 @@ def calculate_aspect_correction_factor(image_width, image_height, aspect_string) # Apply the non-uniform correction formula based on original script logic scaling_factor_percent = amount / 100.0 - correction_factor = current_aspect_ratio # Default + correction_factor = current_aspect_ratio try: if axis == 'X': @@ -213,7 +231,6 @@ def calculate_aspect_correction_factor(image_width, image_height, aspect_string) print(f" Error calculating aspect correction factor: {e}. Returning current ratio {current_aspect_ratio:.4f}.") return current_aspect_ratio - # print(f" Calculated aspect correction factor: {correction_factor:.4f} (from {image_width}x{image_height}, Scaling='{aspect_string}')") return correction_factor @@ -234,16 +251,14 @@ def reconstruct_image_path_with_fallback(asset_dir_path, asset_name, map_type, r if primary_format: try: filename = IMAGE_FILENAME_PATTERN.format( - asset_name=asset_name, - map_type=map_type, + assetname=asset_name, + maptype=map_type, resolution=resolution, - format=primary_format.lower() # Ensure format is lowercase + ext=primary_format.lower() ) primary_path = asset_dir_path / filename if primary_path.is_file(): - # print(f" Found primary path: {str(primary_path)}") # Verbose return str(primary_path) - # else: print(f" Primary path not found: {str(primary_path)}") # Verbose except KeyError as e: print(f" !!! ERROR: Missing key '{e}' in IMAGE_FILENAME_PATTERN. Cannot reconstruct path.") return None # Cannot proceed without valid pattern @@ -252,17 +267,16 @@ def reconstruct_image_path_with_fallback(asset_dir_path, asset_name, map_type, r # Continue to fallback # 2. Try fallback extensions - # print(f" Trying fallback extensions for {map_type}/{resolution}...") # Verbose for ext in FALLBACK_IMAGE_EXTENSIONS: # Skip if we already tried this extension as primary (and it failed) if primary_format and ext.lower() == primary_format.lower(): continue try: fallback_filename = IMAGE_FILENAME_PATTERN.format( - asset_name=asset_name, - map_type=map_type, + assetname=asset_name, + maptype=map_type, resolution=resolution, - format=ext.lower() + ext=ext.lower() ) fallback_path = asset_dir_path / fallback_filename if fallback_path.is_file(): @@ -378,13 +392,13 @@ def update_manifest(manifest_data, asset_name, map_type=None, resolution=None): # --- Core Logic --- -def process_library(context, asset_library_root_override=None): # Add override parameter +def process_library(context, asset_library_root_override=None): global ENABLE_MANIFEST # Declare intent to modify global if needed global PROCESSED_ASSET_LIBRARY_ROOT # Allow modification of global """Scans the library, reads metadata, creates/updates node groups.""" start_time = time.time() print(f"\n--- Starting Node Group Processing ({time.strftime('%Y-%m-%d %H:%M:%S')}) ---") - print(f" DEBUG: Received asset_library_root_override: {asset_library_root_override}") # DEBUG LOG (Indented) + print(f" DEBUG: Received asset_library_root_override: {asset_library_root_override}") # --- Determine Asset Library Root --- if asset_library_root_override: @@ -394,7 +408,7 @@ def process_library(context, asset_library_root_override=None): # Add override p print("!!! ERROR: Processed asset library root not set in script and not provided via argument.") print("--- Script aborted. ---") return False - print(f" DEBUG: Using final PROCESSED_ASSET_LIBRARY_ROOT: {PROCESSED_ASSET_LIBRARY_ROOT}") # DEBUG LOG (Indented) + print(f" DEBUG: Using final PROCESSED_ASSET_LIBRARY_ROOT: {PROCESSED_ASSET_LIBRARY_ROOT}") # --- Pre-run Checks --- print("Performing pre-run checks...") @@ -407,7 +421,7 @@ def process_library(context, asset_library_root_override=None): # Add override p valid_setup = False else: print(f" Asset Library Root: '{root_path}'") - print(f" DEBUG: Checking for templates: '{PARENT_TEMPLATE_NAME}', '{CHILD_TEMPLATE_NAME}'") # DEBUG LOG (Indented) + print(f" DEBUG: Checking for templates: '{PARENT_TEMPLATE_NAME}', '{CHILD_TEMPLATE_NAME}'") # 2. Check Templates template_parent = bpy.data.node_groups.get(PARENT_TEMPLATE_NAME) @@ -420,8 +434,8 @@ def process_library(context, asset_library_root_override=None): # Add override p valid_setup = False if template_parent and template_child: print(f" Found Templates: '{PARENT_TEMPLATE_NAME}', '{CHILD_TEMPLATE_NAME}'") - print(f" DEBUG: Template Parent Found: {template_parent is not None}") # DEBUG LOG (Indented) - print(f" DEBUG: Template Child Found: {template_child is not None}") # DEBUG LOG (Indented) + print(f" DEBUG: Template Parent Found: {template_parent is not None}") + print(f" DEBUG: Template Child Found: {template_child is not None}") # 3. Check Blend File Saved (if manifest enabled) if ENABLE_MANIFEST and not context.blend_data.filepath: @@ -473,7 +487,7 @@ def process_library(context, asset_library_root_override=None): # Add override p metadata_files_found = len(metadata_paths) print(f"Found {metadata_files_found} metadata.json files.") - print(f" DEBUG: Metadata paths found: {metadata_paths}") # DEBUG LOG (Indented) + print(f" DEBUG: Metadata paths found: {metadata_paths}") if metadata_files_found == 0: print("No metadata files found. Nothing to process.") @@ -482,9 +496,9 @@ def process_library(context, asset_library_root_override=None): # Add override p # --- Process Each Metadata File --- for metadata_path in metadata_paths: - asset_dir_path = metadata_path.parent # Get the directory containing the metadata file + asset_dir_path = metadata_path.parent print(f"\n--- Processing Metadata: {metadata_path.relative_to(root_path)} ---") - print(f" DEBUG: Processing file: {metadata_path}") # DEBUG LOG (Indented) + print(f" DEBUG: Processing file: {metadata_path}") try: with open(metadata_path, 'r', encoding='utf-8') as f: metadata = json.load(f) @@ -493,8 +507,7 @@ def process_library(context, asset_library_root_override=None): # Add override p asset_name = metadata.get("asset_name") supplier_name = metadata.get("supplier_name") archetype = metadata.get("archetype") - asset_category = metadata.get("asset_category", "Unknown") # Read asset_category instead of category - # Get map info from the correct keys + asset_category = metadata.get("category", "Unknown") processed_resolutions = metadata.get("processed_map_resolutions", {}) # Default to empty dict merged_resolutions = metadata.get("merged_map_resolutions", {}) # Get merged maps too map_details = metadata.get("map_details", {}) # Default to empty dict @@ -514,7 +527,7 @@ def process_library(context, asset_library_root_override=None): # Add override p errors_encountered += 1 continue # map_details check remains a warning as merged maps won't be in it - print(f" DEBUG: Valid metadata loaded for asset: {asset_name}") # DEBUG LOG (Indented) + print(f" DEBUG: Valid metadata loaded for asset: {asset_name}") print(f" Asset Name: {asset_name}") @@ -603,8 +616,8 @@ def process_library(context, asset_library_root_override=None): # Add override p # Conditional skip based on asset_category - if asset_category not in CATEGORIES_FOR_NODEGROUP_GENERATION: # Check asset_category - print(f" Skipping nodegroup content generation for asset '{asset_name}' (Category: '{asset_category}'). Tag added.") # Use asset_category in log + if asset_category not in CATEGORIES_FOR_NODEGROUP_GENERATION: + print(f" Skipping nodegroup content generation for asset '{asset_name}' (Category: '{asset_category}'). Tag added.") assets_processed += 1 # Still count as processed for summary, even if skipped continue # Skip the rest of the processing for this asset @@ -615,7 +628,7 @@ def process_library(context, asset_library_root_override=None): # Add override p if parent_group is None: print(f" Creating new parent group: '{target_parent_name}'") - print(f" DEBUG: Copying parent template '{PARENT_TEMPLATE_NAME}'") # DEBUG LOG (Indented) + print(f" DEBUG: Copying parent template '{PARENT_TEMPLATE_NAME}'") parent_group = template_parent.copy() if not parent_group: print(f" !!! ERROR: Failed to copy parent template '{PARENT_TEMPLATE_NAME}'. Skipping asset '{asset_name}'.") @@ -626,7 +639,7 @@ def process_library(context, asset_library_root_override=None): # Add override p is_new_parent = True else: print(f" Updating existing parent group: '{target_parent_name}'") - print(f" DEBUG: Found existing parent group.") # DEBUG LOG (Indented) + print(f" DEBUG: Found existing parent group.") parent_groups_updated += 1 # Ensure marked as asset @@ -644,10 +657,9 @@ def process_library(context, asset_library_root_override=None): # Add override p add_tag_if_new(parent_group.asset_data, supplier_name) if archetype: add_tag_if_new(parent_group.asset_data, archetype) - if asset_category: # Use asset_category for tagging + if asset_category: add_tag_if_new(parent_group.asset_data, asset_category) # Add other tags if needed - # else: print(f" Warn: Cannot apply tags, asset_data not available for '{parent_group.name}'.") # Optional warning # Apply Aspect Ratio Correction @@ -667,7 +679,6 @@ def process_library(context, asset_library_root_override=None): # Add override p aspect_node.outputs[0].default_value = correction_factor print(f" Set '{ASPECT_RATIO_NODE_LABEL}' value to {correction_factor:.4f} (was {current_val:.4f})") aspect_ratio_set += 1 - # else: print(f" Warn: Aspect ratio node '{ASPECT_RATIO_NODE_LABEL}' not found in parent group.") # Optional # Apply Highest Resolution Value hr_nodes = find_nodes_by_label(parent_group, HIGHEST_RESOLUTION_NODE_LABEL, 'ShaderNodeValue') @@ -678,7 +689,6 @@ def process_library(context, asset_library_root_override=None): # Add override p hr_node.outputs[0].default_value = highest_resolution_value print(f" Set '{HIGHEST_RESOLUTION_NODE_LABEL}' value to {highest_resolution_value} ({highest_resolution_str}) (was {current_hr_val:.1f})") highest_res_set += 1 # Count successful sets - # else: print(f" Warn: Highest resolution node '{HIGHEST_RESOLUTION_NODE_LABEL}' not found in parent group.") # Optional # Apply Stats (using image_stats_1k) @@ -690,7 +700,7 @@ def process_library(context, asset_library_root_override=None): # Add override p stats_nodes = find_nodes_by_label(parent_group, stats_node_label, 'ShaderNodeCombineXYZ') if stats_nodes: stats_node = stats_nodes[0] - stats = image_stats_1k[map_type_to_stat] # Get stats dict for this map type + stats = image_stats_1k[map_type_to_stat] if stats and isinstance(stats, dict): # Handle potential list format for RGB stats (use first value) or direct float @@ -721,10 +731,6 @@ def process_library(context, asset_library_root_override=None): # Add override p if updated_stat: print(f" Set stats in '{stats_node_label}': Min={min_val:.4f}, Max={max_val:.4f}, Mean={mean_val:.4f}") - # else: print(f" Info: No valid 'stats' dictionary found for map type '{map_type_to_stat}' in image_stats_1k.") # Optional - # else: print(f" Warn: Stats node '{stats_node_label}' not found in parent group.") # Optional - # else: print(f" Info: Map type '{map_type_to_stat}' not present in image_stats_1k for stats application.") # Optional - # else: print(f" Warn: 'image_stats_1k' missing or invalid in metadata.") # Optional # --- Set Asset Preview (only for new parent groups) --- # Use the reference image path found earlier if available @@ -746,14 +752,13 @@ def process_library(context, asset_library_root_override=None): # Add override p # --- Child Group Handling --- # Iterate through the COMBINED map types - print(f" DEBUG: Starting child group loop for asset '{asset_name}'. Map types: {list(all_map_resolutions.keys())}") # DEBUG LOG (Indented) + print(f" DEBUG: Starting child group loop for asset '{asset_name}'. Map types: {list(all_map_resolutions.keys())}") for map_type, resolutions in all_map_resolutions.items(): print(f" Processing Map Type: {map_type}") # Determine if this is a merged map (not in map_details) is_merged_map = map_type not in map_details - # Get details for this map type if available current_map_details = map_details.get(map_type, {}) # For merged maps, primary_format will be None output_format = current_map_details.get("output_format") @@ -770,7 +775,7 @@ def process_library(context, asset_library_root_override=None): # Add override p print(f" !!! WARNING: No placeholder node labeled '{map_type}' found in parent group '{parent_group.name}'. Skipping this map type.") continue holder_node = holder_nodes[0] # Assume first is correct - print(f" DEBUG: Found placeholder node '{holder_node.label}' for map type '{map_type}'.") # DEBUG LOG (Indented) + print(f" DEBUG: Found placeholder node '{holder_node.label}' for map type '{map_type}'.") # Determine child group name (LOGICAL and ENCODED) logical_child_name = f"{asset_name}_{map_type}" @@ -780,8 +785,7 @@ def process_library(context, asset_library_root_override=None): # Add override p is_new_child = False if child_group is None: - print(f" DEBUG: Child group '{target_child_name_b64}' not found. Creating new one.") # DEBUG LOG (Indented) - # print(f" Creating new child group: '{target_child_name_b64}' (logical: '{logical_child_name}')") # Verbose + print(f" DEBUG: Child group '{target_child_name_b64}' not found. Creating new one.") child_group = template_child.copy() if not child_group: print(f" !!! ERROR: Failed to copy child template '{CHILD_TEMPLATE_NAME}'. Skipping map type '{map_type}'.") @@ -791,8 +795,7 @@ def process_library(context, asset_library_root_override=None): # Add override p child_groups_created += 1 is_new_child = True else: - print(f" DEBUG: Found existing child group '{target_child_name_b64}'.") # DEBUG LOG (Indented) - # print(f" Updating existing child group: '{target_child_name_b64}'") # Verbose + print(f" DEBUG: Found existing child group '{target_child_name_b64}'.") child_groups_updated += 1 # Assign child group to placeholder if needed @@ -820,10 +823,6 @@ def process_library(context, asset_library_root_override=None): # Add override p if not link_exists: parent_group.links.new(source_socket, target_socket) print(f" Linked '{holder_node.label}' output to parent output socket '{map_type}'.") - # else: # Optional warnings - # if not source_socket: print(f" Warn: Could not find suitable output socket on placeholder '{holder_node.label}'.") - # if not target_socket: print(f" Warn: Could not find input socket '{map_type}' on parent output node.") - # else: print(f" Warn: Parent group '{parent_group.name}' has no Group Output node.") except Exception as e_link: print(f" !!! ERROR linking sockets for '{map_type}': {e_link}") @@ -837,7 +836,6 @@ def process_library(context, asset_library_root_override=None): # Add override p # Defaulting to Color seems reasonable for most PBR outputs if item.socket_type != 'NodeSocketColor': item.socket_type = 'NodeSocketColor' - # print(f" Set parent output socket '{map_type}' type to Color.") # Optional info except Exception as e_sock_type: print(f" Warn: Could not verify/set socket type for '{map_type}': {e_sock_type}") @@ -850,10 +848,9 @@ def process_library(context, asset_library_root_override=None): # Add override p for resolution in resolutions: # --- Manifest Check (Map/Resolution Level) --- if ENABLE_MANIFEST and is_map_processed(manifest_data, asset_name, map_type, resolution): - # print(f" Skipping {resolution} (Manifest)") # Verbose maps_skipped_manifest += 1 continue - print(f" DEBUG: Processing map '{map_type}' resolution '{resolution}'. Manifest skip check passed.") # DEBUG LOG (Indented) + print(f" DEBUG: Processing map '{map_type}' resolution '{resolution}'. Manifest skip check passed.") print(f" Processing Resolution: {resolution}") @@ -866,7 +863,7 @@ def process_library(context, asset_library_root_override=None): # Add override p resolution=resolution, primary_format=output_format ) - print(f" DEBUG: Reconstructed image path for {map_type}/{resolution}: {image_path_str}") # DEBUG LOG (Indented) + print(f" DEBUG: Reconstructed image path for {map_type}/{resolution}: {image_path_str}") if not image_path_str: # Error already printed by reconstruct function @@ -878,7 +875,7 @@ def process_library(context, asset_library_root_override=None): # Add override p if not image_nodes: print(f" !!! WARNING: No Image Texture node labeled '{resolution}' found in child group '{child_group.name}'. Cannot assign image.") continue # Skip this resolution if node not found - print(f" DEBUG: Found {len(image_nodes)} image node(s) labeled '{resolution}' in child group '{child_group.name}'.") # DEBUG LOG (Indented) + print(f" DEBUG: Found {len(image_nodes)} image node(s) labeled '{resolution}' in child group '{child_group.name}'.") # --- Load Image --- img = None @@ -932,7 +929,6 @@ def process_library(context, asset_library_root_override=None): # Add override p # --- Update Manifest (Map/Resolution Level) --- if update_manifest(manifest_data, asset_name, map_type, resolution): manifest_needs_saving = True - # print(f" Marked {map_type}/{resolution} processed in manifest.") # Verbose maps_processed += 1 else: @@ -986,13 +982,13 @@ def process_library(context, asset_library_root_override=None): # Add override p print(f"Individual Maps Processed: {maps_processed}") print(f"Asset Previews Set: {previews_set}") print(f"Highest Resolution Nodes Set: {highest_res_set}") - print(f"Aspect Ratio Nodes Set: {aspect_ratio_set}") # Added counter + print(f"Aspect Ratio Nodes Set: {aspect_ratio_set}") if errors_encountered > 0: print(f"!!! Errors Encountered: {errors_encountered} !!!") print("---------------------------") # --- Explicit Save --- - print(f" DEBUG: Attempting explicit save for file: {bpy.data.filepath}") # DEBUG LOG (Indented) + print(f" DEBUG: Attempting explicit save for file: {bpy.data.filepath}") try: bpy.ops.wm.save_as_mainfile(filepath=bpy.data.filepath) print("\n--- Explicitly saved the .blend file. ---") @@ -1025,7 +1021,6 @@ if __name__ == "__main__": print(f"Found asset library root argument: {asset_root_arg}") else: print("Info: '--' found but no arguments after it.") - # else: print("Info: No '--' found in arguments.") # Optional debug except Exception as e: print(f"Error parsing command line arguments: {e}") # --- End Argument Parsing --- diff --git a/config.py b/config.py deleted file mode 100644 index 2a4b20c..0000000 --- a/config.py +++ /dev/null @@ -1,87 +0,0 @@ -# config.py -# Core settings defining the pipeline standards and output format. - -# --- Target Output Standards --- -TARGET_FILENAME_PATTERN = "{base_name}_{map_type}_{resolution}.{ext}" -STANDARD_MAP_TYPES = [ - "COL", "NRM", "ROUGH", "METAL", "AO", "DISP", "REFL", - "SSS", "FUZZ", "IDMAP", "MASK" -] -# Map types that should always receive a numeric suffix (e.g., COL-1, COL-2) -# based on preset keyword order, even if only one variant is found. -RESPECT_VARIANT_MAP_TYPES = ["COL"] - -# Subdirectory within the final set folder for non-essential/unknown files -EXTRA_FILES_SUBDIR = "Extra" -OUTPUT_BASE_DIR = "../Asset_Processor_Output" #accepts both relative and absolute paths -METADATA_FILENAME = "metadata.json" - -# --- Blender Integration Settings --- -# Default paths to Blender files for node group and material creation. -# Set these to absolute or relative paths if you want defaults. -# Command-line arguments (--nodegroup-blend, --materials-blend) will override these. -DEFAULT_NODEGROUP_BLEND_PATH = r"G:/02 Content/10-19 Content/19 Catalogs/19.01 Blender Asset Catalogue/_CustomLibraries/Nodes-Linked/PBRSET-Nodes-Testing.blend" # e.g., r"G:\Blender\Libraries\NodeGroups.blend" -DEFAULT_MATERIALS_BLEND_PATH = r"G:/02 Content/10-19 Content/19 Catalogs/19.01 Blender Asset Catalogue/_CustomLibraries/Materials-Append/PBR Materials-Testing.blend" # e.g., r"G:\Blender\Libraries\Materials.blend" -# Path to the Blender executable. Required for running Blender scripts. -# Example: r"C:\Program Files\Blender Foundation\Blender 3.6\blender.exe" -BLENDER_EXECUTABLE_PATH = r"C:/Program Files/Blender Foundation/Blender 4.4/blender.exe" # <<< SET THIS PATH! - -# --- Image Processing Settings --- -# Target resolutions (Largest dimension in pixels) -PNG_COMPRESSION_LEVEL = 6 # 0 (none) to 9 (max) -# Quality for JPG output (0-100) -JPG_QUALITY = 98 -# Resolution dimension threshold (pixels) above which 8-bit images are forced to JPG, overriding input format logic. -RESOLUTION_THRESHOLD_FOR_JPG = 4096 -IMAGE_RESOLUTIONS = {"8K": 8192,"4K": 4096, "2K": 2048, "1K": 1024} -# Aspect ratio decimals (used for metadata, could potentially be removed later) -ASPECT_RATIO_DECIMALS = 2 -# Bit depth rules per standard map type ('respect' or 'force_8bit') -MAP_BIT_DEPTH_RULES = { - "COL": "force_8bit", "NRM": "respect", "ROUGH": "force_8bit", "METAL": "force_8bit", - "AO": "force_8bit", "DISP": "respect", "REFL": "force_bit", "SSS": "respect", - "FUZZ": "force_bit", "IDMAP": "force_8bit", "MASK": "force_8bit", - "DEFAULT": "respect" # Fallback for map types not listed -} -# Output format preferences for 16-bit data -OUTPUT_FORMAT_16BIT_PRIMARY = "png" # Options: exr_dwaa, exr_dwab, exr_zip, png, tif -OUTPUT_FORMAT_16BIT_FALLBACK = "png" -# Output format for 8-bit data -OUTPUT_FORMAT_8BIT = "png" # Could allow 'jpg' later with quality settings - -# Map types that should always be saved in a lossless format (e.g., PNG, EXR) -# regardless of resolution threshold or input format. -#FORCE_LOSSLESS_MAP_TYPES = ["NRM", "NRMRGN"] - -# --- Map Merging Rules --- -# List of dictionaries, each defining a merge operation. -MAP_MERGE_RULES = [ - { - "output_map_type": "NRMRGH", # Suffix or standard name for the merged map - "inputs": { # Map target RGB channels to standard input map type names - "R": "NRM", # Use Red channel from NRM - "G": "NRM", # Use Green channel from NRM - "B": "ROUGH" # Use Red channel from ROUGH (assuming it's grayscale) - }, - "defaults": { # Default values (0.0 - 1.0) if an input map is missing - "R": 0.5, "G": 0.5, "B": 0.5 - }, - # 'respect_inputs' (use 16bit if any input is), 'force_8bit', 'force_16bit' - "output_bit_depth": "respect_inputs" - }, - # Example: Merge Metalness(R), Roughness(G), AO(B) -> "MRA" map often used in engines - # { - # "output_map_type": "MRA", - # "inputs": {"R": "METAL", "G": "ROUGH", "B": "AO"}, - # "defaullllts": {"R": 0.0, "G": 1.0, "B": 1.0}, # Default Metal=0, Rough=1, AO=1 - # "output_bit_depth": "force_8bit" # Usually fine as 8bit - # }, -] - -# --- Metadata Settings --- -CALCULATE_STATS_RESOLUTION = "1K" # Resolution suffix used for calculating stats -DEFAULT_ASSET_CATEGORY = "Surface" # If rules don't identify Asset or Decal - -# --- Internal Settings --- -# Temporary directory prefix for processing folders -TEMP_DIR_PREFIX = "_PROCESS_ASSET_" \ No newline at end of file diff --git a/config/app_settings.json b/config/app_settings.json new file mode 100644 index 0000000..1343766 --- /dev/null +++ b/config/app_settings.json @@ -0,0 +1,52 @@ +{ + "TARGET_FILENAME_PATTERN": "{base_name}_{map_type}_{resolution}.{ext}", + "RESPECT_VARIANT_MAP_TYPES": [ + "COL" + ], + "EXTRA_FILES_SUBDIR": "Extra", + "OUTPUT_BASE_DIR": "../Asset_Processor_Output_Tests", + "OUTPUT_DIRECTORY_PATTERN": "[supplier]_[assetname]", + "OUTPUT_FILENAME_PATTERN": "[assetname]_[maptype]_[resolution].[ext]", + "METADATA_FILENAME": "metadata.json", + "DEFAULT_NODEGROUP_BLEND_PATH": "G:/02 Content/10-19 Content/19 Catalogs/19.01 Blender Asset Catalogue/_CustomLibraries/Nodes-Linked/PBRSET-Nodes-Testing.blend", + "DEFAULT_MATERIALS_BLEND_PATH": "G:/02 Content/10-19 Content/19 Catalogs/19.01 Blender Asset Catalogue/_CustomLibraries/Materials-Append/PBR Materials-Testing.blend", + "BLENDER_EXECUTABLE_PATH": "C:/Program Files/Blender Foundation/Blender 4.4/blender.exe", + "PNG_COMPRESSION_LEVEL": 6, + "JPG_QUALITY": 98, + "RESOLUTION_THRESHOLD_FOR_JPG": 4096, + "IMAGE_RESOLUTIONS": { + "8K": 8192, + "4K": 4096, + "2K": 2048, + "1K": 1024, + "PREVIEW": 128 + }, + "ASPECT_RATIO_DECIMALS": 2, + "OUTPUT_FORMAT_16BIT_PRIMARY": "exr", + "OUTPUT_FORMAT_16BIT_FALLBACK": "png", + "OUTPUT_FORMAT_8BIT": "png", + "MAP_MERGE_RULES": [ + { + "output_map_type": "MAP_NRMRGH", + "inputs": { + "R": "MAP_NRM", + "G": "MAP_NRM", + "B": "MAP_ROUGH" + }, + "defaults": { + "R": 0.5, + "G": 0.5, + "B": 0.5 + }, + "output_bit_depth": "respect_inputs" + } + ], + "CALCULATE_STATS_RESOLUTION": "1K", + "DEFAULT_ASSET_CATEGORY": "Surface", + "TEMP_DIR_PREFIX": "_PROCESS_ASSET_", + "INITIAL_SCALING_MODE": "POT_DOWNSCALE", + "MERGE_DIMENSION_MISMATCH_STRATEGY": "USE_LARGEST", + "general_settings": { + "invert_normal_map_green_channel_globally": false + } +} \ No newline at end of file diff --git a/config/asset_type_definitions.json b/config/asset_type_definitions.json new file mode 100644 index 0000000..55691ad --- /dev/null +++ b/config/asset_type_definitions.json @@ -0,0 +1,44 @@ +{ + "ASSET_TYPE_DEFINITIONS": { + "Surface": { + "color": "#1f3e5d", + "description": "A single Standard PBR material set for a surface.", + "examples": [ + "Set: Wood01_COL + Wood01_NRM + WOOD01_ROUGH", + "Set: Dif_Concrete + Normal_Concrete + Refl_Concrete" + ] + }, + "Model": { + "color": "#b67300", + "description": "A set that contains models, can include PBR textureset", + "examples": [ + "Single = Chair.fbx", + "Set = Plant02.fbx + Plant02_col + Plant02_SSS" + ] + }, + "Decal": { + "color": "#68ac68", + "description": "A alphamasked textureset", + "examples": [ + "Set = DecalGraffiti01_Col + DecalGraffiti01_Alpha", + "Single = DecalLeakStain03" + ] + }, + "Atlas": { + "color": "#955b8b", + "description": "A texture, name usually hints that it's an atlas", + "examples": [ + "Set = FoliageAtlas01_col + FoliageAtlas01_nrm" + ] + }, + "UtilityMap": { + "color": "#706b87", + "description": "A useful image-asset consisting of only a single texture. Therefor each Utilitymap can only contain a single item.", + "examples": [ + "Single = imperfection.png", + "Single = smudges.png", + "Single = scratches.tif" + ] + } + } +} \ No newline at end of file diff --git a/config/file_type_definitions.json b/config/file_type_definitions.json new file mode 100644 index 0000000..018a0d0 --- /dev/null +++ b/config/file_type_definitions.json @@ -0,0 +1,208 @@ +{ + "FILE_TYPE_DEFINITIONS": { + "MAP_COL": { + "bit_depth_rule": "force_8bit", + "color": "#ffaa00", + "description": "Color/Albedo Map", + "examples": [ + "_col.", + "_basecolor.", + "albedo", + "diffuse" + ], + "is_grayscale": false, + "keybind": "C", + "standard_type": "COL" + }, + "MAP_NRM": { + "bit_depth_rule": "respect", + "color": "#cca2f1", + "description": "Normal Map", + "examples": [ + "_nrm.", + "_normal." + ], + "is_grayscale": false, + "keybind": "N", + "standard_type": "NRM" + }, + "MAP_METAL": { + "bit_depth_rule": "force_8bit", + "color": "#dcf4f2", + "description": "Metalness Map", + "examples": [ + "_metal.", + "_met." + ], + "is_grayscale": true, + "keybind": "M", + "standard_type": "METAL" + }, + "MAP_ROUGH": { + "bit_depth_rule": "force_8bit", + "color": "#bfd6bf", + "description": "Roughness Map", + "examples": [ + "_rough.", + "_rgh.", + "_gloss" + ], + "is_grayscale": true, + "keybind": "R", + "standard_type": "ROUGH" + }, + "MAP_GLOSS": { + "bit_depth_rule": "force_8bit", + "color": "#d6bfd6", + "description": "Glossiness Map", + "examples": [ + "_gloss.", + "_gls." + ], + "is_grayscale": true, + "keybind": "R", + "standard_type": "GLOSS" + }, + "MAP_AO": { + "bit_depth_rule": "force_8bit", + "color": "#e3c7c7", + "description": "Ambient Occlusion Map", + "examples": [ + "_ao.", + "_ambientocclusion." + ], + "is_grayscale": true, + "keybind": "", + "standard_type": "AO" + }, + "MAP_DISP": { + "bit_depth_rule": "respect", + "color": "#c6ddd5", + "description": "Displacement/Height Map", + "examples": [ + "_disp.", + "_height." + ], + "is_grayscale": true, + "keybind": "D", + "standard_type": "DISP" + }, + "MAP_REFL": { + "bit_depth_rule": "force_8bit", + "color": "#c2c2b9", + "description": "Reflection/Specular Map", + "examples": [ + "_refl.", + "_specular." + ], + "is_grayscale": true, + "keybind": "M", + "standard_type": "REFL" + }, + "MAP_SSS": { + "bit_depth_rule": "respect", + "color": "#a0d394", + "description": "Subsurface Scattering Map", + "examples": [ + "_sss.", + "_subsurface." + ], + "is_grayscale": true, + "keybind": "", + "standard_type": "SSS" + }, + "MAP_FUZZ": { + "bit_depth_rule": "force_8bit", + "color": "#a2d1da", + "description": "Fuzz/Sheen Map", + "examples": [ + "_fuzz.", + "_sheen." + ], + "is_grayscale": true, + "keybind": "", + "standard_type": "FUZZ" + }, + "MAP_IDMAP": { + "bit_depth_rule": "force_8bit", + "color": "#ca8fb4", + "description": "ID Map (for masking)", + "examples": [ + "_id.", + "_matid." + ], + "is_grayscale": false, + "keybind": "", + "standard_type": "IDMAP" + }, + "MAP_MASK": { + "bit_depth_rule": "force_8bit", + "color": "#c6e2bf", + "description": "Generic Mask Map", + "examples": [ + "_mask." + ], + "is_grayscale": true, + "keybind": "", + "standard_type": "MASK" + }, + "MAP_IMPERFECTION": { + "bit_depth_rule": "force_8bit", + "color": "#e6d1a6", + "description": "Imperfection Map (scratches, dust)", + "examples": [ + "_imp.", + "_imperfection.", + "splatter", + "scratches", + "smudges", + "hairs", + "fingerprints" + ], + "is_grayscale": true, + "keybind": "", + "standard_type": "IMPERFECTION" + }, + "MODEL": { + "bit_depth_rule": "", + "color": "#3db2bd", + "description": "3D Model File", + "examples": [ + ".fbx", + ".obj" + ], + "is_grayscale": false, + "keybind": "", + "standard_type": "" + }, + "EXTRA": { + "bit_depth_rule": "", + "color": "#8c8c8c", + "description": "asset previews or metadata", + "examples": [ + ".txt", + ".zip", + "preview.", + "_flat.", + "_sphere.", + "_Cube.", + "thumb" + ], + "is_grayscale": false, + "keybind": "E", + "standard_type": "" + }, + "FILE_IGNORE": { + "bit_depth_rule": "", + "color": "#673d35", + "description": "File to be ignored", + "examples": [ + "Thumbs.db", + ".DS_Store" + ], + "is_grayscale": false, + "keybind": "X", + "standard_type": "" + } + } +} \ No newline at end of file diff --git a/config/llm_settings.json b/config/llm_settings.json new file mode 100644 index 0000000..e898fe5 --- /dev/null +++ b/config/llm_settings.json @@ -0,0 +1,267 @@ +{ + "llm_predictor_examples": [ + { + "input": "MessyTextures/Concrete_Damage_Set/concrete_col.png\nMessyTextures/Concrete_Damage_Set/concrete_N.png\nMessyTextures/Concrete_Damage_Set/concrete_rough.jpg\nMessyTextures/Concrete_Damage_Set/height_map_concrete.tif\nMessyTextures/Concrete_Damage_Set/Thumbs.db\nMessyTextures/Fabric_Pattern/pattern_01_diffuse.tga\nMessyTextures/Fabric_Pattern/pattern_01_ao.png\nMessyTextures/Fabric_Pattern/pattern_01_normal.png\nMessyTextures/Fabric_Pattern/notes.txt\nMessyTextures/Fabric_Pattern/variant_blue_diffuse.tga\nMessyTextures/Fabric_Pattern/fabric_flat.jpg", + "output": { + "individual_file_analysis": [ + { + "relative_file_path": "MessyTextures/Concrete_Damage_Set/concrete_col.png", + "classified_file_type": "MAP_COL", + "proposed_asset_group_name": "Concrete_Damage_Set" + }, + { + "relative_file_path": "MessyTextures/Concrete_Damage_Set/concrete_N.png", + "classified_file_type": "MAP_NRM", + "proposed_asset_group_name": "Concrete_Damage_Set" + }, + { + "relative_file_path": "MessyTextures/Concrete_Damage_Set/concrete_rough.jpg", + "classified_file_type": "MAP_ROUGH", + "proposed_asset_group_name": "Concrete_Damage_Set" + }, + { + "relative_file_path": "MessyTextures/Concrete_Damage_Set/height_map_concrete.tif", + "classified_file_type": "MAP_DISP", + "proposed_asset_group_name": "Concrete_Damage_Set" + }, + { + "relative_file_path": "MessyTextures/Concrete_Damage_Set/Thumbs.db", + "classified_file_type": "FILE_IGNORE", + "proposed_asset_group_name": null + }, + { + "relative_file_path": "MessyTextures/Fabric_Pattern/pattern_01_diffuse.tga", + "classified_file_type": "MAP_COL", + "proposed_asset_group_name": "Fabric_Pattern_01" + }, + { + "relative_file_path": "MessyTextures/Fabric_Pattern/pattern_01_ao.png", + "classified_file_type": "MAP_AO", + "proposed_asset_group_name": "Fabric_Pattern_01" + }, + { + "relative_file_path": "MessyTextures/Fabric_Pattern/pattern_01_normal.png", + "classified_file_type": "MAP_NRM", + "proposed_asset_group_name": "Fabric_Pattern_01" + }, + { + "relative_file_path": "MessyTextures/Fabric_Pattern/notes.txt", + "classified_file_type": "EXTRA", + "proposed_asset_group_name": "Fabric_Pattern_01" + }, + { + "relative_file_path": "MessyTextures/Fabric_Pattern/variant_blue_diffuse.tga", + "classified_file_type": "MAP_COL", + "proposed_asset_group_name": "Fabric_Pattern_01" + }, + { + "relative_file_path": "MessyTextures/Fabric_Pattern/fabric_flat.jpg", + "classified_file_type": "EXTRA", + "proposed_asset_group_name": "Fabric_Pattern_01" + } + ], + "asset_group_classifications": { + "Concrete_Damage_Set": "Surface", + "Fabric_Pattern_01": "Surface" + } + } + }, + { + "input": "SciFi_Drone/Drone_Model.fbx\nSciFi_Drone/Textures/Drone_BaseColor.png\nSciFi_Drone/Textures/Drone_Metallic.png\nSciFi_Drone/Textures/Drone_Roughness.png\nSciFi_Drone/Textures/Drone_Normal.png\nSciFi_Drone/Textures/Drone_Emissive.jpg\nSciFi_Drone/ReferenceImages/concept.jpg", + "output": { + "individual_file_analysis": [ + { + "relative_file_path": "SciFi_Drone/Drone_Model.fbx", + "classified_file_type": "MODEL", + "proposed_asset_group_name": "SciFi_Drone" + }, + { + "relative_file_path": "SciFi_Drone/Textures/Drone_BaseColor.png", + "classified_file_type": "MAP_COL", + "proposed_asset_group_name": "SciFi_Drone" + }, + { + "relative_file_path": "SciFi_Drone/Textures/Drone_Metallic.png", + "classified_file_type": "MAP_METAL", + "proposed_asset_group_name": "SciFi_Drone" + }, + { + "relative_file_path": "SciFi_Drone/Textures/Drone_Roughness.png", + "classified_file_type": "MAP_ROUGH", + "proposed_asset_group_name": "SciFi_Drone" + }, + { + "relative_file_path": "SciFi_Drone/Textures/Drone_Normal.png", + "classified_file_type": "MAP_NRM", + "proposed_asset_group_name": "SciFi_Drone" + }, + { + "relative_file_path": "SciFi_Drone/Textures/Drone_Emissive.jpg", + "classified_file_type": "EXTRA", + "proposed_asset_group_name": "SciFi_Drone" + }, + { + "relative_file_path": "SciFi_Drone/ReferenceImages/concept.jpg", + "classified_file_type": "EXTRA", + "proposed_asset_group_name": "SciFi_Drone" + } + ], + "asset_group_classifications": { + "SciFi_Drone": "Model" + } + } + }, + { + "input": "21_hairs_deposits.tif\n22_hairs_fabric.tif\n23_hairs_fibres.tif\n24_hairs_fibres.tif\n25_bonus_isolatedFingerprints.tif\n26_bonus_isolatedPalmprint.tif\n27_metal_aluminum.tif\n28_metal_castIron.tif\n29_scratcehes_deposits_shapes.tif\n30_scratches_deposits.tif", + "output": { + "individual_file_analysis": [ + { + "relative_file_path": "21_hairs_deposits.tif", + "classified_file_type": "MAP_IMPERFECTION", + "proposed_asset_group_name": "Hairs_Deposits_21" + }, + { + "relative_file_path": "22_hairs_fabric.tif", + "classified_file_type": "MAP_IMPERFECTION", + "proposed_asset_group_name": "Hairs_Fabric_22" + }, + { + "relative_file_path": "23_hairs_fibres.tif", + "classified_file_type": "MAP_IMPERFECTION", + "proposed_asset_group_name": "Hairs_Fibres_23" + }, + { + "relative_file_path": "24_hairs_fibres.tif", + "classified_file_type": "MAP_IMPERFECTION", + "proposed_asset_group_name": "Hairs_Fibres_24" + }, + { + "relative_file_path": "25_bonus_isolatedFingerprints.tif", + "classified_file_type": "MAP_IMPERFECTION", + "proposed_asset_group_name": "Bonus_IsolatedFingerprints_25" + }, + { + "relative_file_path": "26_bonus_isolatedPalmprint.tif", + "classified_file_type": "MAP_IMPERFECTION", + "proposed_asset_group_name": "Bonus_IsolatedPalmprint_26" + }, + { + "relative_file_path": "27_metal_aluminum.tif", + "classified_file_type": "MAP_IMPERFECTION", + "proposed_asset_group_name": "Metal_Aluminum_27" + }, + { + "relative_file_path": "28_metal_castIron.tif", + "classified_file_type": "MAP_IMPERFECTION", + "proposed_asset_group_name": "Metal_CastIron_28" + }, + { + "relative_file_path": "29_scratcehes_deposits_shapes.tif", + "classified_file_type": "MAP_IMPERFECTION", + "proposed_asset_group_name": "Scratches_Deposits_Shapes_29" + }, + { + "relative_file_path": "30_scratches_deposits.tif", + "classified_file_type": "MAP_IMPERFECTION", + "proposed_asset_group_name": "Scratches_Deposits_30" + } + ], + "asset_group_classifications": { + "Hairs_Deposits_21": "UtilityMap", + "Hairs_Fabric_22": "UtilityMap", + "Hairs_Fibres_23": "UtilityMap", + "Hairs_Fibres_24": "UtilityMap", + "Bonus_IsolatedFingerprints_25": "UtilityMap", + "Bonus_IsolatedPalmprint_26": "UtilityMap", + "Metal_Aluminum_27": "UtilityMap", + "Metal_CastIron_28": "UtilityMap", + "Scratches_Deposits_Shapes_29": "UtilityMap", + "Scratches_Deposits_30": "UtilityMap" + } + } + }, + { + "input": "Part1/TextureSupply_Boards001_A_28x300cm-Albedo.jpg\nPart1/TextureSupply_Boards001_A_28x300cm-Normal.jpg\nPart1/TextureSupply_Boards001_B_28x300cm-Albedo.jpg\nPart1/TextureSupply_Boards001_B_28x300cm-Normal.jpg\nPart1/TextureSupply_Boards001_C_28x300cm-Albedo.jpg\nPart1/TextureSupply_Boards001_C_28x300cm-Normal.jpg\nPart1/TextureSupply_Boards001_D_28x300cm-Albedo.jpg\nPart1/TextureSupply_Boards001_D_28x300cm-Normal.jpg\nPart1/TextureSupply_Boards001_E_28x300cm-Albedo.jpg\nPart1/TextureSupply_Boards001_E_28x300cm-Normal.jpg\nPart1/TextureSupply_Boards001_F_28x300cm-Albedo.jpg\nPart1/TextureSupply_Boards001_F_28x300cm-Normal.jpg", + "output": { + "individual_file_analysis": [ + { + "relative_file_path": "Part1/TextureSupply_Boards001_A_28x300cm-Albedo.jpg", + "classified_file_type": "MAP_COL", + "proposed_asset_group_name": "Boards001_A" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_A_28x300cm-Normal.jpg", + "classified_file_type": "MAP_NRM", + "proposed_asset_group_name": "Boards001_A" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_B_28x300cm-Albedo.jpg", + "classified_file_type": "MAP_COL", + "proposed_asset_group_name": "Boards001_B" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_B_28x300cm-Normal.jpg", + "classified_file_type": "MAP_NRM", + "proposed_asset_group_name": "Boards001_B" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_C_28x300cm-Albedo.jpg", + "classified_file_type": "MAP_COL", + "proposed_asset_group_name": "Boards001_C" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_C_28x300cm-Normal.jpg", + "classified_file_type": "MAP_NRM", + "proposed_asset_group_name": "Boards001_C" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_D_28x300cm-Albedo.jpg", + "classified_file_type": "MAP_COL", + "proposed_asset_group_name": "Boards001_D" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_D_28x300cm-Normal.jpg", + "classified_file_type": "MAP_NRM", + "proposed_asset_group_name": "Boards001_D" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_E_28x300cm-Albedo.jpg", + "classified_file_type": "MAP_COL", + "proposed_asset_group_name": "Boards001_E" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_E_28x300cm-Normal.jpg", + "classified_file_type": "MAP_NRM", + "proposed_asset_group_name": "Boards001_E" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_F_28x300cm-Albedo.jpg", + "classified_file_type": "MAP_COL", + "proposed_asset_group_name": "Boards001_F" + }, + { + "relative_file_path": "Part1/TextureSupply_Boards001_F_28x300cm-Normal.jpg", + "classified_file_type": "MAP_NRM", + "proposed_asset_group_name": "Boards001_F" + } + ], + "asset_group_classifications": { + "Boards001_A": "Surface", + "Boards001_B": "Surface", + "Boards001_C": "Surface", + "Boards001_D": "Surface", + "Boards001_E": "Surface", + "Boards001_F": "Surface" + } + } + } + ], + "asset_type_definition_format": "{KEY} = {DESCRIPTION}, examples of content of {KEY} could be: {EXAMPLES}", + "file_type_definition_format": "{KEY} = {DESCRIPTION}, examples of keywords for {KEY} could be: {EXAMPLES}", + "llm_endpoint_url": "http://100.65.14.122:1234/v1/chat/completions", + "llm_api_key": "", + "llm_model_name": "qwen2.5-coder:3b", + "llm_temperature": 0.5, + "llm_request_timeout": 120, + "llm_predictor_prompt": "You are an expert asset classification system. Your task is to analyze a list of file paths, understand their relationships based on naming and directory structure, and output a structured JSON object that classifies each file individually and then classifies the logical asset groups they belong to.\\n\\nDefinitions:\\n\\nAsset Types: These define the overall category of a logical asset group. Use one of the following keys when classifying asset groups. Each definition is provided as a formatted string (e.g., 'Surface = A single PBR material set..., examples: WoodFloor01, MetalPlate05'):\\n{ASSET_TYPE_DEFINITIONS}\\n\\n\\nFile Types: These define the specific purpose of each individual file. Use one of the following keys when classifying individual files. Each definition is provided as a formatted string (e.g., 'MAP_COL = Color/Albedo Map, examples: _col., _basecolor.'):\\n{FILE_TYPE_DEFINITIONS}\\n\\n\\nCore Task & Logic:\\n\\n1. **Individual File Analysis:**\\n * Examine each `relative_file_path` in the input `FILE_LIST`.\\n * For EACH file, determine its most likely `classified_file_type` using the `FILE_TYPE_DEFINITIONS`. Pay attention to filename suffixes, keywords, and extensions. Use `FILE_IGNORE` for files like `Thumbs.db` or `.DS_Store`. Use `EXTRA` for previews, metadata, or unidentifiable maps.\\n * For EACH file, propose a logical `proposed_asset_group_name` (string). This name should represent the asset the file likely belongs to, based on common base names (e.g., `WoodFloor01` from `WoodFloor01_col.png`, `WoodFloor01_nrm.png`) or directory structure (e.g., `SciFi_Drone` for files within that folder).\\n * Files that seem to be standalone utility maps (like `scratches.png`, `FlowMap.tif`) should get a unique group name derived from their filename (e.g., `Scratches`, `FlowMap`).\\n * If a file doesn't seem to belong to any logical group (e.g., a stray readme file in the root), you can propose `null` or a generic name like `Miscellaneous`.\\n * Be consistent with the proposed names for files belonging to the same logical asset.\\n * Populate the `individual_file_analysis` array with one object for *every* file in the input list, containing `relative_file_path`, `classified_file_type`, and `proposed_asset_group_name`.\\n\\n2. **Asset Group Classification:**\\n * Collect all unique, non-null `proposed_asset_group_name` values generated in the previous step.\\n * For EACH unique group name, determine the overall `asset_type` (using `ASSET_TYPE_DEFINITIONS`) based on the types of files assigned to that group name in the `individual_file_analysis`.\\n * Example: If files proposed as `AssetGroup1` include `MAP_COL`, `MAP_NRM`, `MAP_ROUGH`, classify `AssetGroup1` as `Surface`.\\n * Example: If files proposed as `AssetGroup2` include `MODEL` and texture maps, classify `AssetGroup2` as `Model`.\\n * Example: If `AssetGroup3` only has one file classified as `MAP_IMPERFECTION`, classify `AssetGroup3` as `UtilityMap`.\\n * Populate the `asset_group_classifications` dictionary, mapping each unique `proposed_asset_group_name` to its determined `asset_type`.\\n\\nInput File List:\\n\\ntext\\n{FILE_LIST}\\n\\n\\nOutput Format:\\n\\nYour response MUST be ONLY a single JSON object. You MAY include comments (using // or /* */) within the JSON structure for clarification if needed, but the core structure must be valid JSON. Do NOT include any text, explanations, or introductory phrases before or after the JSON object itself. Ensure all strings are correctly quoted and escaped.\\n\\nCRITICAL: The output JSON structure must strictly adhere to the following format:\\n\\n```json\\n{{\\n \"individual_file_analysis\": [\\n {{\\n // Optional comment about this file\\n \"relative_file_path\": \"string\", // Exact relative path from the input list\\n \"classified_file_type\": \"string\", // Key from FILE_TYPE_DEFINITIONS\\n \"proposed_asset_group_name\": \"string_or_null\" // Your suggested group name for this file\\n }}\\n // ... one object for EVERY file in the input list\\n ],\\n \"asset_group_classifications\": {{\\n // Dictionary mapping unique proposed group names to asset types\\n \"ProposedGroupName1\": \"string\", // Key: proposed_asset_group_name, Value: Key from ASSET_TYPE_DEFINITIONS\\n \"ProposedGroupName2\": \"string\"\\n // ... one entry for each unique, non-null proposed_asset_group_name\\n }}\\n}}\\n```\\n\\nExamples:\\n\\nHere are examples of input file lists and the desired JSON output, illustrating the two-part structure:\\n\\njson\\n[\\n {EXAMPLE_INPUT_OUTPUT_PAIRS}\\n]\\n\\n\\nNow, process the provided FILE_LIST and generate ONLY the JSON output according to these instructions. Remember to include an entry in `individual_file_analysis` for every single input file path." +} \ No newline at end of file diff --git a/config/suppliers.json b/config/suppliers.json new file mode 100644 index 0000000..a10affa --- /dev/null +++ b/config/suppliers.json @@ -0,0 +1,11 @@ +{ + "Dimensiva": { + "normal_map_type": "OpenGL" + }, + "Dinesen": { + "normal_map_type": "OpenGL" + }, + "Poliigon": { + "normal_map_type": "OpenGL" + } +} \ No newline at end of file diff --git a/configuration.py b/configuration.py index 0e65739..6a4bee1 100644 --- a/configuration.py +++ b/configuration.py @@ -1,26 +1,25 @@ -# configuration.py - import json import os -import importlib.util from pathlib import Path import logging -import re # Import the regex module +import re +import collections.abc -log = logging.getLogger(__name__) # Use logger defined in main.py +log = logging.getLogger(__name__) -# --- Constants --- -# Assumes config.py and presets/ are relative to this file's location BASE_DIR = Path(__file__).parent -CORE_CONFIG_PATH = BASE_DIR / "config.py" -PRESETS_DIR = BASE_DIR / "presets" +APP_SETTINGS_PATH = BASE_DIR / "config" / "app_settings.json" +LLM_SETTINGS_PATH = BASE_DIR / "config" / "llm_settings.json" +ASSET_TYPE_DEFINITIONS_PATH = BASE_DIR / "config" / "asset_type_definitions.json" +FILE_TYPE_DEFINITIONS_PATH = BASE_DIR / "config" / "file_type_definitions.json" +USER_SETTINGS_PATH = BASE_DIR / "config" / "user_settings.json" # New path for user settings +SUPPLIERS_CONFIG_PATH = BASE_DIR / "config" / "suppliers.json" +PRESETS_DIR = BASE_DIR / "Presets" -# --- Custom Exception --- class ConfigurationError(Exception): """Custom exception for configuration loading errors.""" pass -# --- Helper Functions --- def _get_base_map_type(target_map_string: str) -> str: """Extracts the base map type (e.g., 'COL') from a potentially numbered string ('COL-1').""" # Use regex to find the leading alphabetical part @@ -70,15 +69,33 @@ def _fnmatch_to_regex(pattern: str) -> str: # For filename matching, we usually want to find the pattern, not match the whole string. return res +def _deep_merge_dicts(base_dict: dict, override_dict: dict) -> dict: + """ + Recursively merges override_dict into base_dict. + If a key exists in both and both values are dicts, it recursively merges them. + Otherwise, the value from override_dict takes precedence. + Modifies base_dict in place and returns it. + """ + for key, value in override_dict.items(): + if isinstance(value, collections.abc.Mapping): + node = base_dict.get(key) # Use .get() to avoid creating empty dicts if not needed for override + if isinstance(node, collections.abc.Mapping): + _deep_merge_dicts(node, value) # node is base_dict[key], modified in place + else: + # If base_dict[key] is not a dict or doesn't exist, override it + base_dict[key] = value + else: + base_dict[key] = value + return base_dict + -# --- Configuration Class --- class Configuration: """ Loads and provides access to core settings combined with a specific preset. """ def __init__(self, preset_name: str): """ - Loads core config and the specified preset file. + Loads core config, user overrides, and the specified preset file. Args: preset_name: The name of the preset (without .json extension). @@ -88,10 +105,34 @@ class Configuration: """ log.debug(f"Initializing Configuration with preset: '{preset_name}'") self.preset_name = preset_name + + # 1. Load core settings self._core_settings: dict = self._load_core_config() + + # 2. Load asset type definitions + self._asset_type_definitions: dict = self._load_asset_type_definitions() + + # 3. Load file type definitions + self._file_type_definitions: dict = self._load_file_type_definitions() + + # 4. Load user settings + user_settings_overrides: dict = self._load_user_settings() + + # 5. Deep merge user settings onto core settings + if user_settings_overrides: + log.info("Applying user setting overrides to core settings.") + # _deep_merge_dicts modifies self._core_settings in place + _deep_merge_dicts(self._core_settings, user_settings_overrides) + + # 6. Load LLM settings + self._llm_settings: dict = self._load_llm_config() + + # 7. Load preset settings (conceptually overrides combined base + user for shared keys) self._preset_settings: dict = self._load_preset(preset_name) + + # 8. Validate and compile (after all base/user/preset settings are established) self._validate_configs() - self._compile_regex_patterns() # Compile regex after validation + self._compile_regex_patterns() log.info(f"Configuration loaded successfully using preset: '{self.preset_name}'") @@ -103,21 +144,14 @@ class Configuration: self.compiled_bit_depth_regex_map: dict[str, re.Pattern] = {} # Map: base_map_type -> list of tuples: (compiled_regex, original_keyword, rule_index) self.compiled_map_keyword_regex: dict[str, list[tuple[re.Pattern, str, int]]] = {} - # Store the original rule order for priority checking later if needed (can be removed if index is stored in tuple) - # self._map_type_rule_order: list[dict] = [] # Keep for now, might be useful elsewhere - # Compile Extra Patterns (case-insensitive) for pattern in self.move_to_extra_patterns: try: - # Use the raw fnmatch pattern directly if it's simple enough for re.search - # Or convert using helper if needed. Let's try direct search first. - # We want to find the pattern *within* the filename. - regex_str = _fnmatch_to_regex(pattern) # Convert wildcards + regex_str = _fnmatch_to_regex(pattern) self.compiled_extra_regex.append(re.compile(regex_str, re.IGNORECASE)) except re.error as e: log.warning(f"Failed to compile 'extra' regex pattern '{pattern}': {e}. Skipping pattern.") - # Compile Model Patterns (case-insensitive) model_patterns = self.asset_category_rules.get('model_patterns', []) for pattern in model_patterns: try: @@ -126,33 +160,23 @@ class Configuration: except re.error as e: log.warning(f"Failed to compile 'model' regex pattern '{pattern}': {e}. Skipping pattern.") - # Compile Bit Depth Variant Patterns (case-sensitive recommended) for map_type, pattern in self.source_bit_depth_variants.items(): try: - # These often rely on specific suffixes, so anchoring might be better? - # Let's stick to the converted pattern for now, assuming it ends with suffix. - regex_str = _fnmatch_to_regex(pattern) # e.g., ".*_DISP16.*" - # If the original pattern ended with *, remove the trailing '.*' for suffix matching + regex_str = _fnmatch_to_regex(pattern) if pattern.endswith('*'): - regex_str = regex_str.removesuffix('.*') # e.g., ".*_DISP16" - # Fallback for < 3.9: if regex_str.endswith('.*'): regex_str = regex_str[:-2] + regex_str = regex_str.removesuffix('.*') - # Use the fnmatch-converted regex directly, allowing matches anywhere in the filename - # This is less strict than anchoring to the end with \\.[^.]+$ - final_regex_str = regex_str # Use the result from _fnmatch_to_regex - self.compiled_bit_depth_regex_map[map_type] = re.compile(final_regex_str, re.IGNORECASE) # Added IGNORECASE + final_regex_str = regex_str + self.compiled_bit_depth_regex_map[map_type] = re.compile(final_regex_str, re.IGNORECASE) log.debug(f" Compiled bit depth variant for '{map_type}' as regex (IGNORECASE): {final_regex_str}") except re.error as e: log.warning(f"Failed to compile 'bit depth' regex pattern '{pattern}' for map type '{map_type}': {e}. Skipping pattern.") - # Compile Map Type Keywords (case-insensitive) based on the new structure - separator = re.escape(self.source_naming_separator) # Escape separator for regex - # Use defaultdict to easily append to lists for the same base type + separator = re.escape(self.source_naming_separator) from collections import defaultdict temp_compiled_map_regex = defaultdict(list) for rule_index, mapping_rule in enumerate(self.map_type_mapping): - # Validate rule structure (dictionary with target_type and keywords) if not isinstance(mapping_rule, dict) or \ 'target_type' not in mapping_rule or \ 'keywords' not in mapping_rule or \ @@ -160,37 +184,23 @@ class Configuration: log.warning(f"Skipping invalid map_type_mapping rule at index {rule_index}: {mapping_rule}. Expected dict with 'target_type' and 'keywords' list.") continue - target_type = mapping_rule['target_type'].upper() # Use the base type directly + target_type = mapping_rule['target_type'].upper() source_keywords = mapping_rule['keywords'] - # Store the rule for potential priority access later (optional, index is now in tuple) - # self._map_type_rule_order.append(mapping_rule) - if target_type not in self.standard_map_types: - log.warning(f"Map rule {rule_index} uses target_type '{target_type}' which is not in core config STANDARD_MAP_TYPES. Classification might be incomplete.") - # Continue processing anyway, as it might be intended for merging etc. - - # Compile keywords for this rule and store with context for keyword in source_keywords: if not isinstance(keyword, str): log.warning(f"Skipping non-string keyword '{keyword}' in rule {rule_index} for target '{target_type}'.") continue try: - # Match keyword potentially surrounded by separators or start/end - # Handle potential wildcards within the keyword using fnmatch conversion kw_regex_part = _fnmatch_to_regex(keyword) - # Build regex to match the keyword part, anchored by separators or string boundaries - # Use non-capturing groups (?:...) - # Capture the keyword part itself for potential use later if needed (group 1) regex_str = rf"(?:^|{separator})({kw_regex_part})(?:$|{separator})" compiled_regex = re.compile(regex_str, re.IGNORECASE) - # Append tuple: (compiled_regex, original_keyword, rule_index) temp_compiled_map_regex[target_type].append((compiled_regex, keyword, rule_index)) log.debug(f" Compiled keyword '{keyword}' (rule {rule_index}) for target '{target_type}' as regex: {regex_str}") except re.error as e: log.warning(f"Failed to compile map keyword regex '{keyword}' for target type '{target_type}': {e}. Skipping keyword.") - # Assign the compiled regex dictionary self.compiled_map_keyword_regex = dict(temp_compiled_map_regex) log.debug(f"Compiled map keyword regex keys: {list(self.compiled_map_keyword_regex.keys())}") @@ -198,48 +208,39 @@ class Configuration: def _load_core_config(self) -> dict: - """Loads settings from the core config.py file.""" - log.debug(f"Loading core config from: {CORE_CONFIG_PATH}") - if not CORE_CONFIG_PATH.is_file(): - raise ConfigurationError(f"Core configuration file not found: {CORE_CONFIG_PATH}") + """Loads settings from the core app_settings.json file.""" + log.debug(f"Loading core config from: {APP_SETTINGS_PATH}") + if not APP_SETTINGS_PATH.is_file(): + raise ConfigurationError(f"Core configuration file not found: {APP_SETTINGS_PATH}") try: - spec = importlib.util.spec_from_file_location("core_config", CORE_CONFIG_PATH) - if spec is None or spec.loader is None: - raise ConfigurationError(f"Could not create module spec for {CORE_CONFIG_PATH}") - core_config_module = importlib.util.module_from_spec(spec) - # Define default values for core settings in case they are missing in config.py - default_core_settings = { - 'TARGET_FILENAME_PATTERN': "{base_name}_{map_type}_{resolution}.{ext}", - 'STANDARD_MAP_TYPES': [], - 'EXTRA_FILES_SUBDIR': "Extra", - 'METADATA_FILENAME': "metadata.json", - 'IMAGE_RESOLUTIONS': {}, - 'ASPECT_RATIO_DECIMALS': 2, - 'MAP_BIT_DEPTH_RULES': {"DEFAULT": "respect"}, - 'OUTPUT_FORMAT_16BIT_PRIMARY': "png", - 'OUTPUT_FORMAT_16BIT_FALLBACK': "png", - 'OUTPUT_FORMAT_8BIT': "png", - 'MAP_MERGE_RULES': [], - 'CALCULATE_STATS_RESOLUTION': "1K", - 'DEFAULT_ASSET_CATEGORY': "Texture", - 'TEMP_DIR_PREFIX': "_PROCESS_ASSET_", - # --- Additions --- - 'JPG_QUALITY': 95, # Default JPG quality - 'RESOLUTION_THRESHOLD_FOR_JPG': 4096, # Default threshold - 'RESPECT_VARIANT_MAP_TYPES': [], # Default for map types that always get suffix - 'FORCE_LOSSLESS_MAP_TYPES': [] # Default for map types that must be lossless - } - # Load attributes from module, using defaults if missing - settings = default_core_settings.copy() - spec.loader.exec_module(core_config_module) - for name in default_core_settings: - if hasattr(core_config_module, name): - settings[name] = getattr(core_config_module, name) - + with open(APP_SETTINGS_PATH, 'r', encoding='utf-8') as f: + settings = json.load(f) log.debug(f"Core config loaded successfully.") return settings + except json.JSONDecodeError as e: + raise ConfigurationError(f"Failed to parse core configuration file {APP_SETTINGS_PATH}: Invalid JSON - {e}") except Exception as e: - raise ConfigurationError(f"Failed to load core configuration from {CORE_CONFIG_PATH}: {e}") + raise ConfigurationError(f"Failed to read core configuration file {APP_SETTINGS_PATH}: {e}") + + def _load_llm_config(self) -> dict: + """Loads settings from the llm_settings.json file.""" + log.debug(f"Loading LLM config from: {LLM_SETTINGS_PATH}") + if not LLM_SETTINGS_PATH.is_file(): + # Log a warning but don't raise an error, allow fallback if possible + log.warning(f"LLM configuration file not found: {LLM_SETTINGS_PATH}. LLM features might be disabled or use defaults.") + return {} + try: + with open(LLM_SETTINGS_PATH, 'r', encoding='utf-8') as f: + settings = json.load(f) + log.debug(f"LLM config loaded successfully.") + return settings + except json.JSONDecodeError as e: + log.error(f"Failed to parse LLM configuration file {LLM_SETTINGS_PATH}: Invalid JSON - {e}") + return {} + except Exception as e: + log.error(f"Failed to read LLM configuration file {LLM_SETTINGS_PATH}: {e}") + return {} + def _load_preset(self, preset_name: str) -> dict: """Loads the specified preset JSON file.""" @@ -261,9 +262,79 @@ class Configuration: except Exception as e: raise ConfigurationError(f"Failed to read preset file {preset_file}: {e}") + def _load_asset_type_definitions(self) -> dict: + """Loads asset type definitions from the asset_type_definitions.json file.""" + log.debug(f"Loading asset type definitions from: {ASSET_TYPE_DEFINITIONS_PATH}") + if not ASSET_TYPE_DEFINITIONS_PATH.is_file(): + raise ConfigurationError(f"Asset type definitions file not found: {ASSET_TYPE_DEFINITIONS_PATH}") + try: + with open(ASSET_TYPE_DEFINITIONS_PATH, 'r', encoding='utf-8') as f: + data = json.load(f) + if "ASSET_TYPE_DEFINITIONS" not in data: + raise ConfigurationError(f"Key 'ASSET_TYPE_DEFINITIONS' not found in {ASSET_TYPE_DEFINITIONS_PATH}") + settings = data["ASSET_TYPE_DEFINITIONS"] + if not isinstance(settings, dict): + raise ConfigurationError(f"'ASSET_TYPE_DEFINITIONS' in {ASSET_TYPE_DEFINITIONS_PATH} must be a dictionary.") + log.debug(f"Asset type definitions loaded successfully.") + return settings + except json.JSONDecodeError as e: + raise ConfigurationError(f"Failed to parse asset type definitions file {ASSET_TYPE_DEFINITIONS_PATH}: Invalid JSON - {e}") + except Exception as e: + raise ConfigurationError(f"Failed to read asset type definitions file {ASSET_TYPE_DEFINITIONS_PATH}: {e}") + + def _load_file_type_definitions(self) -> dict: + """Loads file type definitions from the file_type_definitions.json file.""" + log.debug(f"Loading file type definitions from: {FILE_TYPE_DEFINITIONS_PATH}") + if not FILE_TYPE_DEFINITIONS_PATH.is_file(): + raise ConfigurationError(f"File type definitions file not found: {FILE_TYPE_DEFINITIONS_PATH}") + try: + with open(FILE_TYPE_DEFINITIONS_PATH, 'r', encoding='utf-8') as f: + data = json.load(f) + if "FILE_TYPE_DEFINITIONS" not in data: + raise ConfigurationError(f"Key 'FILE_TYPE_DEFINITIONS' not found in {FILE_TYPE_DEFINITIONS_PATH}") + settings = data["FILE_TYPE_DEFINITIONS"] + if not isinstance(settings, dict): + raise ConfigurationError(f"'FILE_TYPE_DEFINITIONS' in {FILE_TYPE_DEFINITIONS_PATH} must be a dictionary.") + log.debug(f"File type definitions loaded successfully.") + return settings + except json.JSONDecodeError as e: + raise ConfigurationError(f"Failed to parse file type definitions file {FILE_TYPE_DEFINITIONS_PATH}: Invalid JSON - {e}") + except Exception as e: + raise ConfigurationError(f"Failed to read file type definitions file {FILE_TYPE_DEFINITIONS_PATH}: {e}") + + def _load_user_settings(self) -> dict: + """Loads user override settings from config/user_settings.json.""" + log.debug(f"Attempting to load user settings from: {USER_SETTINGS_PATH}") + if not USER_SETTINGS_PATH.is_file(): + log.info(f"User settings file not found: {USER_SETTINGS_PATH}. Proceeding without user overrides.") + return {} + try: + with open(USER_SETTINGS_PATH, 'r', encoding='utf-8') as f: + settings = json.load(f) + log.info(f"User settings loaded successfully from {USER_SETTINGS_PATH}.") + return settings + except json.JSONDecodeError as e: + log.warning(f"Failed to parse user settings file {USER_SETTINGS_PATH}: Invalid JSON - {e}. Using empty user settings.") + return {} + except Exception as e: + log.warning(f"Failed to read user settings file {USER_SETTINGS_PATH}: {e}. Using empty user settings.") + return {} + def _validate_configs(self): """Performs basic validation checks on loaded settings.""" log.debug("Validating loaded configurations...") + + # Validate new definition files first + if not isinstance(self._asset_type_definitions, dict): + raise ConfigurationError("Asset type definitions were not loaded correctly or are not a dictionary.") + if not self._asset_type_definitions: # Check if empty + raise ConfigurationError("Asset type definitions are empty.") + + if not isinstance(self._file_type_definitions, dict): + raise ConfigurationError("File type definitions were not loaded correctly or are not a dictionary.") + if not self._file_type_definitions: # Check if empty + raise ConfigurationError("File type definitions are empty.") + # Preset validation required_preset_keys = [ "preset_name", "supplier_name", "source_naming", "map_type_mapping", @@ -281,6 +352,15 @@ class Configuration: raise ConfigurationError(f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' must be a dictionary.") if 'target_type' not in rule or not isinstance(rule['target_type'], str): raise ConfigurationError(f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' is missing 'target_type' string.") + + valid_file_type_keys = self._file_type_definitions.keys() + if rule['target_type'] not in valid_file_type_keys: + raise ConfigurationError( + f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' " + f"has an invalid 'target_type': '{rule['target_type']}'. " + f"Must be one of {list(valid_file_type_keys)}." + ) + if 'keywords' not in rule or not isinstance(rule['keywords'], list): raise ConfigurationError(f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' is missing 'keywords' list.") for kw_index, keyword in enumerate(rule['keywords']): @@ -288,20 +368,40 @@ class Configuration: raise ConfigurationError(f"Preset '{self.preset_name}': Keyword at index {kw_index} in rule {index} ('{rule['target_type']}') must be a string.") - # Core validation (check types or specific values if needed) if not isinstance(self._core_settings.get('TARGET_FILENAME_PATTERN'), str): raise ConfigurationError("Core config 'TARGET_FILENAME_PATTERN' must be a string.") + if not isinstance(self._core_settings.get('OUTPUT_DIRECTORY_PATTERN'), str): + raise ConfigurationError("Core config 'OUTPUT_DIRECTORY_PATTERN' must be a string.") + if not isinstance(self._core_settings.get('OUTPUT_FILENAME_PATTERN'), str): + raise ConfigurationError("Core config 'OUTPUT_FILENAME_PATTERN' must be a string.") if not isinstance(self._core_settings.get('IMAGE_RESOLUTIONS'), dict): raise ConfigurationError("Core config 'IMAGE_RESOLUTIONS' must be a dictionary.") - if not isinstance(self._core_settings.get('STANDARD_MAP_TYPES'), list): - raise ConfigurationError("Core config 'STANDARD_MAP_TYPES' must be a list.") - # Add more checks as necessary + + # Validate DEFAULT_ASSET_CATEGORY + valid_asset_type_keys = self._asset_type_definitions.keys() + default_asset_category_value = self._core_settings.get('DEFAULT_ASSET_CATEGORY') + if not default_asset_category_value: + raise ConfigurationError("Core config 'DEFAULT_ASSET_CATEGORY' is missing.") + if default_asset_category_value not in valid_asset_type_keys: + raise ConfigurationError( + f"Core config 'DEFAULT_ASSET_CATEGORY' ('{default_asset_category_value}') " + f"is not a valid key in ASSET_TYPE_DEFINITIONS. " + f"Must be one of {list(valid_asset_type_keys)}." + ) + + if self._llm_settings: + required_llm_keys = [ + "llm_predictor_examples", "llm_endpoint_url", "llm_api_key", + "llm_model_name", "llm_temperature", "llm_request_timeout", + "llm_predictor_prompt" + ] + for key in required_llm_keys: + if key not in self._llm_settings: + # Log warning instead of raising error to allow partial functionality + log.warning(f"LLM config is missing recommended key: '{key}'. LLM features might not work correctly.") log.debug("Configuration validation passed.") - # --- Accessor Methods/Properties --- - # Use @property for direct access, methods for potentially complex lookups/defaults - @property def supplier_name(self) -> str: return self._preset_settings.get('supplier_name', 'DefaultSupplier') @@ -309,20 +409,31 @@ class Configuration: @property def default_asset_category(self) -> str: """Gets the default asset category from core settings.""" - # Provide a fallback default just in case it's missing from config.py - return self._core_settings.get('DEFAULT_ASSET_CATEGORY', 'Texture') + # Fallback should align with a valid key, and validation should catch issues. + return self._core_settings.get('DEFAULT_ASSET_CATEGORY', 'Surface') @property def target_filename_pattern(self) -> str: - return self._core_settings['TARGET_FILENAME_PATTERN'] # Assumes validation passed + return self._core_settings['TARGET_FILENAME_PATTERN'] + + @property + def output_directory_pattern(self) -> str: + """Gets the output directory pattern ONLY from core settings.""" + # Default pattern if missing in core settings (should be caught by validation) + default_pattern = "[supplier]/[assetname]" + return self._core_settings.get('OUTPUT_DIRECTORY_PATTERN', default_pattern) + + @property + def output_filename_pattern(self) -> str: + """Gets the output filename pattern ONLY from core settings.""" + # Default pattern if missing in core settings (should be caught by validation) + default_pattern = "[assetname]_[maptype]_[resolution].[ext]" + return self._core_settings.get('OUTPUT_FILENAME_PATTERN', default_pattern) @property def image_resolutions(self) -> dict[str, int]: return self._core_settings['IMAGE_RESOLUTIONS'] - @property - def standard_map_types(self) -> list[str]: - return self._core_settings['STANDARD_MAP_TYPES'] @property def map_type_mapping(self) -> list: @@ -383,12 +494,35 @@ class Configuration: @property def jpg_quality(self) -> int: """Gets the configured JPG quality level.""" - return self._core_settings.get('JPG_QUALITY', 95) # Use default if somehow missing + return self._core_settings.get('JPG_QUALITY', 95) + + @property + def invert_normal_green_globally(self) -> bool: + """Gets the global setting for inverting the green channel of normal maps.""" + # Default to False if the setting is missing in the core config + return self._core_settings.get('invert_normal_map_green_channel_globally', False) + + @property + def overwrite_existing(self) -> bool: + """Gets the setting for overwriting existing files from core settings.""" + return self._core_settings.get('overwrite_existing', False) + + @property + def png_compression_level(self) -> int: + """Gets the PNG compression level from core settings.""" + return self._core_settings.get('PNG_COMPRESSION', 6) # Default to 6 if not found @property def resolution_threshold_for_jpg(self) -> int: """Gets the pixel dimension threshold for using JPG for 8-bit images.""" - return self._core_settings.get('RESOLUTION_THRESHOLD_FOR_JPG', 4096) + value = self._core_settings.get('RESOLUTION_THRESHOLD_FOR_JPG', 4096) + log.info(f"CONFIGURATION_DEBUG: resolution_threshold_for_jpg property returning: {value} (type: {type(value)})") + # Ensure it's an int, as downstream might expect it. + # The .get() default is an int, but if the JSON had null or a string, it might be different. + if not isinstance(value, int): + log.warning(f"CONFIGURATION_DEBUG: RESOLUTION_THRESHOLD_FOR_JPG was not an int, got {type(value)}. Defaulting to 4096.") + return 4096 + return value @property def respect_variant_map_types(self) -> list: @@ -401,11 +535,44 @@ class Configuration: """Gets the list of map types that must always be saved losslessly.""" return self._core_settings.get('FORCE_LOSSLESS_MAP_TYPES', []) - def get_bit_depth_rule(self, map_type: str) -> str: - """Gets the bit depth rule ('respect' or 'force_8bit') for a given standard map type.""" - rules = self._core_settings.get('MAP_BIT_DEPTH_RULES', {}) - default_rule = rules.get('DEFAULT', 'respect') - return rules.get(map_type, default_rule) + def get_bit_depth_rule(self, map_type_input: str) -> str: + """ + Gets the bit depth rule ('respect', 'force_8bit', 'force_16bit') for a given map type identifier. + The map_type_input can be an FTD key (e.g., "MAP_COL") or a suffixed FTD key (e.g., "MAP_COL-1"). + """ + if not self._file_type_definitions: # Check if the attribute exists and is not empty + log.warning("File type definitions not loaded. Cannot determine bit depth rule.") + return "respect" + + file_type_definitions = self._file_type_definitions + + # 1. Try direct match with map_type_input as FTD key + definition = file_type_definitions.get(map_type_input) + if definition: + rule = definition.get('bit_depth_rule') + if rule in ['respect', 'force_8bit', 'force_16bit']: + return rule + else: + log.warning(f"FTD key '{map_type_input}' found, but 'bit_depth_rule' is missing or invalid: '{rule}'. Defaulting to 'respect'.") + return "respect" + + # 2. Try to derive base FTD key by stripping common variant suffixes + # Regex to remove trailing suffixes like -, -, _ + base_ftd_key_candidate = re.sub(r"(-[\w\d]+|_[\w\d]+)$", "", map_type_input) + if base_ftd_key_candidate != map_type_input: + definition = file_type_definitions.get(base_ftd_key_candidate) + if definition: + rule = definition.get('bit_depth_rule') + if rule in ['respect', 'force_8bit', 'force_16bit']: + log.debug(f"Derived base FTD key '{base_ftd_key_candidate}' from '{map_type_input}' and found bit depth rule: {rule}") + return rule + else: + log.warning(f"Derived base FTD key '{base_ftd_key_candidate}' from '{map_type_input}', but 'bit_depth_rule' is missing/invalid: '{rule}'. Defaulting to 'respect'.") + return "respect" + + # If no match found after trying direct and derived keys + log.warning(f"Map type identifier '{map_type_input}' (or its derived base) not found in FILE_TYPE_DEFINITIONS. Defaulting bit depth rule to 'respect'.") + return "respect" def get_16bit_output_formats(self) -> tuple[str, str]: """Gets the primary and fallback format names for 16-bit output.""" @@ -415,4 +582,369 @@ class Configuration: def get_8bit_output_format(self) -> str: """Gets the format name for 8-bit output.""" - return self._core_settings.get('OUTPUT_FORMAT_8BIT', 'png').lower() \ No newline at end of file + return self._core_settings.get('OUTPUT_FORMAT_8BIT', 'png').lower() + + def get_standard_map_type_aliases(self) -> list[str]: + """ + Derives a sorted list of unique standard map type aliases + from FILE_TYPE_DEFINITIONS. + """ + aliases = set() + # _file_type_definitions is guaranteed to be a dict by the loader + for _key, definition in self._file_type_definitions.items(): + if isinstance(definition, dict): + standard_type = definition.get('standard_type') + if standard_type and isinstance(standard_type, str) and standard_type.strip(): + aliases.add(standard_type) + return sorted(list(aliases)) + + def get_asset_type_definitions(self) -> dict: + """Returns the _asset_type_definitions dictionary.""" + return self._asset_type_definitions + + def get_asset_type_keys(self) -> list: + """Returns a list of valid asset type keys from core settings.""" + return list(self.get_asset_type_definitions().keys()) + + def get_file_type_definitions_with_examples(self) -> dict: + """Returns the _file_type_definitions dictionary (including descriptions and examples).""" + return self._file_type_definitions + + def get_file_type_keys(self) -> list: + """Returns a list of valid file type keys from core settings.""" + return list(self.get_file_type_definitions_with_examples().keys()) + + def get_llm_examples(self) -> list: + """Returns the list of LLM input/output examples from LLM settings.""" + # Use empty list as fallback if LLM settings file is missing/invalid + return self._llm_settings.get('llm_predictor_examples', []) + + @property + def llm_predictor_prompt(self) -> str: + """Returns the LLM predictor prompt string from LLM settings.""" + return self._llm_settings.get('llm_predictor_prompt', '') + + @property + def llm_endpoint_url(self) -> str: + """Returns the LLM endpoint URL from LLM settings.""" + return self._llm_settings.get('llm_endpoint_url', '') + + @property + def llm_api_key(self) -> str: + """Returns the LLM API key from LLM settings.""" + return self._llm_settings.get('llm_api_key', '') + + @property + def llm_model_name(self) -> str: + """Returns the LLM model name from LLM settings.""" + return self._llm_settings.get('llm_model_name', '') + + @property + def llm_temperature(self) -> float: + """Returns the LLM temperature from LLM settings.""" + return self._llm_settings.get('llm_temperature', 0.5) + + @property + def llm_request_timeout(self) -> int: + """Returns the LLM request timeout in seconds from LLM settings.""" + return self._llm_settings.get('llm_request_timeout', 120) + + @property + def FILE_TYPE_DEFINITIONS(self) -> dict: + return self._file_type_definitions + + @property + def keybind_config(self) -> dict[str, list[str]]: + """ + Processes FILE_TYPE_DEFINITIONS to create a mapping of keybinds + to their associated file type keys. + Example: {'C': ['MAP_COL'], 'R': ['MAP_ROUGH', 'MAP_GLOSS']} + """ + keybinds = {} + # _file_type_definitions is guaranteed to be a dict by the loader + for ftd_key, ftd_value in self._file_type_definitions.items(): + if isinstance(ftd_value, dict) and 'keybind' in ftd_value: + key = ftd_value['keybind'] + if key not in keybinds: + keybinds[key] = [] + keybinds[key].append(ftd_key) + + # Ensure toggleable keybinds have their file types in a consistent order if necessary + # For example, for 'R': ['MAP_ROUGH', 'MAP_GLOSS'] + # The order from app_settings.json is generally preserved by dict iteration in Python 3.7+ + # but explicit sorting could be added if a specific cycle order is critical beyond config file order. + # For now, we rely on the order they appear in the config. + return keybinds + +def load_base_config() -> dict: + """ + Loads base configuration by merging app_settings.json, user_settings.json (if exists), + asset_type_definitions.json, and file_type_definitions.json. + Does not load presets or perform full validation beyond basic file loading. + Returns a dictionary containing the merged settings. If app_settings.json + fails to load, an empty dictionary is returned. If other files + fail, errors are logged, and the function proceeds with what has been loaded. + """ + base_settings = {} + + # 1. Load app_settings.json (critical) + if not APP_SETTINGS_PATH.is_file(): + log.error(f"Critical: Base application settings file not found: {APP_SETTINGS_PATH}. Returning empty configuration.") + return {} + try: + with open(APP_SETTINGS_PATH, 'r', encoding='utf-8') as f: + base_settings = json.load(f) + log.info(f"Successfully loaded base application settings from: {APP_SETTINGS_PATH}") + except json.JSONDecodeError as e: + log.error(f"Critical: Failed to parse base application settings file {APP_SETTINGS_PATH}: Invalid JSON - {e}. Returning empty configuration.") + return {} + except Exception as e: + log.error(f"Critical: Failed to read base application settings file {APP_SETTINGS_PATH}: {e}. Returning empty configuration.") + return {} + + # 2. Attempt to load user_settings.json + user_settings_overrides = {} + if USER_SETTINGS_PATH.is_file(): + try: + with open(USER_SETTINGS_PATH, 'r', encoding='utf-8') as f: + user_settings_overrides = json.load(f) + log.info(f"User settings loaded successfully for base_config from {USER_SETTINGS_PATH}.") + except json.JSONDecodeError as e: + log.warning(f"Failed to parse user settings file {USER_SETTINGS_PATH} for base_config: Invalid JSON - {e}. Proceeding without these user overrides.") + except Exception as e: + log.warning(f"Failed to read user settings file {USER_SETTINGS_PATH} for base_config: {e}. Proceeding without these user overrides.") + + # 3. Deep merge user settings onto base_settings + if user_settings_overrides: + log.info("Applying user setting overrides to base_settings in load_base_config.") + # _deep_merge_dicts modifies base_settings in place + _deep_merge_dicts(base_settings, user_settings_overrides) + + # 4. Load asset_type_definitions.json (non-critical, merge if successful) + if not ASSET_TYPE_DEFINITIONS_PATH.is_file(): + log.error(f"Asset type definitions file not found: {ASSET_TYPE_DEFINITIONS_PATH}. Proceeding without it.") + else: + try: + with open(ASSET_TYPE_DEFINITIONS_PATH, 'r', encoding='utf-8') as f: + asset_defs_data = json.load(f) + if "ASSET_TYPE_DEFINITIONS" in asset_defs_data: + if isinstance(asset_defs_data["ASSET_TYPE_DEFINITIONS"], dict): + # Merge into base_settings, which might already contain user overrides + base_settings['ASSET_TYPE_DEFINITIONS'] = asset_defs_data["ASSET_TYPE_DEFINITIONS"] + log.info(f"Successfully loaded and merged ASSET_TYPE_DEFINITIONS from: {ASSET_TYPE_DEFINITIONS_PATH}") + else: + log.error(f"Value under 'ASSET_TYPE_DEFINITIONS' in {ASSET_TYPE_DEFINITIONS_PATH} is not a dictionary. Skipping merge.") + else: + log.error(f"Key 'ASSET_TYPE_DEFINITIONS' not found in {ASSET_TYPE_DEFINITIONS_PATH}. Skipping merge.") + except json.JSONDecodeError as e: + log.error(f"Failed to parse asset type definitions file {ASSET_TYPE_DEFINITIONS_PATH}: Invalid JSON - {e}. Skipping merge.") + except Exception as e: + log.error(f"Failed to read asset type definitions file {ASSET_TYPE_DEFINITIONS_PATH}: {e}. Skipping merge.") + + # 5. Load file_type_definitions.json (non-critical, merge if successful) + if not FILE_TYPE_DEFINITIONS_PATH.is_file(): + log.error(f"File type definitions file not found: {FILE_TYPE_DEFINITIONS_PATH}. Proceeding without it.") + else: + try: + with open(FILE_TYPE_DEFINITIONS_PATH, 'r', encoding='utf-8') as f: + file_defs_data = json.load(f) + if "FILE_TYPE_DEFINITIONS" in file_defs_data: + if isinstance(file_defs_data["FILE_TYPE_DEFINITIONS"], dict): + # Merge into base_settings + base_settings['FILE_TYPE_DEFINITIONS'] = file_defs_data["FILE_TYPE_DEFINITIONS"] + log.info(f"Successfully loaded and merged FILE_TYPE_DEFINITIONS from: {FILE_TYPE_DEFINITIONS_PATH}") + else: + log.error(f"Value under 'FILE_TYPE_DEFINITIONS' in {FILE_TYPE_DEFINITIONS_PATH} is not a dictionary. Skipping merge.") + else: + log.error(f"Key 'FILE_TYPE_DEFINITIONS' not found in {FILE_TYPE_DEFINITIONS_PATH}. Skipping merge.") + except json.JSONDecodeError as e: + log.error(f"Failed to parse file type definitions file {FILE_TYPE_DEFINITIONS_PATH}: Invalid JSON - {e}. Skipping merge.") + except Exception as e: + log.error(f"Failed to read file type definitions file {FILE_TYPE_DEFINITIONS_PATH}: {e}. Skipping merge.") + + return base_settings + +def save_llm_config(settings_dict: dict): + """ + Saves the provided LLM settings dictionary to llm_settings.json. + """ + log.debug(f"Saving LLM config to: {LLM_SETTINGS_PATH}") + try: + with open(LLM_SETTINGS_PATH, 'w', encoding='utf-8') as f: + json.dump(settings_dict, f, indent=4) + # Use info level for successful save + log.info(f"LLM config saved successfully to {LLM_SETTINGS_PATH}") + except Exception as e: + log.error(f"Failed to save LLM configuration file {LLM_SETTINGS_PATH}: {e}") + # Re-raise as ConfigurationError to signal failure upstream + raise ConfigurationError(f"Failed to save LLM configuration: {e}") +def save_user_config(settings_dict: dict): + """Saves the provided settings dictionary to user_settings.json.""" + log.debug(f"Saving user config to: {USER_SETTINGS_PATH}") + try: + # Ensure parent directory exists (though 'config/' should always exist) + USER_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) + with open(USER_SETTINGS_PATH, 'w', encoding='utf-8') as f: + json.dump(settings_dict, f, indent=4) + log.info(f"User config saved successfully to {USER_SETTINGS_PATH}") + except Exception as e: + log.error(f"Failed to save user configuration file {USER_SETTINGS_PATH}: {e}") + raise ConfigurationError(f"Failed to save user configuration: {e}") +def save_base_config(settings_dict: dict): + """ + Saves the provided settings dictionary to app_settings.json. + """ + log.debug(f"Saving base config to: {APP_SETTINGS_PATH}") + try: + with open(APP_SETTINGS_PATH, 'w', encoding='utf-8') as f: + json.dump(settings_dict, f, indent=4) + log.debug(f"Base config saved successfully.") + except Exception as e: + log.error(f"Failed to save base configuration file {APP_SETTINGS_PATH}: {e}") + raise ConfigurationError(f"Failed to save configuration: {e}") + +def load_asset_definitions() -> dict: + """ + Reads config/asset_type_definitions.json. + Returns the dictionary under the "ASSET_TYPE_DEFINITIONS" key. + Handles file not found or JSON errors gracefully (e.g., return empty dict, log error). + """ + log.debug(f"Loading asset type definitions from: {ASSET_TYPE_DEFINITIONS_PATH}") + if not ASSET_TYPE_DEFINITIONS_PATH.is_file(): + log.error(f"Asset type definitions file not found: {ASSET_TYPE_DEFINITIONS_PATH}") + return {} + try: + with open(ASSET_TYPE_DEFINITIONS_PATH, 'r', encoding='utf-8') as f: + data = json.load(f) + if "ASSET_TYPE_DEFINITIONS" not in data: + log.error(f"Key 'ASSET_TYPE_DEFINITIONS' not found in {ASSET_TYPE_DEFINITIONS_PATH}") + return {} + settings = data["ASSET_TYPE_DEFINITIONS"] + if not isinstance(settings, dict): + log.error(f"'ASSET_TYPE_DEFINITIONS' in {ASSET_TYPE_DEFINITIONS_PATH} must be a dictionary.") + return {} + log.debug(f"Asset type definitions loaded successfully.") + return settings + except json.JSONDecodeError as e: + log.error(f"Failed to parse asset type definitions file {ASSET_TYPE_DEFINITIONS_PATH}: Invalid JSON - {e}") + return {} + except Exception as e: + log.error(f"Failed to read asset type definitions file {ASSET_TYPE_DEFINITIONS_PATH}: {e}") + return {} + +def save_asset_definitions(data: dict): + """ + Takes a dictionary (representing the content for the "ASSET_TYPE_DEFINITIONS" key). + Writes it to config/asset_type_definitions.json under the root key "ASSET_TYPE_DEFINITIONS". + Handles potential I/O errors. + """ + log.debug(f"Saving asset type definitions to: {ASSET_TYPE_DEFINITIONS_PATH}") + try: + with open(ASSET_TYPE_DEFINITIONS_PATH, 'w', encoding='utf-8') as f: + json.dump({"ASSET_TYPE_DEFINITIONS": data}, f, indent=4) + log.info(f"Asset type definitions saved successfully to {ASSET_TYPE_DEFINITIONS_PATH}") + except Exception as e: + log.error(f"Failed to save asset type definitions file {ASSET_TYPE_DEFINITIONS_PATH}: {e}") + raise ConfigurationError(f"Failed to save asset type definitions: {e}") + +def load_file_type_definitions() -> dict: + """ + Reads config/file_type_definitions.json. + Returns the dictionary under the "FILE_TYPE_DEFINITIONS" key. + Handles errors gracefully. + """ + log.debug(f"Loading file type definitions from: {FILE_TYPE_DEFINITIONS_PATH}") + if not FILE_TYPE_DEFINITIONS_PATH.is_file(): + log.error(f"File type definitions file not found: {FILE_TYPE_DEFINITIONS_PATH}") + return {} + try: + with open(FILE_TYPE_DEFINITIONS_PATH, 'r', encoding='utf-8') as f: + data = json.load(f) + if "FILE_TYPE_DEFINITIONS" not in data: + log.error(f"Key 'FILE_TYPE_DEFINITIONS' not found in {FILE_TYPE_DEFINITIONS_PATH}") + return {} + settings = data["FILE_TYPE_DEFINITIONS"] + if not isinstance(settings, dict): + log.error(f"'FILE_TYPE_DEFINITIONS' in {FILE_TYPE_DEFINITIONS_PATH} must be a dictionary.") + return {} + log.debug(f"File type definitions loaded successfully.") + return settings + except json.JSONDecodeError as e: + log.error(f"Failed to parse file type definitions file {FILE_TYPE_DEFINITIONS_PATH}: Invalid JSON - {e}") + return {} + except Exception as e: + log.error(f"Failed to read file type definitions file {FILE_TYPE_DEFINITIONS_PATH}: {e}") + return {} + +def save_file_type_definitions(data: dict): + """ + Takes a dictionary (representing content for "FILE_TYPE_DEFINITIONS" key). + Writes it to config/file_type_definitions.json under the root key "FILE_TYPE_DEFINITIONS". + Handles errors. + """ + log.debug(f"Saving file type definitions to: {FILE_TYPE_DEFINITIONS_PATH}") + try: + with open(FILE_TYPE_DEFINITIONS_PATH, 'w', encoding='utf-8') as f: + json.dump({"FILE_TYPE_DEFINITIONS": data}, f, indent=4) + log.info(f"File type definitions saved successfully to {FILE_TYPE_DEFINITIONS_PATH}") + except Exception as e: + log.error(f"Failed to save file type definitions file {FILE_TYPE_DEFINITIONS_PATH}: {e}") + raise ConfigurationError(f"Failed to save file type definitions: {e}") + +def load_supplier_settings() -> dict: + """ + Reads config/suppliers.json. + Returns the entire dictionary. + Handles file not found (return empty dict) or JSON errors. + If the loaded data is a list (old format), convert it in memory to the new + dictionary format, defaulting normal_map_type to "OpenGL" for each supplier. + """ + log.debug(f"Loading supplier settings from: {SUPPLIERS_CONFIG_PATH}") + if not SUPPLIERS_CONFIG_PATH.is_file(): + log.warning(f"Supplier settings file not found: {SUPPLIERS_CONFIG_PATH}. Returning empty dict.") + return {} + try: + with open(SUPPLIERS_CONFIG_PATH, 'r', encoding='utf-8') as f: + data = json.load(f) + + if isinstance(data, list): + log.warning(f"Supplier settings in {SUPPLIERS_CONFIG_PATH} is in the old list format. Converting to new dictionary format.") + new_data = {} + for supplier_name in data: + if isinstance(supplier_name, str): + new_data[supplier_name] = {"normal_map_type": "OpenGL"} + else: + log.warning(f"Skipping non-string item '{supplier_name}' during old format conversion of supplier settings.") + log.info(f"Supplier settings converted to new format: {new_data}") + return new_data + + if not isinstance(data, dict): + log.error(f"Supplier settings in {SUPPLIERS_CONFIG_PATH} must be a dictionary. Found {type(data)}. Returning empty dict.") + return {} + + log.debug(f"Supplier settings loaded successfully.") + return data + except json.JSONDecodeError as e: + log.error(f"Failed to parse supplier settings file {SUPPLIERS_CONFIG_PATH}: Invalid JSON - {e}. Returning empty dict.") + return {} + except Exception as e: + log.error(f"Failed to read supplier settings file {SUPPLIERS_CONFIG_PATH}: {e}. Returning empty dict.") + return {} + +def save_supplier_settings(data: dict): + """ + Takes a dictionary (in the new format). + Writes it directly to config/suppliers.json. + Handles errors. + """ + log.debug(f"Saving supplier settings to: {SUPPLIERS_CONFIG_PATH}") + if not isinstance(data, dict): + log.error(f"Data for save_supplier_settings must be a dictionary. Got {type(data)}.") + raise ConfigurationError(f"Invalid data type for saving supplier settings: {type(data)}") + try: + with open(SUPPLIERS_CONFIG_PATH, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2) # Using indent=2 as per the example for suppliers.json + log.info(f"Supplier settings saved successfully to {SUPPLIERS_CONFIG_PATH}") + except Exception as e: + log.error(f"Failed to save supplier settings file {SUPPLIERS_CONFIG_PATH}: {e}") + raise ConfigurationError(f"Failed to save supplier settings: {e}") diff --git a/documentation/definitions_editor_plan.md b/documentation/definitions_editor_plan.md new file mode 100644 index 0000000..b69ff7a --- /dev/null +++ b/documentation/definitions_editor_plan.md @@ -0,0 +1,137 @@ +# Plan for New Definitions Editor UI + +## 1. Overview + +This document outlines the plan to create a new, dedicated UI for managing "Asset Type Definitions", "File Type Definitions", and "Supplier Settings". This editor will provide a more structured and user-friendly way to manage these core application configurations, which are currently stored in separate JSON files. + +## 2. General Design Principles + +* **Dedicated Dialog:** The editor will be a new `QDialog` (e.g., `DefinitionsEditorDialog`). +* **Access Point:** Launched from the `MainWindow` menu bar (e.g., under a "Definitions" menu or "Edit" -> "Edit Definitions..."). +* **Tabbed Interface:** The dialog will use a `QTabWidget` to separate the management of different definition types. +* **List/Details View:** Each tab will generally follow a two-pane layout: + * **Left Pane:** A `QListWidget` displaying the primary keys or names of the definitions (e.g., asset type names, file type IDs, supplier names). Includes "Add" and "Remove" buttons for managing these primary entries. + * **Right Pane:** A details area (e.g., `QGroupBox` with a `QFormLayout`) that shows the specific settings for the item selected in the left-pane list. +* **Data Persistence:** The dialog will load from and save to the respective JSON configuration files: + * Asset Types: `config/asset_type_definitions.json` + * File Types: `config/file_type_definitions.json` + * Supplier Settings: `config/suppliers.json` (This file will be refactored from a simple list to a dictionary of supplier objects). +* **User Experience:** Standard "Save" and "Cancel" buttons, with a check for unsaved changes. + +## 3. Tab-Specific Plans + +### 3.1. Asset Type Definitions Tab + +* **Manages:** `config/asset_type_definitions.json` +* **UI Sketch:** + ```mermaid + graph LR + subgraph AssetTypeTab [Asset Type Definitions Tab] + direction LR + AssetList[QListWidget (Asset Type Keys e.g., "Surface")] --> AssetDetailsGroup{Details for Selected Asset Type}; + end + + subgraph AssetDetailsGroup + direction TB + Desc[Description: QTextEdit] + Color[Color: QPushButton ("Choose Color...") + Color Swatch Display] + Examples[Examples: QListWidget + Add/Remove Example Buttons] + end + AssetActions["Add Asset Type (Prompt for Name)\nRemove Selected Asset Type"] --> AssetList + ``` +* **Details:** + * **Left Pane:** `QListWidget` for asset type names. "Add Asset Type" (prompts for new key) and "Remove Selected Asset Type" buttons. + * **Right Pane (Details):** + * `description`: `QTextEdit`. + * `color`: `QPushButton` opening `QColorDialog`, with an adjacent `QLabel` to display the color swatch. + * `examples`: `QListWidget` with "Add Example" (`QInputDialog.getText`) and "Remove Selected Example" buttons. + +### 3.2. File Type Definitions Tab + +* **Manages:** `config/file_type_definitions.json` +* **UI Sketch:** + ```mermaid + graph LR + subgraph FileTypeTab [File Type Definitions Tab] + direction LR + FileList[QListWidget (File Type Keys e.g., "MAP_COL")] --> FileDetailsGroup{Details for Selected File Type}; + end + + subgraph FileDetailsGroup + direction TB + DescF[Description: QTextEdit] + ColorF[Color: QPushButton ("Choose Color...") + Color Swatch Display] + ExamplesF[Examples: QListWidget + Add/Remove Example Buttons] + StdType[Standard Type: QLineEdit] + BitDepth[Bit Depth Rule: QComboBox ("respect", "force_8bit", "force_16bit")] + IsGrayscale[Is Grayscale: QCheckBox] + Keybind[Keybind: QLineEdit (1 char)] + end + FileActions["Add File Type (Prompt for ID)\nRemove Selected File Type"] --> FileList + ``` +* **Details:** + * **Left Pane:** `QListWidget` for file type IDs. "Add File Type" (prompts for new key) and "Remove Selected File Type" buttons. + * **Right Pane (Details):** + * `description`: `QTextEdit`. + * `color`: `QPushButton` opening `QColorDialog`, with an adjacent `QLabel` for color swatch. + * `examples`: `QListWidget` with "Add Example" and "Remove Selected Example" buttons. + * `standard_type`: `QLineEdit`. + * `bit_depth_rule`: `QComboBox` (options: "respect", "force_8bit", "force_16bit"). + * `is_grayscale`: `QCheckBox`. + * `keybind`: `QLineEdit` (validation for single character recommended). + +### 3.3. Supplier Settings Tab + +* **Manages:** `config/suppliers.json` (This file will be refactored to a dictionary structure, e.g., `{"SupplierName": {"normal_map_type": "OpenGL", ...}}`). +* **UI Sketch:** + ```mermaid + graph LR + subgraph SupplierTab [Supplier Settings Tab] + direction LR + SupplierList[QListWidget (Supplier Names)] --> SupplierDetailsGroup{Details for Selected Supplier}; + end + + subgraph SupplierDetailsGroup + direction TB + NormalMapType[Normal Map Type: QComboBox ("OpenGL", "DirectX")] + %% Future supplier-specific settings can be added here + end + SupplierActions["Add Supplier (Prompt for Name)\nRemove Selected Supplier"] --> SupplierList + ``` +* **Details:** + * **Left Pane:** `QListWidget` for supplier names. "Add Supplier" (prompts for new name) and "Remove Selected Supplier" buttons. + * **Right Pane (Details):** + * `normal_map_type`: `QComboBox` (options: "OpenGL", "DirectX"). Default for new suppliers: "OpenGL". + * *(Space for future supplier-specific settings).* +* **Data Handling Note for `config/suppliers.json`:** + * The editor will load from and save to `config/suppliers.json` using the new dictionary format (supplier name as key, object of settings as value). + * Initial implementation might require `config/suppliers.json` to be manually updated to this new format if it currently exists as a simple list. Alternatively, the editor could attempt an automatic conversion on first load if the old list format is detected, or prompt the user. For the first pass, assuming the editor works with the new format is simpler. + +## 4. Implementation Steps (High-Level) + +1. **(Potentially Manual First Step) Refactor `config/suppliers.json`:** If `config/suppliers.json` exists as a list, manually convert it to the new dictionary structure (e.g., `{"SupplierName": {"normal_map_type": "OpenGL"}}`) before starting UI development for this tab, or plan for the editor to handle this conversion. +2. **Create `DefinitionsEditorDialog` Class:** Inherit from `QDialog`. +3. **Implement UI Structure:** Main `QTabWidget`, and for each tab, the two-pane layout with `QListWidget`, `QGroupBox` for details, and relevant input widgets (`QLineEdit`, `QTextEdit`, `QComboBox`, `QCheckBox`, `QPushButton`). +4. **Implement Loading Logic:** + * For each tab, read data from its corresponding JSON file. + * Populate the left-pane `QListWidget` with the primary keys/names. + * Store the full data structure internally (e.g., in dictionaries within the dialog instance). +5. **Implement Display Logic:** + * When an item is selected in a `QListWidget`, populate the right-pane detail fields with the data for that item. +6. **Implement Editing Logic:** + * Ensure that changes made in the detail fields (text edits, combobox selections, checkbox states, color choices, list example modifications) update the corresponding internal data structure for the currently selected item. +7. **Implement Add/Remove Functionality:** + * For each definition type (Asset Type, File Type, Supplier), implement the "Add" and "Remove" buttons. + * "Add": Prompt for a unique key/name, create a new default entry in the internal data, and add it to the `QListWidget`. + * "Remove": Remove the selected item from the `QListWidget` and the internal data. + * For "examples" lists within Asset and File types, implement their "Add Example" and "Remove Selected Example" buttons. +8. **Implement Saving Logic:** + * When the main "Save" button is clicked: + * Write the (potentially modified) Asset Type definitions data structure to `config/asset_type_definitions.json`. + * Write File Type definitions to `config/file_type_definitions.json`. + * Write Supplier settings (in the new dictionary format) to `config/suppliers.json`. + * Consider creating new dedicated save functions in `configuration.py` for each of these files if they don't already exist or if existing ones are not suitable. +9. **Implement Unsaved Changes Check & Cancel Logic.** +10. **Integrate Dialog Launch:** Add a menu action in `MainWindow.py` to open the `DefinitionsEditorDialog`. + +This plan provides a comprehensive approach to creating a dedicated editor for these crucial application definitions. \ No newline at end of file diff --git a/documentation/preferences_refactor_plan.md b/documentation/preferences_refactor_plan.md new file mode 100644 index 0000000..ab50c73 --- /dev/null +++ b/documentation/preferences_refactor_plan.md @@ -0,0 +1,113 @@ +# Refactoring Plan for Preferences Window (ConfigEditorDialog) + +## 1. Overview + +This document outlines the plan to refactor the preferences window (`gui/config_editor_dialog.py`). The primary goal is to address issues related to misaligned scope, poor user experience for certain data types, and incomplete interactivity. The refactoring will focus on making the `ConfigEditorDialog` a robust editor for settings in `config/app_settings.json` that are intended to be overridden by the user via `config/user_settings.json`. + +## 2. Assessment Summary + +* **Misaligned Scope:** The dialog currently includes UI for "Asset Type Definitions" and "File Type Definitions". However, these are managed in separate dedicated JSON files ([`config/asset_type_definitions.json`](config/asset_type_definitions.json) and [`config/file_type_definitions.json`](config/file_type_definitions.json)) and are not saved by this dialog (which targets `config/user_settings.json`). +* **Poor UX for Data Types:** + * Lists (e.g., `RESPECT_VARIANT_MAP_TYPES`) are edited as comma-separated strings. + * Dictionary-like structures (e.g., `IMAGE_RESOLUTIONS`) are handled inconsistently (JSON defines as dict, UI attempts list-of-pairs). + * Editing complex list-of-objects (e.g., `MAP_MERGE_RULES`) is functionally incomplete. +* **Incomplete Interactivity:** Many table-based editors lack "Add/Remove Row" functionality and proper cell delegates for intuitive editing. +* **LLM Settings:** Confirmed to be correctly managed by the separate `LLMEditorWidget` and `config/llm_settings.json`, so they are out of scope for this specific dialog refactor. + +## 3. Refactoring Phases and Plan Details + +```mermaid +graph TD + A[Start: Current State] --> B{Phase 1: Correct Scope & Critical UX/Data Fixes}; + B --> C{Phase 2: Enhance MAP_MERGE_RULES Editor}; + C --> D{Phase 3: General UX & Table Interactivity}; + D --> E[End: Refactored Preferences Window]; + + subgraph "Phase 1: Correct Scope & Critical UX/Data Fixes" + B1[Remove Definitions Editing from ConfigEditorDialog] + B2[Improve List Editing for RESPECT_VARIANT_MAP_TYPES] + B3[Fix IMAGE_RESOLUTIONS Handling (Dictionary)] + B4[Handle Simple Nested Settings (e.g., general_settings)] + end + + subgraph "Phase 2: Enhance MAP_MERGE_RULES Editor" + C1[Implement Add/Remove for Merge Rules] + C2[Improve Rule Detail Editing (ComboBoxes, SpinBoxes)] + end + + subgraph "Phase 3: General UX & Table Interactivity" + D1[Implement IMAGE_RESOLUTIONS Table Add/Remove Buttons] + D2[Implement Necessary Table Cell Delegates (e.g., for IMAGE_RESOLUTIONS values)] + D3[Review/Refine Tab Layout & Widget Grouping] + end + + B --> B1; B --> B2; B --> B3; B --> B4; + C --> C1; C --> C2; + D --> D1; D --> D2; D --> D3; +``` + +### Phase 1: Correct Scope & Critical UX/Data Fixes (in `gui/config_editor_dialog.py`) + +1. **Remove Definitions Editing:** + * **Action:** In `populate_definitions_tab`, remove the inner `QTabWidget` and the code that creates/populates the "Asset Types" and "File Types" tables. + * The `DEFAULT_ASSET_CATEGORY` `QComboBox` (for the setting from `app_settings.json`) should remain. Its items should be populated using keys obtained from the `Configuration` class (which loads the actual `ASSET_TYPE_DEFINITIONS` from its dedicated file). + * **Rationale:** Simplifies the dialog to settings managed via `user_settings.json`. Editing of the full definition files requires dedicated UI (see Future Enhancements note). + +2. **Improve `RESPECT_VARIANT_MAP_TYPES` Editing:** + * **Action:** In `populate_output_naming_tab`, replace the `QLineEdit` for `RESPECT_VARIANT_MAP_TYPES` with a `QListWidget` and "Add"/"Remove" buttons. + * "Add" button: Use `QInputDialog.getItem` with items populated from `Configuration.get_file_type_keys()` (or similar method accessing loaded `FILE_TYPE_DEFINITIONS`) to allow users to select a valid file type key. + * "Remove" button: Remove the selected item from the `QListWidget`. + * Update `save_settings` to read the list of strings from this `QListWidget`. + * Update `populate_widgets_from_settings` to populate this `QListWidget`. + +3. **Fix `IMAGE_RESOLUTIONS` Handling:** + * **Action:** In `populate_image_processing_tab`: + * The `QTableWidget` for `IMAGE_RESOLUTIONS` should have two columns: "Name" (string, for the dictionary key) and "Resolution (px)" (integer, for the dictionary value). + * In `populate_image_resolutions_table`, ensure it correctly populates from the dictionary structure in `self.settings['IMAGE_RESOLUTIONS']` (from `app_settings.json`). + * In `save_settings`, ensure it correctly reads data from the table and reconstructs the `IMAGE_RESOLUTIONS` dictionary (e.g., `{"4K": 4096, "2K": 2048}`) when saving to `user_settings.json`. + * ComboBoxes `CALCULATE_STATS_RESOLUTION` and `RESOLUTION_THRESHOLD_FOR_JPG` should be populated with the *keys* (names like "4K", "2K") from the `IMAGE_RESOLUTIONS` dictionary. `RESOLUTION_THRESHOLD_FOR_JPG` should also include "Never" and "Always" options. The `save_settings` method needs to correctly map these special ComboBox values back to appropriate storable values if necessary (e.g., sentinel numbers or specific strings if the backend configuration expects them for "Never"/"Always"). + +4. **Handle Simple Nested Settings (e.g., `general_settings`):** + * **Action:** For `general_settings.invert_normal_map_green_channel_globally` (from `config/app_settings.json`): + * Add a `QCheckBox` labeled "Invert Normal Map Green Channel Globally" to an appropriate tab (e.g., "Image Processing" or a "General" tab after layout review). + * Update `populate_widgets_from_settings` to read `self.settings.get('general_settings', {}).get('invert_normal_map_green_channel_globally', False)`. + * Update `save_settings` to write this value back to `target_file_content.setdefault('general_settings', {})['invert_normal_map_green_channel_globally'] = widget.isChecked()`. + +### Phase 2: Enhance `MAP_MERGE_RULES` Editor (in `gui/config_editor_dialog.py`) + +1. **Rule Management:** + * **Action:** In `populate_map_merging_tab`: + * Connect the "Add Rule" button: + * Create a default new rule dictionary (e.g., `{"output_map_type": "NEW_RULE", "inputs": {}, "defaults": {}, "output_bit_depth": "respect_inputs"}`). + * Add it to the internal list of rules that will be saved (e.g., a copy of `self.settings['MAP_MERGE_RULES']` that gets modified). + * Add a new `QListWidgetItem` for it and select it to display its details. + * Connect the "Remove Rule" button: + * Remove the selected rule from the internal list and the `QListWidget`. + * Clear the details panel. + +2. **Rule Details Panel Improvements (`display_merge_rule_details`):** + * **`output_map_type`:** Change the `QLineEdit` to a `QComboBox`. Populate its items from `Configuration.get_file_type_keys()`. + * **`inputs` Table:** The "Input Map Type" column cells should use a `QComboBox` delegate, populated with `Configuration.get_file_type_keys()` plus an empty/None option. + * **`defaults` Table:** The "Default Value" column cells should use a `QDoubleSpinBox` delegate (e.g., range 0.0 to 1.0, or 0-255 if appropriate for specific channel types). + * Ensure changes in these detail editors update the underlying rule data associated with the selected `QListWidgetItem` and the internal list of rules. + +### Phase 3: General UX & Table Interactivity (in `gui/config_editor_dialog.py`) + +1. **Implement `IMAGE_RESOLUTIONS` Table Add/Remove Buttons:** + * **Action:** In `populate_image_processing_tab`, connect the "Add Row" and "Remove Row" buttons for the `IMAGE_RESOLUTIONS` table. + * "Add Row": Prompt for "Name" (string) and "Resolution (px)" (integer). + * "Remove Row": Remove the selected row from the table and the underlying data. +2. **Implement Necessary Table Cell Delegates:** + * **Action:** For the `IMAGE_RESOLUTIONS` table, the "Resolution (px)" column should use a `QSpinBox` delegate or a `QLineEdit` with integer validation to ensure correct data input. +3. **Review/Refine Tab Layout & Widget Grouping:** + * **Action:** After the functional changes, review the overall layout of tabs and the grouping of settings within `gui/config_editor_dialog.py`. + * Ensure settings from `config/app_settings.json` are logically placed and clearly labeled. + * Verify widget labels are descriptive and tooltips are helpful where needed. + * Confirm correct mapping between UI widgets and the keys in `app_settings.json` (e.g., `OUTPUT_FILENAME_PATTERN` vs. `TARGET_FILENAME_PATTERN`). + +## 4. Future Enhancements (Out of Scope for this Refactor) + +* **Dedicated Editors for Definitions:** As per user feedback, if `ASSET_TYPE_DEFINITIONS` and `FILE_TYPE_DEFINITIONS` require UI-based editing, dedicated dialogs/widgets should be created. These would read from and save to their respective files ([`config/asset_type_definitions.json`](config/asset_type_definitions.json) and [`config/file_type_definitions.json`](config/file_type_definitions.json)) and could adopt a list/details UI similar to the `MAP_MERGE_RULES` editor. +* **Live Updates:** Consider mechanisms for applying some settings without requiring an application restart, if feasible for specific settings. + +This plan aims to create a more focused, usable, and correct preferences window. \ No newline at end of file diff --git a/gui/__pycache__/main_window.cpython-310.pyc b/gui/__pycache__/main_window.cpython-310.pyc deleted file mode 100644 index d9bc28f..0000000 Binary files a/gui/__pycache__/main_window.cpython-310.pyc and /dev/null differ diff --git a/gui/__pycache__/main_window.cpython-313.pyc b/gui/__pycache__/main_window.cpython-313.pyc deleted file mode 100644 index bb97ec8..0000000 Binary files a/gui/__pycache__/main_window.cpython-313.pyc and /dev/null differ diff --git a/gui/__pycache__/prediction_handler.cpython-310.pyc b/gui/__pycache__/prediction_handler.cpython-310.pyc deleted file mode 100644 index e6df774..0000000 Binary files a/gui/__pycache__/prediction_handler.cpython-310.pyc and /dev/null differ diff --git a/gui/__pycache__/prediction_handler.cpython-313.pyc b/gui/__pycache__/prediction_handler.cpython-313.pyc deleted file mode 100644 index 0e37b3d..0000000 Binary files a/gui/__pycache__/prediction_handler.cpython-313.pyc and /dev/null differ diff --git a/gui/__pycache__/preset_editor_dialog.cpython-310.pyc b/gui/__pycache__/preset_editor_dialog.cpython-310.pyc deleted file mode 100644 index fa2d3ab..0000000 Binary files a/gui/__pycache__/preset_editor_dialog.cpython-310.pyc and /dev/null differ diff --git a/gui/__pycache__/preset_editor_dialog.cpython-313.pyc b/gui/__pycache__/preset_editor_dialog.cpython-313.pyc deleted file mode 100644 index e6ebe77..0000000 Binary files a/gui/__pycache__/preset_editor_dialog.cpython-313.pyc and /dev/null differ diff --git a/gui/__pycache__/preview_table_model.cpython-313.pyc b/gui/__pycache__/preview_table_model.cpython-313.pyc deleted file mode 100644 index 1a6820d..0000000 Binary files a/gui/__pycache__/preview_table_model.cpython-313.pyc and /dev/null differ diff --git a/gui/__pycache__/processing_handler.cpython-310.pyc b/gui/__pycache__/processing_handler.cpython-310.pyc deleted file mode 100644 index 5464979..0000000 Binary files a/gui/__pycache__/processing_handler.cpython-310.pyc and /dev/null differ diff --git a/gui/__pycache__/processing_handler.cpython-313.pyc b/gui/__pycache__/processing_handler.cpython-313.pyc deleted file mode 100644 index 6c1d666..0000000 Binary files a/gui/__pycache__/processing_handler.cpython-313.pyc and /dev/null differ diff --git a/gui/asset_restructure_handler.py b/gui/asset_restructure_handler.py new file mode 100644 index 0000000..ebe46b6 --- /dev/null +++ b/gui/asset_restructure_handler.py @@ -0,0 +1,224 @@ +# gui/asset_restructure_handler.py +import logging +from PySide6.QtCore import QObject, Slot, QModelIndex +from PySide6.QtGui import QColor # Might be needed if copying logic directly, though unlikely now +from pathlib import Path +from .unified_view_model import UnifiedViewModel +from rule_structure import SourceRule, AssetRule, FileRule + +log = logging.getLogger(__name__) + +class AssetRestructureHandler(QObject): + """ + Handles the model restructuring logic triggered by changes + to FileRule target asset overrides in the UnifiedViewModel. + """ + def __init__(self, model: UnifiedViewModel, parent=None): + super().__init__(parent) + if not isinstance(model, UnifiedViewModel): + raise TypeError("AssetRestructureHandler requires a UnifiedViewModel instance.") + self.model = model + # Connect to the modified signal (passes FileRule object) + self.model.targetAssetOverrideChanged.connect(self.handle_target_asset_override) + # Connect to the new signal for AssetRule name changes + self.model.assetNameChanged.connect(self.handle_asset_name_changed) + log.debug("AssetRestructureHandler initialized.") + + @Slot(FileRule, str, QModelIndex) + def handle_target_asset_override(self, file_rule_item: FileRule, new_target_name: str, index: QModelIndex): + """ + Slot connected to UnifiedViewModel.targetAssetOverrideChanged. + Orchestrates model changes based on the new target asset path. + + Args: + file_rule_item: The FileRule object whose override changed. + new_target_name: The new target asset path (string). + index: The QModelIndex of the changed item (passed by the signal). + """ + if not isinstance(file_rule_item, FileRule): + log.warning(f"Handler received targetAssetOverrideChanged for non-FileRule item: {type(file_rule_item)}. Aborting.") + return + + # Crucially, use file_rule_item for all logic. 'index' is for context or if model interaction is *unavoidable* (which it shouldn't be here). + log.debug(f"Handler received targetAssetOverrideChanged: OBJECT='{file_rule_item!r}', FILE_PATH='{file_rule_item.file_path}', NEW_NAME='{new_target_name}'") + + # Ensure new_target_name is a string or None (already string from signal, but good practice if it could be object) + effective_new_target_name = str(new_target_name).strip() if new_target_name is not None else None + if effective_new_target_name == "": effective_new_target_name = None # Treat empty string as None + + # --- Get necessary context --- + old_parent_asset = getattr(file_rule_item, 'parent_asset', None) + if not old_parent_asset: + log.error(f"Handler: File item '{Path(file_rule_item.file_path).name}' has no parent asset. Cannot restructure.") + # Note: Data change already happened in setData, cannot easily revert here. + return + + source_rule = getattr(old_parent_asset, 'parent_source', None) + if not source_rule: + log.error(f"Handler: Could not find SourceRule for parent asset '{old_parent_asset.asset_name}'. Cannot restructure.") + return + + # --- Logic based on the new target name --- + target_parent_asset = None + target_parent_index = QModelIndex() # This will be the QModelIndex of the target AssetRule + move_occurred = False + + # 1. Find existing target parent AssetRule within the same SourceRule + if effective_new_target_name: + for i, asset in enumerate(source_rule.assets): + if asset.asset_name == effective_new_target_name: + target_parent_asset = asset + # Get QModelIndex for the target parent AssetRule + try: + source_rule_row = self.model._source_rules.index(source_rule) + source_rule_index = self.model.createIndex(source_rule_row, 0, source_rule) + target_parent_index = self.model.index(i, 0, source_rule_index) # QModelIndex for the target AssetRule + if not target_parent_index.isValid(): + log.error(f"Handler: Failed to create valid QModelIndex for existing target parent '{effective_new_target_name}'.") + target_parent_asset = None # Reset if index is invalid + except ValueError: + log.error(f"Handler: Could not find SourceRule index while looking for target parent '{effective_new_target_name}'.") + target_parent_asset = None # Reset if index is invalid + break + + # 2. Handle Move or Creation + if target_parent_asset: # An existing AssetRule to move to was found + # --- Move to Existing Parent --- + if target_parent_asset != old_parent_asset: + log.info(f"Handler: Moving file '{Path(file_rule_item.file_path).name}' to existing asset '{target_parent_asset.asset_name}'.") + # The 'index' parameter IS the QModelIndex of the FileRule being changed. + # No need to re-fetch or re-validate it if the signal emits it correctly. + # The core issue was using a stale index to get the *object*, now we *have* the object. + source_file_qmodelindex = index + + if not source_file_qmodelindex or not source_file_qmodelindex.isValid(): # Should always be valid if signal emits it + log.error(f"Handler: Received invalid QModelIndex for source file '{Path(file_rule_item.file_path).name}'. Cannot move.") + return + + if self.model.moveFileRule(source_file_qmodelindex, target_parent_index): # target_parent_index is for the AssetRule + move_occurred = True + else: + log.error(f"Handler: Model failed to move file rule to existing asset '{target_parent_asset.asset_name}'.") + else: + # Target is the same as the old parent. No move needed. + log.debug(f"Handler: Target asset '{effective_new_target_name}' is the same as the current parent. No move required.") + + elif effective_new_target_name: # No existing AssetRule found, but a new name is provided. Create it. + # --- Create New Parent AssetRule and Move --- + log.info(f"Handler: Creating new asset '{effective_new_target_name}' and moving file '{Path(file_rule_item.file_path).name}'.") + new_asset_qmodelindex = self.model.createAssetRule(source_rule, effective_new_target_name, copy_from_asset=old_parent_asset) + + if new_asset_qmodelindex.isValid(): + target_parent_asset = new_asset_qmodelindex.internalPointer() # Get the newly created AssetRule object + target_parent_index = new_asset_qmodelindex # The QModelIndex of the new AssetRule + + source_file_qmodelindex = index + if not source_file_qmodelindex or not source_file_qmodelindex.isValid(): # Should always be valid + log.error(f"Handler: Received invalid QModelIndex for source file '{Path(file_rule_item.file_path).name}'. Cannot move to new asset.") + self.model.removeAssetRule(target_parent_asset) # Attempt to clean up newly created asset + return + + if self.model.moveFileRule(source_file_qmodelindex, target_parent_index): # Move to the new AssetRule + move_occurred = True + else: + log.error(f"Handler: Model failed to move file rule to newly created asset '{effective_new_target_name}'.") + # Consider removing the newly created asset if the move fails + self.model.removeAssetRule(target_parent_asset) # Attempt to clean up + else: + log.error(f"Handler: Model failed to create new asset rule '{effective_new_target_name}'. Cannot move file.") + + else: # effective_new_target_name is None or empty (override cleared) + log.debug(f"Handler: Target asset override cleared for '{Path(file_rule_item.file_path).name}'. File remains in parent '{old_parent_asset.asset_name}'.") + # No move occurs in this interpretation if the override is simply cleared. + # The file_rule_item.target_asset_name_override is now None (set by model.setData). + + # 3. Cleanup Empty Old Parent (only if a move occurred and old parent is now empty) + if move_occurred and old_parent_asset and not old_parent_asset.files and old_parent_asset != target_parent_asset: + log.info(f"Handler: Attempting to remove empty old parent asset '{old_parent_asset.asset_name}'.") + if not self.model.removeAssetRule(old_parent_asset): + log.warning(f"Handler: Model failed to remove empty old parent asset '{old_parent_asset.asset_name}'.") + elif move_occurred: + log.debug(f"Handler: Old parent asset '{old_parent_asset.asset_name}' still contains files or is the target. No removal needed.") + + log.debug(f"Handler finished processing targetAssetOverrideChanged for '{Path(file_rule_item.file_path).name}'.") + + def _get_qmodelindex_for_item(self, item_to_find): + """ + Helper to find the QModelIndex for a given FileRule or AssetRule item. + Returns a valid QModelIndex or QModelIndex() if not found/invalid. + """ + if isinstance(item_to_find, FileRule): + parent_asset = getattr(item_to_find, 'parent_asset', None) + if not parent_asset: return QModelIndex() + source_rule = getattr(parent_asset, 'parent_source', None) + if not source_rule: return QModelIndex() + + try: + source_rule_row = self.model._source_rules.index(source_rule) + source_rule_index = self.model.createIndex(source_rule_row, 0, source_rule) + if not source_rule_index.isValid(): return QModelIndex() + + parent_asset_row = source_rule.assets.index(parent_asset) + parent_asset_index = self.model.index(parent_asset_row, 0, source_rule_index) + if not parent_asset_index.isValid(): return QModelIndex() + + item_row = parent_asset.files.index(item_to_find) + return self.model.index(item_row, 0, parent_asset_index) + except ValueError: + log.error(f"Error finding item {item_to_find} in model hierarchy during QModelIndex reconstruction.") + return QModelIndex() + + elif isinstance(item_to_find, AssetRule): + source_rule = getattr(item_to_find, 'parent_source', None) + if not source_rule: return QModelIndex() + try: + source_rule_row = self.model._source_rules.index(source_rule) + source_rule_index = self.model.createIndex(source_rule_row, 0, source_rule) + if not source_rule_index.isValid(): return QModelIndex() + + item_row = source_rule.assets.index(item_to_find) + return self.model.index(item_row, 0, source_rule_index) + except ValueError: + log.error(f"Error finding asset {item_to_find.asset_name} in model hierarchy during QModelIndex reconstruction.") + return QModelIndex() + return QModelIndex() + + @Slot(AssetRule, str, QModelIndex) + def handle_asset_name_changed(self, asset_rule_item: AssetRule, new_name: str, index: QModelIndex): + """ + Slot connected to UnifiedViewModel.assetNameChanged. + Handles logic when an AssetRule's name is changed. + + Args: + asset_rule_item: The AssetRule object whose name changed. + new_name: The new name of the asset. + index: The QModelIndex of the changed AssetRule item. + """ + if not isinstance(asset_rule_item, AssetRule): + log.warning(f"Handler received assetNameChanged for non-AssetRule item: {type(asset_rule_item)}. Aborting.") + return + + # The 'old_name' is not directly passed by the new signal signature. + # If needed, it would have to be inferred or stored prior to the change. + # However, the model's setData already handles updating child FileRule targets. + # This handler's main job is to react to the AssetRule object itself. + log.debug(f"Handler received assetNameChanged: OBJECT='{asset_rule_item!r}', ASSET_NAME='{asset_rule_item.asset_name}', NEW_NAME='{new_name}'") + + + # The UnifiedViewModel.setData has already updated FileRule.target_asset_name_override + # for any FileRules that were pointing to the *old* asset name across the entire model. + + # The primary purpose of this handler slot, given the problem description, + # is to ensure that if any restructuring or disk operations were tied to an AssetRule's + # name, they would now correctly use 'asset_rule_item' (the actual object) + # and 'new_name'. + + # For this specific task, confirming correct identification is key. + # If this handler were also responsible for renaming directories on disk, + # this is where that logic would go, using asset_rule_item and new_name. + # The old name would need to be retrieved differently if essential for such an operation, + # e.g. by storing it temporarily before the model's setData commits the change, + # or by having the signal pass it (which it currently doesn't in the revised design). + # For now, the model handles the critical part of updating linked FileRules. + + log.info(f"Handler correctly identified AssetRule '{new_name}' for processing using the direct object. Model's setData handles related FileRule target updates.") \ No newline at end of file diff --git a/gui/base_prediction_handler.py b/gui/base_prediction_handler.py new file mode 100644 index 0000000..83ed46e --- /dev/null +++ b/gui/base_prediction_handler.py @@ -0,0 +1,133 @@ +import logging +import time +from abc import ABC, abstractmethod +from pathlib import Path +from typing import List, Any + +from PySide6.QtCore import QObject, Signal, Slot, QThread + +# Assuming rule_structure defines SourceRule +try: + from rule_structure import SourceRule +except ImportError: + print("ERROR (BasePredictionHandler): Failed to import SourceRule. Predictions might fail.") + # Define a placeholder if the import fails to allow type hinting + class SourceRule: pass + +from abc import ABCMeta +from PySide6.QtCore import QObject + +# Combine metaclasses to avoid conflict between QObject and ABC +class QtABCMeta(type(QObject), ABCMeta): + pass +log = logging.getLogger(__name__) + +class BasePredictionHandler(QObject, ABC, metaclass=QtABCMeta): + """ + Abstract base class for prediction handlers that generate SourceRule hierarchies. + Designed to be run in a separate QThread. + """ + # --- Standardized Signals --- + # Emitted when prediction is successfully completed. + # Args: input_source_identifier (str), results (List[SourceRule]) + prediction_ready = Signal(str, list) + + # Emitted when an error occurs during prediction. + # Args: input_source_identifier (str), error_message (str) + prediction_error = Signal(str, str) + + # Emitted for status updates during the prediction process. + # Args: status_message (str) + status_update = Signal(str) + + def __init__(self, input_source_identifier: str, parent: QObject = None): + """ + Initializes the base handler. + + Args: + input_source_identifier: The unique identifier for the input source (e.g., file path). + parent: The parent QObject. + """ + super().__init__(parent) + self.input_source_identifier = input_source_identifier + self._is_running = False + self._is_cancelled = False + + @property + def is_running(self) -> bool: + """Returns True if the handler is currently processing.""" + return self._is_running + + @Slot() + def run(self): + """ + Main execution slot intended to be connected to QThread.started. + Handles the overall process: setup, execution, error handling, signaling. + """ + log.debug(f"--> Entered BasePredictionHandler.run() for {self.input_source_identifier}") + if self._is_running: + log.warning(f"Handler for '{self.input_source_identifier}' is already running. Aborting.") + return + if self._is_cancelled: + log.info(f"Handler for '{self.input_source_identifier}' was cancelled before starting.") + # Optionally emit an error or specific signal for cancellation before start + return + + self._is_running = True + self._is_cancelled = False + thread_id = QThread.currentThread() # Use currentThread() for PySide6 + log.info(f"[{time.time():.4f}][T:{thread_id}] Starting prediction run for: {self.input_source_identifier}") + self.status_update.emit(f"Starting analysis for '{Path(self.input_source_identifier).name}'...") + + try: + # --- Execute Core Logic --- + results = self._perform_prediction() + + if self._is_cancelled: + log.info(f"Prediction cancelled during execution for: {self.input_source_identifier}") + self.prediction_error.emit(self.input_source_identifier, "Prediction cancelled by user.") + else: + # --- Emit Success Signal --- + log.info(f"[{time.time():.4f}][T:{thread_id}] Prediction successful for '{self.input_source_identifier}'. Emitting results.") + self.prediction_ready.emit(self.input_source_identifier, results) + self.status_update.emit(f"Analysis complete for '{Path(self.input_source_identifier).name}'.") + + except Exception as e: + # --- Emit Error Signal --- + log.exception(f"[{time.time():.4f}][T:{thread_id}] Error during prediction for '{self.input_source_identifier}': {e}") + error_msg = f"Error analyzing '{Path(self.input_source_identifier).name}': {e}" + self.prediction_error.emit(self.input_source_identifier, error_msg) + # Status update might be redundant if error is shown elsewhere, but can be useful + # Status update might be redundant if error is shown elsewhere, but can be useful + + finally: + # --- Cleanup --- + self._is_running = False + log.info(f"[{time.time():.4f}][T:{thread_id}] Finished prediction run for: {self.input_source_identifier}") + # Note: The thread itself should be managed (quit/deleteLater) by the caller + # based on the signals emitted (prediction_ready, prediction_error). + + @Slot() + def cancel(self): + """ + Sets the cancellation flag. The running process should check this flag periodically. + """ + log.info(f"Cancellation requested for handler: {self.input_source_identifier}") + self._is_cancelled = True + self.status_update.emit(f"Cancellation requested for '{Path(self.input_source_identifier).name}'...") + + + @abstractmethod + def _perform_prediction(self) -> List[SourceRule]: + """ + Abstract method to be implemented by concrete subclasses. + This method contains the specific logic for generating the SourceRule list. + It should periodically check `self._is_cancelled`. + + Returns: + A list of SourceRule objects representing the prediction results. + + Raises: + Exception: If any critical error occurs during the prediction process. + """ + pass \ No newline at end of file diff --git a/gui/config_editor_dialog.py b/gui/config_editor_dialog.py new file mode 100644 index 0000000..fd3ed80 --- /dev/null +++ b/gui/config_editor_dialog.py @@ -0,0 +1,1549 @@ + +import json +import os # Added for path operations +import copy # Added for deepcopy +from PySide6.QtWidgets import ( + QDialog, QVBoxLayout, QHBoxLayout, QTabWidget, QWidget, + QLineEdit, QSpinBox, QDoubleSpinBox, QComboBox, QCheckBox, + QPushButton, QFileDialog, QLabel, QTableWidget, + QTableWidgetItem, QDialogButtonBox, QMessageBox, QListWidget, + QListWidgetItem, QFormLayout, QGroupBox, QStackedWidget, QInputDialog, + QHeaderView, QSizePolicy +) +from PySide6.QtGui import QColor, QPainter +from PySide6.QtCore import Qt, QEvent +from PySide6.QtWidgets import QColorDialog, QStyledItemDelegate, QApplication + +# Assuming configuration.py is in the parent directory or accessible +try: + from configuration import load_base_config, save_user_config, ConfigurationError +except ImportError: + # Fallback import for testing or different project structure + from ..configuration import load_base_config, save_user_config, ConfigurationError, Configuration + + +# --- Custom Delegate for Color Editing --- +class ColorDelegate(QStyledItemDelegate): + def paint(self, painter: QPainter, option, index): + # Get color string from model data (EditRole is where we store it) + color_str = index.model().data(index, Qt.EditRole) + if isinstance(color_str, str) and color_str.startswith('#'): + color = QColor(color_str) + if color.isValid(): + painter.fillRect(option.rect, color) + # Optionally draw text (e.g., the hex code) centered + # painter.drawText(option.rect, Qt.AlignCenter, color_str) + return # Prevent default painting + + # Fallback to default painting if no valid color + super().paint(painter, option, index) + + def createEditor(self, parent, option, index): + # No editor needed, handled by editorEvent + return None + + def editorEvent(self, event, model, option, index): + if event.type() == QEvent.MouseButtonPress and event.button() == Qt.LeftButton: + current_color_str = model.data(index, Qt.EditRole) + initial_color = QColor(current_color_str) if isinstance(current_color_str, str) else Qt.white + + color = QColorDialog.getColor(initial_color, None, "Select Color") + + if color.isValid(): + new_color_str = color.name() # Get #RRGGBB format + model.setData(index, new_color_str, Qt.EditRole) + # Trigger update for the background role as well, although paint should handle it + # model.setData(index, QColor(new_color_str), Qt.BackgroundRole) + return True # Event handled + return False # Event not handled + + def setModelData(self, editor, model, index): + # Not strictly needed as setData is called in editorEvent + pass + +# --- Custom Delegate for ComboBox Editing in Tables --- +class ComboBoxDelegate(QStyledItemDelegate): + def __init__(self, items=None, parent=None): + super().__init__(parent) + self.items = items if items is not None else [] + + def createEditor(self, parent, option, index): + editor = QComboBox(parent) + editor.addItems(self.items) + return editor + + def setEditorData(self, editor, index): + value = index.model().data(index, Qt.EditRole) + if value is not None: + editor.setCurrentText(str(value)) + + def setModelData(self, editor, model, index): + value = editor.currentText() + model.setData(index, value, Qt.EditRole) + + def updateEditorGeometry(self, editor, option, index): + editor.setGeometry(option.rect) + +# --- Custom Delegate for DoubleSpinBox Editing in Tables --- +class DoubleSpinBoxDelegate(QStyledItemDelegate): + def __init__(self, min_val=0.0, max_val=1.0, decimals=2, step=0.01, parent=None): + super().__init__(parent) + self.min_val = min_val + self.max_val = max_val + self.decimals = decimals + self.step = step + + def createEditor(self, parent, option, index): + editor = QDoubleSpinBox(parent) + editor.setMinimum(self.min_val) + editor.setMaximum(self.max_val) + editor.setDecimals(self.decimals) + editor.setSingleStep(self.step) + return editor + + def setEditorData(self, editor, index): + value = index.model().data(index, Qt.EditRole) + try: + editor.setValue(float(value)) + except (TypeError, ValueError): + editor.setValue(self.min_val) # Default if conversion fails + + def setModelData(self, editor, model, index): + editor.interpretText() # Ensure the editor's value is up-to-date + value = editor.value() + model.setData(index, value, Qt.EditRole) + + def updateEditorGeometry(self, editor, option, index): + editor.setGeometry(option.rect) + + +class SpinBoxDelegate(QStyledItemDelegate): + def __init__(self, min_val=1, max_val=32768, step=1, parent=None): + super().__init__(parent) + self.min_val = min_val + self.max_val = max_val + self.step = step + + def createEditor(self, parent, option, index): + editor = QSpinBox(parent) + editor.setMinimum(self.min_val) + editor.setMaximum(self.max_val) + editor.setSingleStep(self.step) + return editor + + def setEditorData(self, editor, index): + value = index.model().data(index, Qt.EditRole) + try: + editor.setValue(int(value)) + except (TypeError, ValueError): + editor.setValue(self.min_val) # Default if conversion fails + + def setModelData(self, editor, model, index): + editor.interpretText() # Ensure the editor's value is up-to-date + value = editor.value() + model.setData(index, value, Qt.EditRole) + + def updateEditorGeometry(self, editor, option, index): + editor.setGeometry(option.rect) + +class ConfigEditorDialog(QDialog): + def __init__(self, parent=None): + super().__init__(parent) + self.setWindowTitle("Configuration Editor") + self.setGeometry(100, 100, 800, 600) + + self.settings = {} + self.widgets = {} # Dictionary to hold references to created widgets + + self.main_layout = QVBoxLayout(self) + self.tab_widget = QTabWidget() + self.main_layout.addWidget(self.tab_widget) + + self.button_box = QDialogButtonBox(QDialogButtonBox.Save | QDialogButtonBox.Cancel) + self.button_box.accepted.connect(self.save_settings) + self.button_box.rejected.connect(self.reject) + self.main_layout.addWidget(self.button_box) + + self.load_settings() # Load settings FIRST + self.create_tabs() # THEN create widgets based on settings + self.populate_widgets_from_settings() # Populate widgets after creation + + def load_settings(self): + """Loads settings from the configuration file.""" + try: + self.settings = load_base_config() + # Store a deep copy of the initial user-configurable settings for granular save. + # These are settings from the effective configuration (base + user + defs) + # that this dialog manages and are intended for user_settings.json. + # Exclude definitions that are stored in separate files or not directly managed here. + self.original_user_configurable_settings = {} # Initialize first + if self.settings: # Ensure settings were loaded + keys_to_copy = [ + k for k in self.settings + if k not in ["ASSET_TYPE_DEFINITIONS", "FILE_TYPE_DEFINITIONS"] + ] + # Create a temporary dictionary with only the keys to be copied + temp_original_settings = { + k: self.settings[k] for k in keys_to_copy if k in self.settings + } + self.original_user_configurable_settings = copy.deepcopy(temp_original_settings) + print("Original user-configurable settings (relevant parts) deep copied for comparison.") # Debug print + else: + # If self.settings is None or empty, original_user_configurable_settings remains an empty dict. + print("Settings not loaded or empty; original_user_configurable_settings initialized as empty.") # Debug print + except Exception as e: + QMessageBox.critical(self, "Loading Error", f"Failed to load configuration: {e}") + self.settings = {} # Use empty settings on failure + self.original_user_configurable_settings = {} + # Optionally disable save button or widgets if loading fails + self.button_box.button(QDialogButtonBox.Save).setEnabled(False) + + def create_tabs(self): + """Creates tabs based on the redesigned UI plan.""" + if not self.settings: + return + + # --- Create Tabs --- + self.tabs = { + "general": QWidget(), + "output_naming": QWidget(), + "image_processing": QWidget(), + "definitions": QWidget(), + "map_merging": QWidget(), + "postprocess_scripts": QWidget() + } + self.tab_widget.addTab(self.tabs["general"], "General") + self.tab_widget.addTab(self.tabs["output_naming"], "Output & Naming") + self.tab_widget.addTab(self.tabs["image_processing"], "Image Processing") + self.tab_widget.addTab(self.tabs["definitions"], "Definitions") + self.tab_widget.addTab(self.tabs["map_merging"], "Map Merging") + self.tab_widget.addTab(self.tabs["postprocess_scripts"], "Postprocess Scripts") + + + # --- Setup Layouts for Tabs --- + self.tab_layouts = {name: QVBoxLayout(tab) for name, tab in self.tabs.items()} + + # --- Populate Tabs --- + self.populate_general_tab(self.tab_layouts["general"]) + self.populate_output_naming_tab(self.tab_layouts["output_naming"]) + self.populate_image_processing_tab(self.tab_layouts["image_processing"]) + self.populate_definitions_tab(self.tab_layouts["definitions"]) + self.populate_map_merging_tab(self.tab_layouts["map_merging"]) + self.populate_postprocess_scripts_tab(self.tab_layouts["postprocess_scripts"]) + + def create_widget_for_setting(self, parent_layout, key, value, setting_key_prefix=""): + """Creates an appropriate widget for a single setting key-value pair.""" + full_key = f"{setting_key_prefix}{key}" if setting_key_prefix else key + label_text = key.replace('_', ' ').title() + label = QLabel(label_text + ":") + widget = None + layout_to_add = None # Use this for widgets needing extra controls (like browse button) + + if isinstance(value, str): + widget = QLineEdit(value) + elif isinstance(value, int): + widget = QSpinBox() + widget.setRange(-2147483648, 2147483647) + widget.setValue(value) + elif isinstance(value, float): + widget = QDoubleSpinBox() + widget.setRange(-1.7976931348623157e+308, 1.7976931348623157e+308) + widget.setValue(value) + elif isinstance(value, bool): + widget = QCheckBox() + widget.setChecked(value) + elif isinstance(value, list): # Handle simple lists as comma-separated strings + widget = QLineEdit(", ".join(map(str, value))) + # Complex dicts/lists like ASSET_TYPE_DEFINITIONS, MAP_MERGE_RULES etc. are handled in dedicated methods + + if widget: + parent_layout.addRow(label, widget) + self.widgets[full_key] = widget + else: + # Optionally handle unsupported types or log a warning + # print(f"Skipping widget creation for key '{full_key}' with unsupported type: {type(value)}") + pass + + def populate_general_tab(self, layout): + """Populates the General tab according to the plan.""" + # Clear existing widgets in the layout first + while layout.count(): + item = layout.takeAt(0) + widget = item.widget() + if widget: + widget.deleteLater() + sub_layout = item.layout() + if sub_layout: + # Basic clearing for sub-layouts like QHBoxLayout used below + while sub_layout.count(): + sub_item = sub_layout.takeAt(0) + sub_widget = sub_item.widget() + if sub_widget: + sub_widget.deleteLater() + + # Clear any potentially lingering widget references for this tab + self.widgets.pop("OUTPUT_BASE_DIR", None) + self.widgets.pop("EXTRA_FILES_SUBDIR", None) + self.widgets.pop("METADATA_FILENAME", None) + self.widgets.pop("TEMP_DIR_PREFIX", None) + + form_layout = QFormLayout() + + # 1. OUTPUT_BASE_DIR: QLineEdit + QPushButton + output_dir_label = QLabel("Output Base Directory:") + output_dir_edit = QLineEdit() + output_dir_button = QPushButton("Browse...") + # Ensure lambda captures the correct widget reference + output_dir_button.clicked.connect( + lambda checked=False, w=output_dir_edit: self.browse_path(w, "OUTPUT_BASE_DIR", is_dir=True) + ) + output_dir_layout = QHBoxLayout() + output_dir_layout.addWidget(output_dir_edit) + output_dir_layout.addWidget(output_dir_button) + form_layout.addRow(output_dir_label, output_dir_layout) + self.widgets["OUTPUT_BASE_DIR"] = output_dir_edit + + # 2. EXTRA_FILES_SUBDIR: QLineEdit + extra_subdir_label = QLabel("Subdirectory for Extra Files:") + extra_subdir_edit = QLineEdit() + form_layout.addRow(extra_subdir_label, extra_subdir_edit) + self.widgets["EXTRA_FILES_SUBDIR"] = extra_subdir_edit + + # 3. METADATA_FILENAME: QLineEdit + metadata_label = QLabel("Metadata Filename:") + metadata_edit = QLineEdit() + form_layout.addRow(metadata_label, metadata_edit) + self.widgets["METADATA_FILENAME"] = metadata_edit + + # 4. TEMP_DIR_PREFIX: QLineEdit + temp_dir_label = QLabel("Temporary Directory Prefix:") + temp_dir_edit = QLineEdit() + temp_dir_edit.setToolTip("Prefix for temporary directories created during processing.") + form_layout.addRow(temp_dir_label, temp_dir_edit) + self.widgets["TEMP_DIR_PREFIX"] = temp_dir_edit + + layout.addLayout(form_layout) + layout.addStretch() # Keep stretch at the end + + def populate_output_naming_tab(self, layout): + """Populates the Output & Naming tab according to the plan.""" + # Clear existing widgets in the layout first + while layout.count(): + item = layout.takeAt(0) + widget = item.widget() + if widget: + widget.deleteLater() + sub_layout = item.layout() + if sub_layout: + # Basic clearing for sub-layouts + while sub_layout.count(): + sub_item = sub_layout.takeAt(0) + sub_widget = sub_item.widget() + if sub_widget: + sub_widget.deleteLater() + sub_sub_layout = sub_item.layout() + if sub_sub_layout: # Clear nested layouts (like the button HBox) + while sub_sub_layout.count(): + ss_item = sub_sub_layout.takeAt(0) + ss_widget = ss_item.widget() + if ss_widget: + ss_widget.deleteLater() + + + # Clear potentially lingering widget references for this tab + self.widgets.pop("TARGET_FILENAME_PATTERN", None) + # self.widgets.pop("RESPECT_VARIANT_MAP_TYPES_LISTWIDGET", None) # This was an intermediate key, ensure it's gone + self.widgets.pop("RESPECT_VARIANT_MAP_TYPES", None) # This is the correct key for the QListWidget + self.widgets.pop("ASPECT_RATIO_DECIMALS", None) + self.widgets.pop("OUTPUT_DIRECTORY_PATTERN", None) + self.widgets.pop("OUTPUT_FILENAME_PATTERN", None) + + main_tab_layout = QVBoxLayout() + + form_layout = QFormLayout() + + # 1. TARGET_FILENAME_PATTERN: QLineEdit + target_filename_label = QLabel("Output Filename Pattern:") + target_filename_edit = QLineEdit() + target_filename_edit.setToolTip( + "Define the output filename structure.\n" + "Placeholders: {asset_name}, {map_type}, {resolution}, {variant}, {udim}" + ) + form_layout.addRow(target_filename_label, target_filename_edit) + self.widgets["TARGET_FILENAME_PATTERN"] = target_filename_edit + + # 2. RESPECT_VARIANT_MAP_TYPES: QListWidget + Add/Remove Buttons + respect_variant_label = QLabel("Map Types Respecting Variants:") + + self.respect_variant_list_widget = QListWidget() + self.respect_variant_list_widget.setToolTip("List of map types that should respect variant naming.") + self.widgets["RESPECT_VARIANT_MAP_TYPES"] = self.respect_variant_list_widget # Use the actual setting key + + respect_variant_buttons_layout = QHBoxLayout() + add_respect_variant_button = QPushButton("Add") + add_respect_variant_button.clicked.connect(self.add_respect_variant_map_type) + remove_respect_variant_button = QPushButton("Remove") + remove_respect_variant_button.clicked.connect(self.remove_respect_variant_map_type) + respect_variant_buttons_layout.addWidget(add_respect_variant_button) + respect_variant_buttons_layout.addWidget(remove_respect_variant_button) + respect_variant_buttons_layout.addStretch() + + respect_variant_layout = QVBoxLayout() + respect_variant_layout.addWidget(self.respect_variant_list_widget) + respect_variant_layout.addLayout(respect_variant_buttons_layout) + + form_layout.addRow(respect_variant_label, respect_variant_layout) + # self.widgets["RESPECT_VARIANT_MAP_TYPES"] will now refer to the list widget for population/saving logic + + # 3. ASPECT_RATIO_DECIMALS: QSpinBox + aspect_ratio_label = QLabel("Aspect Ratio Precision (Decimals):") + aspect_ratio_spinbox = QSpinBox() + aspect_ratio_spinbox.setRange(0, 6) # Min: 0, Max: ~6 + form_layout.addRow(aspect_ratio_label, aspect_ratio_spinbox) + self.widgets["ASPECT_RATIO_DECIMALS"] = aspect_ratio_spinbox + + # 4. OUTPUT_DIRECTORY_PATTERN: QLineEdit + output_dir_pattern_label = QLabel("Output Directory Pattern:") + output_dir_pattern_edit = QLineEdit() + output_dir_pattern_edit.setToolTip( + "Define the output subdirectory structure relative to Output Base Directory.\n" + "Placeholders: {supplier}, {asset_name}, {asset_category}, etc." + ) + form_layout.addRow(output_dir_pattern_label, output_dir_pattern_edit) + self.widgets["OUTPUT_DIRECTORY_PATTERN"] = output_dir_pattern_edit + + # 5. OUTPUT_FILENAME_PATTERN: QLineEdit (Note: app_settings.json has TARGET_FILENAME_PATTERN and OUTPUT_FILENAME_PATTERN) + # Assuming this is the one from app_settings.json line 9 + output_filename_pattern_label = QLabel("Output Filename Pattern (Legacy/Alternative):") + output_filename_pattern_edit = QLineEdit() + output_filename_pattern_edit.setToolTip( + "Alternative output filename structure if different from Target Filename Pattern.\n" + "Placeholders: {assetname}, {maptype}, {resolution}, {ext}, etc." + ) + form_layout.addRow(output_filename_pattern_label, output_filename_pattern_edit) + self.widgets["OUTPUT_FILENAME_PATTERN"] = output_filename_pattern_edit + + + main_tab_layout.addLayout(form_layout) + + layout.addLayout(main_tab_layout) + layout.addStretch() # Keep stretch at the end of the tab's main layout + + + def populate_image_processing_tab(self, layout): + """Populates the Image Processing tab according to the plan.""" + # Clear existing widgets in the layout first + while layout.count(): + item = layout.takeAt(0) + widget = item.widget() + if widget: + widget.deleteLater() + sub_layout = item.layout() + if sub_layout: + # Basic clearing for sub-layouts + while sub_layout.count(): + sub_item = sub_layout.takeAt(0) + sub_widget = sub_item.widget() + if sub_widget: + sub_widget.deleteLater() + sub_sub_layout = sub_item.layout() + if sub_sub_layout: # Clear nested layouts (like button HBox) + while sub_sub_layout.count(): + ss_item = sub_sub_layout.takeAt(0) + ss_widget = ss_item.widget() + if ss_widget: + ss_widget.deleteLater() + + # Clear potentially lingering widget references for this tab + keys_to_clear = [ + "IMAGE_RESOLUTIONS_TABLE", "CALCULATE_STATS_RESOLUTION", + "PNG_COMPRESSION_LEVEL", "JPG_QUALITY", + "RESOLUTION_THRESHOLD_FOR_JPG", "OUTPUT_FORMAT_8BIT", + "OUTPUT_FORMAT_16BIT_PRIMARY", "OUTPUT_FORMAT_16BIT_FALLBACK", + "general_settings.invert_normal_map_green_channel_globally", + "INITIAL_SCALING_MODE" + ] + for key in keys_to_clear: + self.widgets.pop(key, None) + + main_tab_layout = QVBoxLayout() + + # --- IMAGE_RESOLUTIONS Section --- + resolutions_layout = QVBoxLayout() + resolutions_label = QLabel("Defined Image Resolutions") + resolutions_layout.addWidget(resolutions_label) + + resolutions_table = QTableWidget() + resolutions_table.setColumnCount(2) + resolutions_table.setHorizontalHeaderLabels(["Name", "Resolution (px)"]) + resolutions_table.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred) # Adjust size policy + resolutions_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.Stretch) + resolutions_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.Interactive) # Allow user resize, or ResizeToContents + + # Set SpinBox delegate for "Resolution (px)" column + # Ensure self.resolution_delegate is initialized if not already + if not hasattr(self, 'resolution_delegate'): + self.resolution_delegate = SpinBoxDelegate(min_val=1, max_val=65536, parent=resolutions_table) # Max typical texture size + resolutions_table.setItemDelegateForColumn(1, self.resolution_delegate) + + resolutions_layout.addWidget(resolutions_table) + self.widgets["IMAGE_RESOLUTIONS_TABLE"] = resolutions_table + + resolutions_button_layout = QHBoxLayout() + add_res_button = QPushButton("Add Row") + remove_res_button = QPushButton("Remove Row") + + # Ensure methods exist before connecting + if hasattr(self, 'add_image_resolution_row') and hasattr(self, 'remove_image_resolution_row'): + add_res_button.clicked.connect(self.add_image_resolution_row) + remove_res_button.clicked.connect(self.remove_image_resolution_row) + else: + print("Warning: add_image_resolution_row or remove_image_resolution_row not found during connect.") + + resolutions_button_layout.addWidget(add_res_button) + resolutions_button_layout.addWidget(remove_res_button) + resolutions_button_layout.addStretch() # Push buttons left + resolutions_layout.addLayout(resolutions_button_layout) + + main_tab_layout.addLayout(resolutions_layout) + + # --- Form Layout for other settings --- + form_layout = QFormLayout() + + # CALCULATE_STATS_RESOLUTION: QComboBox + stats_res_label = QLabel("Resolution for Stats Calculation:") + stats_res_combo = QComboBox() + # Population deferred - will be populated from IMAGE_RESOLUTIONS_TABLE + form_layout.addRow(stats_res_label, stats_res_combo) + self.widgets["CALCULATE_STATS_RESOLUTION"] = stats_res_combo + + # PNG_COMPRESSION_LEVEL: QSpinBox + png_level_label = QLabel("PNG Compression Level:") + png_level_spinbox = QSpinBox() + png_level_spinbox.setRange(0, 9) + form_layout.addRow(png_level_label, png_level_spinbox) + self.widgets["PNG_COMPRESSION_LEVEL"] = png_level_spinbox + + # JPG_QUALITY: QSpinBox + jpg_quality_label = QLabel("JPG Quality:") + jpg_quality_spinbox = QSpinBox() + jpg_quality_spinbox.setRange(1, 100) + form_layout.addRow(jpg_quality_label, jpg_quality_spinbox) + self.widgets["JPG_QUALITY"] = jpg_quality_spinbox + + # RESOLUTION_THRESHOLD_FOR_JPG: QComboBox + jpg_threshold_label = QLabel("Use JPG Above Resolution:") + jpg_threshold_combo = QComboBox() + # Population deferred - will be populated from IMAGE_RESOLUTIONS_TABLE + "Never"/"Always" + form_layout.addRow(jpg_threshold_label, jpg_threshold_combo) + self.widgets["RESOLUTION_THRESHOLD_FOR_JPG"] = jpg_threshold_combo + + # OUTPUT_FORMAT_8BIT: QComboBox + format_8bit_label = QLabel("Output Format (8-bit):") + format_8bit_combo = QComboBox() + format_8bit_combo.addItems(["png", "jpg"]) + form_layout.addRow(format_8bit_label, format_8bit_combo) + self.widgets["OUTPUT_FORMAT_8BIT"] = format_8bit_combo + + # OUTPUT_FORMAT_16BIT_PRIMARY: QComboBox + format_16bit_primary_label = QLabel("Primary Output Format (16-bit+):") + format_16bit_primary_combo = QComboBox() + format_16bit_primary_combo.addItems(["png", "exr", "tif"]) + form_layout.addRow(format_16bit_primary_label, format_16bit_primary_combo) + self.widgets["OUTPUT_FORMAT_16BIT_PRIMARY"] = format_16bit_primary_combo + + # OUTPUT_FORMAT_16BIT_FALLBACK: QComboBox + format_16bit_fallback_label = QLabel("Fallback Output Format (16-bit+):") + format_16bit_fallback_combo = QComboBox() + format_16bit_fallback_combo.addItems(["png", "exr", "tif"]) + form_layout.addRow(format_16bit_fallback_label, format_16bit_fallback_combo) + self.widgets["OUTPUT_FORMAT_16BIT_FALLBACK"] = format_16bit_fallback_combo + + main_tab_layout.addLayout(form_layout) + + # Add general_settings.invert_normal_map_green_channel_globally QCheckBox + invert_normal_checkbox = QCheckBox("Invert Normal Map Green Channel Globally") + invert_normal_checkbox.setToolTip("Applies green channel inversion for normal maps project-wide.") + # Add to form_layout or main_tab_layout. Let's add to form_layout for consistency. + form_layout.addRow(invert_normal_checkbox) # Label can be omitted if checkbox text is descriptive + self.widgets["general_settings.invert_normal_map_green_channel_globally"] = invert_normal_checkbox + + # INITIAL_SCALING_MODE: QComboBox + initial_scaling_label = QLabel("Initial Scaling Mode:") + initial_scaling_combo = QComboBox() + initial_scaling_combo.addItems(["POT_DOWNSCALE", "POT_UPSCALE", "NONE", "ASPECT_PRESERVING_DOWNSCALE"]) # Add likely options + initial_scaling_combo.setToolTip("Determines how images are initially scaled if they are not power-of-two.") + form_layout.addRow(initial_scaling_label, initial_scaling_combo) + self.widgets["INITIAL_SCALING_MODE"] = initial_scaling_combo + + layout.addLayout(main_tab_layout) + layout.addStretch() # Keep stretch at the end of the tab's main layout + + def populate_definitions_tab(self, layout): + """Populates the Definitions tab according to the plan.""" + # Clear existing widgets in the layout first + while layout.count(): + item = layout.takeAt(0) + widget = item.widget() + if widget: + widget.deleteLater() + sub_layout = item.layout() + if sub_layout: + # Recursively clear sub-layouts + while sub_layout.count(): + sub_item = sub_layout.takeAt(0) + sub_widget = sub_item.widget() + if sub_widget: + sub_widget.deleteLater() + sub_sub_layout = sub_item.layout() + if sub_sub_layout: + # Clear nested layouts (like button HBox or inner tabs) + while sub_sub_layout.count(): + ss_item = sub_sub_layout.takeAt(0) + ss_widget = ss_item.widget() + if ss_widget: + ss_widget.deleteLater() + # Add more levels if necessary, but this covers the planned structure + + # Clear potentially lingering widget references for this tab + self.widgets.pop("DEFAULT_ASSET_CATEGORY", None) + self.widgets.pop("ASSET_TYPE_DEFINITIONS_TABLE", None) + self.widgets.pop("FILE_TYPE_DEFINITIONS_TABLE", None) + # Remove references to widgets no longer used in this tab's structure + self.widgets.pop("MAP_BIT_DEPTH_RULES_TABLE", None) + + + overall_layout = QVBoxLayout() + + # --- Top Widget: DEFAULT_ASSET_CATEGORY --- + default_category_layout = QHBoxLayout() # Use QHBox for label + combo + default_category_label = QLabel("Default Asset Category:") + default_category_combo = QComboBox() + # Population is deferred, will happen in populate_widgets_from_settings + default_category_layout.addWidget(default_category_label) + default_category_layout.addWidget(default_category_combo) + default_category_layout.addStretch() # Push label/combo left + overall_layout.addLayout(default_category_layout) + self.widgets["DEFAULT_ASSET_CATEGORY"] = default_category_combo + + # Inner QTabWidget and its contents (Asset Types and File Types tables) are removed + # as per Phase 1, Item 1 of the refactoring plan. + # The DEFAULT_ASSET_CATEGORY QComboBox remains above, part of overall_layout. + + layout.addLayout(overall_layout) + layout.addStretch() # Keep stretch at the end of the tab's main layout + + + def populate_map_merging_tab(self, layout): + """Populates the Map Merging tab according to the plan.""" + # Clear existing widgets in the layout first + while layout.count(): + item = layout.takeAt(0) + widget = item.widget() + if widget: + widget.deleteLater() + sub_layout = item.layout() + if sub_layout: + # Basic clearing for sub-layouts + while sub_layout.count(): + sub_item = sub_layout.takeAt(0) + sub_widget = sub_item.widget() + if sub_widget: + sub_widget.deleteLater() + # Clear nested layouts if needed (e.g., button layout) + sub_sub_layout = sub_item.layout() + if sub_sub_layout: + while sub_sub_layout.count(): + ss_item = sub_sub_layout.takeAt(0) + ss_widget = ss_item.widget() + if ss_widget: + ss_widget.deleteLater() + + # Clear potentially lingering widget references for this tab + self.widgets.pop("MAP_MERGE_RULES_DATA", None) + self.widgets.pop("MERGE_DIMENSION_MISMATCH_STRATEGY", None) + # Clear references to the list and details group if they exist + if hasattr(self, 'merge_rules_list'): + del self.merge_rules_list + if hasattr(self, 'merge_rule_details_group'): + del self.merge_rule_details_group + if hasattr(self, 'merge_rule_details_layout'): + del self.merge_rule_details_layout + if hasattr(self, 'merge_rule_widgets'): + del self.merge_rule_widgets + + + top_form_layout = QFormLayout() + + # MERGE_DIMENSION_MISMATCH_STRATEGY: QComboBox + merge_strategy_label = QLabel("Merge Dimension Mismatch Strategy:") + merge_strategy_combo = QComboBox() + merge_strategy_combo.addItems(["USE_LARGEST", "USE_SMALLEST", "ERROR_OUT"]) # Add likely options + merge_strategy_combo.setToolTip("How to handle merging maps of different dimensions.") + top_form_layout.addRow(merge_strategy_label, merge_strategy_combo) + self.widgets["MERGE_DIMENSION_MISMATCH_STRATEGY"] = merge_strategy_combo + + layout.addLayout(top_form_layout) # Add this form layout to the main tab layout + + # Layout: QHBoxLayout for rules list and details. + h_layout = QHBoxLayout() + layout.addLayout(h_layout) + + # Left Side: QListWidget displaying output_map_type for each rule. + left_layout = QVBoxLayout() + left_layout.addWidget(QLabel("Merge Rules:")) + self.merge_rules_list = QListWidget() + self.merge_rules_list.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding) # Allow list to expand vertically + self.merge_rules_list.currentItemChanged.connect(self.display_merge_rule_details) + left_layout.addWidget(self.merge_rules_list) + + button_layout = QHBoxLayout() + add_button = QPushButton("Add Rule") + remove_button = QPushButton("Remove Rule") + add_button.clicked.connect(self.add_merge_rule) + remove_button.clicked.connect(self.remove_merge_rule) + button_layout.addWidget(add_button) + button_layout.addWidget(remove_button) + left_layout.addLayout(button_layout) + + h_layout.addLayout(left_layout, 1) # Give list more space + + # Right Side: QStackedWidget or dynamically populated QWidget showing details + self.merge_rule_details_group = QGroupBox("Rule Details") + self.merge_rule_details_group.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred) # Allow groupbox to expand horizontally + self.merge_rule_details_layout = QFormLayout(self.merge_rule_details_group) + h_layout.addWidget(self.merge_rule_details_group, 2) # Give details form more space + + self.merge_rule_widgets = {} # Widgets for the currently displayed rule + + if "MAP_MERGE_RULES" in self.settings: + # Make a deep copy for local modification if needed, or manage through QListWidgetItems directly + self.current_map_merge_rules = copy.deepcopy(self.settings.get("MAP_MERGE_RULES", [])) + self.populate_merge_rules_list(self.current_map_merge_rules) + # self.widgets["MAP_MERGE_RULES_DATA"] = self.current_map_merge_rules # This will be the list of dicts + else: + self.current_map_merge_rules = [] + self.populate_merge_rules_list([]) # Populate with empty list + + layout.addStretch() + + + def populate_postprocess_scripts_tab(self, layout): + """Populates the Postprocess Scripts tab according to the plan.""" + # Clear existing widgets in the layout first + while layout.count(): + item = layout.takeAt(0) + widget = item.widget() + if widget: + widget.deleteLater() + sub_layout = item.layout() + if sub_layout: + # Basic clearing for sub-layouts (like the QHBoxLayouts used below) + while sub_layout.count(): + sub_item = sub_layout.takeAt(0) + sub_widget = sub_item.widget() + if sub_widget: + sub_widget.deleteLater() + + # Clear potentially lingering widget references for this tab + self.widgets.pop("DEFAULT_NODEGROUP_BLEND_PATH", None) + self.widgets.pop("DEFAULT_MATERIALS_BLEND_PATH", None) + self.widgets.pop("BLENDER_EXECUTABLE_PATH", None) + + form_layout = QFormLayout() + + # 1. DEFAULT_NODEGROUP_BLEND_PATH: QLineEdit + QPushButton + nodegroup_label = QLabel("Default Node Group Library (.blend):") + nodegroup_widget = QLineEdit() + nodegroup_button = QPushButton("Browse...") + nodegroup_button.clicked.connect( + lambda checked=False, w=nodegroup_widget: self.browse_path(w, "DEFAULT_NODEGROUP_BLEND_PATH") + ) + nodegroup_layout = QHBoxLayout() + nodegroup_layout.addWidget(nodegroup_widget) + nodegroup_layout.addWidget(nodegroup_button) + form_layout.addRow(nodegroup_label, nodegroup_layout) + self.widgets["DEFAULT_NODEGROUP_BLEND_PATH"] = nodegroup_widget + + # 2. DEFAULT_MATERIALS_BLEND_PATH: QLineEdit + QPushButton + materials_label = QLabel("Default Materials Library (.blend):") + materials_widget = QLineEdit() + materials_button = QPushButton("Browse...") + materials_button.clicked.connect( + lambda checked=False, w=materials_widget: self.browse_path(w, "DEFAULT_MATERIALS_BLEND_PATH") + ) + materials_layout = QHBoxLayout() + materials_layout.addWidget(materials_widget) + materials_layout.addWidget(materials_button) + form_layout.addRow(materials_label, materials_layout) + self.widgets["DEFAULT_MATERIALS_BLEND_PATH"] = materials_widget + + # 3. BLENDER_EXECUTABLE_PATH: QLineEdit + QPushButton + blender_label = QLabel("Blender Executable Path:") + blender_widget = QLineEdit() + blender_button = QPushButton("Browse...") + blender_button.clicked.connect( + lambda checked=False, w=blender_widget: self.browse_path(w, "BLENDER_EXECUTABLE_PATH") + ) + blender_layout = QHBoxLayout() + blender_layout.addWidget(blender_widget) + blender_layout.addWidget(blender_button) + form_layout.addRow(blender_label, blender_layout) + self.widgets["BLENDER_EXECUTABLE_PATH"] = blender_widget + + layout.addLayout(form_layout) + layout.addStretch() + + def create_asset_definitions_table_widget(self, layout, definitions_data): + """Creates a QTableWidget for editing asset type definitions.""" + table = QTableWidget() + # Columns: "Type Name", "Description", "Color", "Examples (comma-sep.)" + table.setColumnCount(4) + table.setHorizontalHeaderLabels(["Type Name", "Description", "Color", "Examples (comma-sep.)"]) + # Row count will be set when populating + + # TODO: Implement "Add Row" and "Remove Row" buttons + # TODO: Implement custom delegate for "Color" column (QPushButton) + # TODO: Implement custom delegate for "Examples" column (QLineEdit) + + layout.addWidget(table) + self.widgets["ASSET_TYPE_DEFINITIONS_TABLE"] = table + + def create_file_type_definitions_table_widget(self, layout, definitions_data): + """Creates a QTableWidget for editing file type definitions.""" + table = QTableWidget() + # Columns: "Type ID", "Description", "Color", "Examples (comma-sep.)", "Standard Type", "Bit Depth Rule" + table.setColumnCount(6) + table.setHorizontalHeaderLabels(["Type ID", "Description", "Color", "Examples (comma-sep.)", "Standard Type", "Bit Depth Rule"]) + # Row count will be set when populating + + # TODO: Implement "Add Row" and "Remove Row" buttons + # TODO: Implement custom delegate for "Color" column (QPushButton) + # TODO: Implement custom delegate for "Examples" column (QLineEdit) + # TODO: Implement custom delegate for "Standard Type" column (QComboBox) + # TODO: Implement custom delegate for "Bit Depth Rule" column (QComboBox) + + layout.addWidget(table) + self.widgets["FILE_TYPE_DEFINITIONS_TABLE"] = table + + def create_image_resolutions_table_widget(self, layout, resolutions_data): + """Creates a QTableWidget for editing image resolutions.""" + table = QTableWidget() + # Columns: "Name", "Resolution (px)" + table.setColumnCount(2) + table.setHorizontalHeaderLabels(["Name", "Resolution (px)"]) + # Row count will be set when populating + + # TODO: Implement "Add Row" and "Remove Row" buttons + # TODO: Implement custom delegate for "Resolution (px)" column (e.g., QLineEdit with validation or two SpinBoxes) + + layout.addWidget(table) + self.widgets["IMAGE_RESOLUTIONS_TABLE"] = table + + def create_map_bit_depth_rules_table_widget(self, layout, rules_data: dict): + """Creates a QTableWidget for editing map bit depth rules (Map Type -> Rule).""" + table = QTableWidget() + # Columns: "Map Type", "Rule (respect/force_8bit)" + table.setColumnCount(2) + table.setHorizontalHeaderLabels(["Map Type", "Rule (respect/force_8bit)"]) + # Row count will be set when populating + + # TODO: Implement "Add Row" and "Remove Row" buttons + # TODO: Implement custom delegate for "Rule" column (QComboBox) + + layout.addWidget(table) + self.widgets["MAP_BIT_DEPTH_RULES_TABLE"] = table + + + def create_map_merge_rules_widget(self, layout, rules_data): + """Creates the Map Merging UI (ListWidget + Details Form) according to the plan.""" + # This method is called by populate_map_merging_tab and sets up the QHBoxLayout, + # ListWidget, and details group box. The details population is handled by + # display_merge_rule_details. + pass # Structure is already set up in populate_map_merging_tab + + def populate_merge_rules_list(self, rules_data): + """Populates the list widget with map merge rules.""" + self.merge_rules_list.clear() + for rule in rules_data: + # Use output_map_type for the display text + item_text = rule.get("output_map_type", "Unnamed Rule") + item = QListWidgetItem(item_text) + item.setData(Qt.UserRole, rule) # Store the rule dictionary in the item + self.merge_rules_list.addItem(item) + + def display_merge_rule_details(self, current, previous): + """Displays details of the selected merge rule according to the plan.""" + # Clear previous widgets + for i in reversed(range(self.merge_rule_details_layout.count())): + widget_item = self.merge_rule_details_layout.itemAt(i) + if widget_item: + widget = widget_item.widget() + if widget: + widget.deleteLater() + layout = widget_item.layout() + if layout: + # Recursively delete widgets in layout + while layout.count(): + item = layout.takeAt(0) + widget = item.widget() + if widget: + widget.deleteLater() + elif item.layout(): + # Handle nested layouts if necessary + pass # For simplicity, assuming no deeply nested layouts here + + self.merge_rule_widgets.clear() + + if current: + rule_data = current.data(Qt.UserRole) + if rule_data: + # Rule Detail Form: + # output_map_type: QLineEdit. Label: "Output Map Type Name". + if "output_map_type" in rule_data: + label = QLabel("Output Map Type Name:") + combo_output_map_type = QComboBox() + file_type_keys = list(self.settings.get("FILE_TYPE_DEFINITIONS", {}).keys()) + if not file_type_keys: # Fallback if no keys found + file_type_keys = ["NEW_RULE", rule_data["output_map_type"]] # Add current value as an option + + # Ensure current value is in list, add if not (e.g. for "NEW_RULE") + if rule_data["output_map_type"] not in file_type_keys: + file_type_keys.insert(0, rule_data["output_map_type"]) + + combo_output_map_type.addItems(file_type_keys) + combo_output_map_type.setCurrentText(rule_data["output_map_type"]) + combo_output_map_type.currentIndexChanged.connect( + lambda index, cb=combo_output_map_type: self.update_rule_output_map_type(cb.currentText()) + ) + self.merge_rule_details_layout.addRow(label, combo_output_map_type) + self.merge_rule_widgets["output_map_type"] = combo_output_map_type + + # inputs: QTableWidget (Fixed Rows: R, G, B, A. Columns: "Channel", "Input Map Type"). Label: "Channel Inputs". + if "inputs" in rule_data and isinstance(rule_data["inputs"], dict): + group = QGroupBox("Channel Inputs") + group_layout = QVBoxLayout(group) + input_table = QTableWidget(4, 2) # R, G, B, A rows, 2 columns + input_table.setHorizontalHeaderLabels(["Channel", "Input Map Type"]) + input_table.setVerticalHeaderLabels(["R", "G", "B", "A"]) + + file_type_keys_with_none = [""] + list(self.settings.get("FILE_TYPE_DEFINITIONS", {}).keys()) + inputs_delegate = ComboBoxDelegate(items=file_type_keys_with_none, parent=input_table) + input_table.setItemDelegateForColumn(1, inputs_delegate) + + channels = ["R", "G", "B", "A"] + for i, channel_key in enumerate(channels): + input_map_type = rule_data["inputs"].get(channel_key, "") + channel_item = QTableWidgetItem(channel_key) + channel_item.setFlags(channel_item.flags() & ~Qt.ItemIsEditable) # Make channel name not editable + input_table.setItem(i, 0, channel_item) + + map_type_item = QTableWidgetItem(input_map_type) + input_table.setItem(i, 1, map_type_item) + + input_table.itemChanged.connect(lambda item, table=input_table, data_key="inputs": self.update_rule_data_from_table(item, table, data_key)) + group_layout.addWidget(input_table) + self.merge_rule_details_layout.addRow(group) + self.merge_rule_widgets["inputs_table"] = input_table + + + # defaults: QTableWidget (Fixed Rows: R, G, B, A. Columns: "Channel", "Default Value"). Label: "Channel Defaults (if input missing)". + if "defaults" in rule_data and isinstance(rule_data["defaults"], dict): + group = QGroupBox("Channel Defaults (if input missing)") + group_layout = QVBoxLayout(group) + defaults_table = QTableWidget(4, 2) # R, G, B, A rows, 2 columns + defaults_table.setHorizontalHeaderLabels(["Channel", "Default Value"]) + defaults_table.setVerticalHeaderLabels(["R", "G", "B", "A"]) + + defaults_delegate = DoubleSpinBoxDelegate(min_val=0.0, max_val=1.0, decimals=3, step=0.01, parent=defaults_table) # Example range + defaults_table.setItemDelegateForColumn(1, defaults_delegate) + + channels = ["R", "G", "B", "A"] + for i, channel_key in enumerate(channels): + default_value = rule_data["defaults"].get(channel_key, 0.0 if channel_key != "A" else 1.0) # A defaults to 1.0 + channel_item = QTableWidgetItem(channel_key) + channel_item.setFlags(channel_item.flags() & ~Qt.ItemIsEditable) # Make channel name not editable + defaults_table.setItem(i, 0, channel_item) + + value_item = QTableWidgetItem(str(default_value)) + defaults_table.setItem(i, 1, value_item) + + defaults_table.itemChanged.connect(lambda item, table=defaults_table, data_key="defaults": self.update_rule_data_from_table(item, table, data_key)) + group_layout.addWidget(defaults_table) + self.merge_rule_details_layout.addRow(group) + self.merge_rule_widgets["defaults_table"] = defaults_table + + + # output_bit_depth: QComboBox (Options: "respect_inputs", "force_8bit", "force_16bit"). Label: "Output Bit Depth". + if "output_bit_depth" in rule_data: + label = QLabel("Output Bit Depth:") + widget = QComboBox() + options = ["respect_inputs", "force_8bit", "force_16bit"] + widget.addItems(options) + if rule_data["output_bit_depth"] in options: + widget.setCurrentText(rule_data["output_bit_depth"]) + self.merge_rule_details_layout.addRow(label, widget) + self.merge_rule_widgets["output_bit_depth"] = widget + + # Add stretch to push widgets to the top + self.merge_rule_details_layout.addStretch() + + + # Connect output_bit_depth QComboBox to update rule data + if "output_bit_depth" in self.merge_rule_widgets and isinstance(self.merge_rule_widgets["output_bit_depth"], QComboBox): + self.merge_rule_widgets["output_bit_depth"].currentTextChanged.connect( + lambda text, key="output_bit_depth": self.update_rule_data_simple_field(text, key) + ) + + + def update_rule_output_map_type(self, new_text): + """Updates the output_map_type in the rule data and QListWidgetItem text.""" + current_list_item = self.merge_rules_list.currentItem() + if current_list_item: + rule_data = current_list_item.data(Qt.UserRole) + if rule_data and isinstance(rule_data, dict): + rule_data["output_map_type"] = new_text + current_list_item.setData(Qt.UserRole, rule_data) # Update the stored data + current_list_item.setText(new_text) # Update the display text in the list + + def update_rule_data_from_table(self, item: QTableWidgetItem, table_widget: QTableWidget, data_key: str): + """Updates the rule data when a table item changes (for inputs or defaults).""" + current_list_item = self.merge_rules_list.currentItem() + if not current_list_item: + return + + rule_data = current_list_item.data(Qt.UserRole) + if not rule_data or not isinstance(rule_data, dict): + return + + row = item.row() + col = item.column() + + if col == 1: # Only update for the value column (Input Map Type or Default Value) + channel_key_item = table_widget.verticalHeaderItem(row) + if not channel_key_item: # Should have vertical headers R,G,B,A + channel_key_item = table_widget.item(row, 0) # Fallback if no vertical header + + if channel_key_item: + channel_key = channel_key_item.text() + new_value = item.text() + + if data_key == "inputs": + if "inputs" not in rule_data or not isinstance(rule_data["inputs"], dict): + rule_data["inputs"] = {} + rule_data["inputs"][channel_key] = new_value + elif data_key == "defaults": + if "defaults" not in rule_data or not isinstance(rule_data["defaults"], dict): + rule_data["defaults"] = {} + try: + rule_data["defaults"][channel_key] = float(new_value) + except ValueError: + # Handle error or revert, for now, just print + print(f"Invalid float value for default: {new_value}") + # Optionally revert item text: item.setText(str(rule_data["defaults"].get(channel_key, 0.0))) + return + + current_list_item.setData(Qt.UserRole, rule_data) # Update the stored data + # print(f"Updated rule data for {channel_key} in {data_key}: {new_value}") # Debug + + def update_rule_data_simple_field(self, new_value, field_key): + """Updates a simple field in the rule data (e.g., output_bit_depth).""" + current_list_item = self.merge_rules_list.currentItem() + if current_list_item: + rule_data = current_list_item.data(Qt.UserRole) + if rule_data and isinstance(rule_data, dict): + rule_data[field_key] = new_value + current_list_item.setData(Qt.UserRole, rule_data) # Update the stored data + # print(f"Updated rule field {field_key} to: {new_value}") # Debug + + + def browse_path(self, widget, key, is_dir=False): + """Opens a file or directory dialog based on the setting key and is_dir flag.""" + if is_dir: + path = QFileDialog.getExistingDirectory(self, "Select Directory", widget.text()) + elif 'BLEND_PATH' in key.upper(): + path, _ = QFileDialog.getOpenFileName(self, "Select File", widget.text(), "Blender Files (*.blend)") + else: + path, _ = QFileDialog.getOpenFileName(self, "Select File", widget.text()) + + if path: + widget.setText(path) + + def add_respect_variant_map_type(self): + """Adds a map type to the RESPECT_VARIANT_MAP_TYPES list.""" + # Ensure configuration and file_type_definitions are loaded + if not hasattr(self, 'settings') or "FILE_TYPE_DEFINITIONS" not in self.settings: + QMessageBox.warning(self, "Configuration Error", "File type definitions are not loaded.") + return + + file_type_definitions = self.settings.get("FILE_TYPE_DEFINITIONS", {}) + map_type_keys = list(file_type_definitions.keys()) + if not map_type_keys: + QMessageBox.warning(self, "No Map Types", "No map types available to add.") + return + + item, ok = QInputDialog.getItem(self, "Add Map Type", + "Select map type to add:", map_type_keys, 0, False) + if ok and item: + # Check if item already exists + for i in range(self.respect_variant_list_widget.count()): + if self.respect_variant_list_widget.item(i).text() == item: + QMessageBox.information(self, "Duplicate", f"Map type '{item}' is already in the list.") + return + self.respect_variant_list_widget.addItem(item) + + def remove_respect_variant_map_type(self): + """Removes the selected map type from the RESPECT_VARIANT_MAP_TYPES list.""" + selected_items = self.respect_variant_list_widget.selectedItems() + if not selected_items: + QMessageBox.warning(self, "No Selection", "Please select a map type to remove.") + return + for item in selected_items: + self.respect_variant_list_widget.takeItem(self.respect_variant_list_widget.row(item)) + + def add_merge_rule(self): + """Adds a new default map merge rule.""" + new_rule = { + "output_map_type": "NEW_RULE", + "inputs": {"R": "", "G": "", "B": "", "A": ""}, + "defaults": {"R": 0.0, "G": 0.0, "B": 0.0, "A": 1.0}, + "output_bit_depth": "respect_inputs" + } + + # Add to the internal list that backs the UI + # self.current_map_merge_rules.append(new_rule) # This list is now managed by QListWidgetItems + + item_text = new_rule.get("output_map_type", "Unnamed Rule") + item = QListWidgetItem(item_text) + item.setData(Qt.UserRole, copy.deepcopy(new_rule)) # Store a mutable copy for this item + self.merge_rules_list.addItem(item) + self.merge_rules_list.setCurrentItem(item) # Select the new item to display its details + + def remove_merge_rule(self): + """Removes the currently selected map merge rule.""" + current_item = self.merge_rules_list.currentItem() + if not current_item: + QMessageBox.warning(self, "No Selection", "Please select a rule to remove.") + return + + # No need to manage self.current_map_merge_rules separately if Qt.UserRole is the source of truth + # rule_to_remove = current_item.data(Qt.UserRole) + # if rule_to_remove in self.current_map_merge_rules: + # self.current_map_merge_rules.remove(rule_to_remove) + + row = self.merge_rules_list.row(current_item) + self.merge_rules_list.takeItem(row) + + # Clear details panel or select next/previous + if self.merge_rules_list.count() > 0: + self.merge_rules_list.setCurrentRow(max(0, row -1)) # Select previous or first + else: + self.display_merge_rule_details(None, None) # Clear details if list is empty + + # Ensure this method is defined within the class ConfigEditorDialog + def add_image_resolution_row(self): + """Adds a new row to the IMAGE_RESOLUTIONS table after prompting the user.""" + table = self.widgets.get("IMAGE_RESOLUTIONS_TABLE") + if not table: + return + + name, ok_name = QInputDialog.getText(self, "Add Resolution", "Enter Name (e.g., 16K):") + if not ok_name or not name.strip(): + if ok_name and not name.strip(): # User pressed OK but entered empty name + QMessageBox.warning(self, "Invalid Input", "Name cannot be empty.") + return # User cancelled or entered empty name + + # Check for duplicate name + for r in range(table.rowCount()): + if table.item(r, 0) and table.item(r, 0).text() == name: + QMessageBox.warning(self, "Duplicate Name", f"The resolution name '{name}' already exists.") + return + + resolution, ok_res = QInputDialog.getInt(self, "Add Resolution", "Enter Resolution (px):", 1024, 1, 65536, 1) + if not ok_res: + return # User cancelled + + row_position = table.rowCount() + table.insertRow(row_position) + table.setItem(row_position, 0, QTableWidgetItem(name)) + table.setItem(row_position, 1, QTableWidgetItem(str(resolution))) + + self._update_resolution_dependent_combos() + + def remove_image_resolution_row(self): + """Removes the selected row(s) from the IMAGE_RESOLUTIONS table.""" + table = self.widgets.get("IMAGE_RESOLUTIONS_TABLE") + if not table: + return + + selected_rows = sorted(list(set(index.row() for index in table.selectedIndexes())), reverse=True) + if not selected_rows: + QMessageBox.warning(self, "No Selection", "Please select row(s) to remove.") + return + + for row in selected_rows: + table.removeRow(row) + + self._update_resolution_dependent_combos() + + def _update_resolution_dependent_combos(self): + """Updates ComboBoxes that depend on IMAGE_RESOLUTIONS.""" + table = self.widgets.get("IMAGE_RESOLUTIONS_TABLE") + stats_combo = self.widgets.get("CALCULATE_STATS_RESOLUTION") + jpg_threshold_combo = self.widgets.get("RESOLUTION_THRESHOLD_FOR_JPG") + + if not table or (not stats_combo and not jpg_threshold_combo): + return + + current_stats_selection = stats_combo.currentText() if stats_combo else None + current_jpg_threshold_selection = jpg_threshold_combo.currentText() if jpg_threshold_combo else None + + resolution_names = [] + for r in range(table.rowCount()): + name_item = table.item(r, 0) + if name_item and name_item.text(): + resolution_names.append(name_item.text()) + + if stats_combo: + stats_combo.clear() + stats_combo.addItems(resolution_names) + if current_stats_selection in resolution_names: + stats_combo.setCurrentText(current_stats_selection) + elif resolution_names: # Select first item if previous selection is gone + stats_combo.setCurrentIndex(0) + + if jpg_threshold_combo: + jpg_threshold_combo.clear() + jpg_threshold_options = ["Never", "Always"] + resolution_names + jpg_threshold_combo.addItems(jpg_threshold_options) + if current_jpg_threshold_selection in jpg_threshold_options: + jpg_threshold_combo.setCurrentText(current_jpg_threshold_selection) + elif jpg_threshold_options: # Select first item if previous selection is gone + jpg_threshold_combo.setCurrentIndex(0) + + + def pick_color(self, widget): + """Opens a color dialog and sets the selected color in the widget.""" + color = QColorDialog.getColor(QColor(widget.text())) + if color.isValid(): + widget.setText(color.name()) # Get color as hex string + + def save_settings(self): + """ + Reads values from widgets, compares them to the original loaded settings, + and saves only the changed values to config/user_settings.json, preserving + other existing user settings. + """ + # 1a. Load Current Target File (user_settings.json) + user_settings_path = os.path.join("config", "user_settings.json") + + target_file_content = {} + if os.path.exists(user_settings_path): + try: + with open(user_settings_path, 'r') as f: + target_file_content = json.load(f) + except json.JSONDecodeError: + QMessageBox.warning(self, "Warning", + f"File {user_settings_path} is corrupted or not valid JSON. " + f"It will be overwritten if changes are saved.") + target_file_content = {} # Start fresh if corrupted + except Exception as e: + QMessageBox.critical(self, "Error Loading User Settings", + f"Failed to load {user_settings_path}: {e}. " + f"Proceeding with empty user settings for this save operation.") + target_file_content = {} + + # 1b. Get current settings from UI by populating a full settings dictionary + full_ui_state = copy.deepcopy(self.settings) # Start with the loaded settings structure + + # --- Populate full_ui_state from ALL widgets --- + for widget_config_key, widget_obj in self.widgets.items(): + keys_path = widget_config_key.split('.') + current_level_dict = full_ui_state + for i, part_of_key in enumerate(keys_path): + if i == len(keys_path) - 1: + if isinstance(widget_obj, QLineEdit): + current_level_dict[part_of_key] = widget_obj.text() + elif isinstance(widget_obj, QSpinBox): + current_level_dict[part_of_key] = widget_obj.value() + elif isinstance(widget_obj, QDoubleSpinBox): + current_level_dict[part_of_key] = widget_obj.value() + elif isinstance(widget_obj, QCheckBox): + if widget_config_key == "general_settings.invert_normal_map_green_channel_globally": + if 'general_settings' not in full_ui_state: + full_ui_state['general_settings'] = {} + full_ui_state['general_settings']['invert_normal_map_green_channel_globally'] = widget_obj.isChecked() + else: + current_level_dict[part_of_key] = widget_obj.isChecked() + elif isinstance(widget_obj, QListWidget) and widget_config_key == "RESPECT_VARIANT_MAP_TYPES": + items = [widget_obj.item(i_item).text() for i_item in range(widget_obj.count())] + current_level_dict[part_of_key] = items + elif isinstance(widget_obj, QComboBox): + if widget_config_key == "RESOLUTION_THRESHOLD_FOR_JPG": + selected_text = widget_obj.currentText() + image_resolutions_data = full_ui_state.get('IMAGE_RESOLUTIONS', {}) + if selected_text == "Never": current_level_dict[part_of_key] = 999999 + elif selected_text == "Always": current_level_dict[part_of_key] = 1 + elif isinstance(image_resolutions_data, dict) and selected_text in image_resolutions_data: + current_level_dict[part_of_key] = image_resolutions_data[selected_text] + else: current_level_dict[part_of_key] = selected_text # Fallback + else: + current_level_dict[part_of_key] = widget_obj.currentText() + elif widget_config_key == "IMAGE_RESOLUTIONS_TABLE" and isinstance(widget_obj, QTableWidget): + table = widget_obj + resolutions_dict = {} + for row in range(table.rowCount()): + name_item = table.item(row, 0) + res_item = table.item(row, 1) + if name_item and name_item.text() and res_item and res_item.text(): + name = name_item.text() + try: + resolutions_dict[name] = int(res_item.text()) + except ValueError: + print(f"Warning: Resolution value '{res_item.text()}' for '{name}' is not an integer. Skipping.") + full_ui_state['IMAGE_RESOLUTIONS'] = resolutions_dict + else: + if part_of_key not in current_level_dict or not isinstance(current_level_dict[part_of_key], dict): + current_level_dict[part_of_key] = {} + current_level_dict = current_level_dict[part_of_key] + + # Special handling for MAP_MERGE_RULES - build from QListWidget items + if hasattr(self, 'merge_rules_list'): + updated_merge_rules = [] + for i in range(self.merge_rules_list.count()): + item = self.merge_rules_list.item(i) + rule_data = item.data(Qt.UserRole) + if rule_data: + updated_merge_rules.append(copy.deepcopy(rule_data)) # Add a copy to avoid issues if UserRole is reused + full_ui_state['MAP_MERGE_RULES'] = updated_merge_rules + + # --- End of populating full_ui_state --- + + # 2. Identify Changes by comparing with self.original_user_configurable_settings + changed_settings_count = 0 + for key_to_check, original_value in self.original_user_configurable_settings.items(): + current_value_from_ui = full_ui_state.get(key_to_check) + if current_value_from_ui != original_value: + target_file_content[key_to_check] = copy.deepcopy(current_value_from_ui) + changed_settings_count += 1 + print(f"Setting '{key_to_check}' changed. Old: {original_value}, New: {current_value_from_ui}") + + # 3. Save Updated Content to user_settings.json + if changed_settings_count > 0 or not os.path.exists(user_settings_path): + try: + save_user_config(target_file_content) + QMessageBox.information(self, "Settings Saved", + f"User settings saved successfully to {user_settings_path}.\n" + f"{changed_settings_count} setting(s) updated. " + "Some changes may require an application restart.") + self.accept() + except ConfigurationError as e: + QMessageBox.critical(self, "Saving Error", f"Failed to save user configuration: {e}") + except Exception as e: + QMessageBox.critical(self, "Saving Error", f"An unexpected error occurred while saving: {e}") + else: + QMessageBox.information(self, "No Changes", "No changes were made to user-configurable settings.") + self.accept() + + def populate_widgets_from_settings(self): + """Populates the created widgets with loaded settings.""" + if not self.settings or not self.widgets: + return + + for key, value in self.settings.items(): + # Handle simple settings directly if they have a corresponding widget + if key == "general_settings": # Handle nested dictionary + if isinstance(value, dict): + for sub_key, sub_value in value.items(): + widget_full_key = f"{key}.{sub_key}" + if widget_full_key in self.widgets: + widget = self.widgets[widget_full_key] + if isinstance(widget, QCheckBox) and isinstance(sub_value, bool): + widget.setChecked(sub_value) + elif key in self.widgets and isinstance(self.widgets[key], (QLineEdit, QSpinBox, QDoubleSpinBox, QCheckBox, QComboBox, QListWidget)): # Added QListWidget + widget = self.widgets[key] + if key == "RESPECT_VARIANT_MAP_TYPES" and isinstance(widget, QListWidget): + widget.clear() + if isinstance(value, list): + for item_text in value: # value is the list of strings from settings + widget.addItem(str(item_text)) + elif isinstance(widget, QLineEdit): + # This case should not be hit for RESPECT_VARIANT_MAP_TYPES anymore + if isinstance(value, (str, int, float, bool)): + widget.setText(str(value)) + elif isinstance(widget, QSpinBox) and isinstance(value, int): + widget.setValue(value) + elif isinstance(widget, QDoubleSpinBox) and isinstance(value, (int, float)): + widget.setValue(float(value)) + elif isinstance(widget, QCheckBox) and isinstance(value, bool): + widget.setChecked(value) + elif isinstance(widget, QComboBox): + if value in [widget.itemText(i) for i in range(widget.count())]: + widget.setCurrentText(value) + + + # Handle complex structures with dedicated widgets (Tables and Lists) + elif key == "ASSET_TYPE_DEFINITIONS" and "ASSET_TYPE_DEFINITIONS_TABLE" in self.widgets: + self.populate_asset_definitions_table(self.widgets["ASSET_TYPE_DEFINITIONS_TABLE"], value) + elif key == "FILE_TYPE_DEFINITIONS" and "FILE_TYPE_DEFINITIONS_TABLE" in self.widgets: + self.populate_file_type_definitions_table(self.widgets["FILE_TYPE_DEFINITIONS_TABLE"], value) + elif key == "IMAGE_RESOLUTIONS" and "IMAGE_RESOLUTIONS_TABLE" in self.widgets: + self.populate_image_resolutions_table(self.widgets["IMAGE_RESOLUTIONS_TABLE"], value) + # Populate ComboBoxes that depend on Image Resolutions - now handled by _update_resolution_dependent_combos + # Call it here to ensure initial population is correct after table is filled. + self._update_resolution_dependent_combos() + # Restore original selection if possible + if "CALCULATE_STATS_RESOLUTION" in self.settings and self.widgets.get("CALCULATE_STATS_RESOLUTION"): + if self.settings["CALCULATE_STATS_RESOLUTION"] in [self.widgets["CALCULATE_STATS_RESOLUTION"].itemText(i) for i in range(self.widgets["CALCULATE_STATS_RESOLUTION"].count())]: + self.widgets["CALCULATE_STATS_RESOLUTION"].setCurrentText(self.settings["CALCULATE_STATS_RESOLUTION"]) + + if "RESOLUTION_THRESHOLD_FOR_JPG" in self.settings and self.widgets.get("RESOLUTION_THRESHOLD_FOR_JPG"): + # Map stored integer value back to text for selection + stored_jpg_threshold_val = self.settings["RESOLUTION_THRESHOLD_FOR_JPG"] + current_text_selection = None + if isinstance(stored_jpg_threshold_val, int): + if stored_jpg_threshold_val == 999999: current_text_selection = "Never" + elif stored_jpg_threshold_val == 1: current_text_selection = "Always" + else: # Try to find by value in the resolutions + res_table = self.widgets["IMAGE_RESOLUTIONS_TABLE"] + for r_idx in range(res_table.rowCount()): + if res_table.item(r_idx, 1) and int(res_table.item(r_idx, 1).text()) == stored_jpg_threshold_val: + current_text_selection = res_table.item(r_idx, 0).text() + break + elif isinstance(stored_jpg_threshold_val, str): # If it was already a name + current_text_selection = stored_jpg_threshold_val + + if current_text_selection and current_text_selection in [self.widgets["RESOLUTION_THRESHOLD_FOR_JPG"].itemText(i) for i in range(self.widgets["RESOLUTION_THRESHOLD_FOR_JPG"].count())]: + self.widgets["RESOLUTION_THRESHOLD_FOR_JPG"].setCurrentText(current_text_selection) + + + elif key == "MAP_BIT_DEPTH_RULES" and "MAP_BIT_DEPTH_RULES_TABLE" in self.widgets: + self.populate_map_bit_depth_rules_table(self.widgets["MAP_BIT_DEPTH_RULES_TABLE"], value) + + + elif key == "MAP_MERGE_RULES" and hasattr(self, 'merge_rules_list'): # Check if the list widget exists + self.populate_merge_rules_list(value) + # Select the first item to display details if the list is not empty + if self.merge_rules_list.count() > 0: + self.merge_rules_list.setCurrentRow(0) + + + def populate_asset_definitions_table(self, table: QTableWidget, definitions_data: dict): + """Populates the asset definitions table.""" + table.setRowCount(len(definitions_data)) + row = 0 + for asset_type, details in definitions_data.items(): + item_type_name = QTableWidgetItem(asset_type) + item_description = QTableWidgetItem(details.get("description", "")) + table.setItem(row, 0, item_type_name) + table.setItem(row, 1, item_description) + + # Color column - Set item with color string as data + color_str = details.get("color", "#ffffff") # Default to white if missing + item_color = QTableWidgetItem() # No text needed, delegate handles paint + item_color.setData(Qt.EditRole, color_str) # Store hex string for delegate/editing + # item_color.setBackground(QColor(color_str)) # Optional: Set initial background via item + table.setItem(row, 2, item_color) + + # Examples column + examples_list = details.get("examples", []) + examples_str = ", ".join(examples_list) if isinstance(examples_list, list) else "" + item_examples = QTableWidgetItem(examples_str) + table.setItem(row, 3, item_examples) + + # Background color is now handled by the delegate's paint method based on data + + row += 1 + + # After populating the Asset Types table, populate the DEFAULT_ASSET_CATEGORY ComboBox + if "DEFAULT_ASSET_CATEGORY" in self.widgets and isinstance(self.widgets["DEFAULT_ASSET_CATEGORY"], QComboBox): + asset_types = list(definitions_data.keys()) + self.widgets["DEFAULT_ASSET_CATEGORY"].addItems(asset_types) + # Set the current value if it exists in settings + if "DEFAULT_ASSET_CATEGORY" in self.settings and self.settings["DEFAULT_ASSET_CATEGORY"] in asset_types: + self.widgets["DEFAULT_ASSET_CATEGORY"].setCurrentText(self.settings["DEFAULT_ASSET_CATEGORY"]) + + + def populate_file_type_definitions_table(self, table: QTableWidget, definitions_data: dict): + """Populates the file type definitions table.""" + table.setRowCount(len(definitions_data)) + row = 0 + for file_type, details in definitions_data.items(): + item_type_id = QTableWidgetItem(file_type) + item_description = QTableWidgetItem(details.get("description", "")) + table.setItem(row, 0, item_type_id) + table.setItem(row, 1, item_description) + + # Color column - Set item with color string as data + color_str = details.get("color", "#ffffff") # Default to white if missing + item_color = QTableWidgetItem() # No text needed, delegate handles paint + item_color.setData(Qt.EditRole, color_str) # Store hex string for delegate/editing + # item_color.setBackground(QColor(color_str)) # Optional: Set initial background via item + table.setItem(row, 2, item_color) + + # Examples column + examples_list = details.get("examples", []) + examples_str = ", ".join(examples_list) if isinstance(examples_list, list) else "" + item_examples = QTableWidgetItem(examples_str) + table.setItem(row, 3, item_examples) + + # Standard Type column (simple QTableWidgetItem for now) + standard_type_str = details.get("standard_type", "") + item_standard_type = QTableWidgetItem(standard_type_str) + table.setItem(row, 4, item_standard_type) + + # Bit Depth Rule column (simple QTableWidgetItem for now) + bit_depth_rule_str = details.get("bit_depth_rule", "") + item_bit_depth_rule = QTableWidgetItem(bit_depth_rule_str) + table.setItem(row, 5, item_bit_depth_rule) + + # Background color is now handled by the delegate's paint method based on data + + row += 1 + + def populate_image_resolutions_table(self, table: QTableWidget, resolutions_data: dict): + """Populates the image resolutions table from a dictionary.""" + table.setRowCount(0) # Clear existing rows before populating + table.setRowCount(len(resolutions_data)) + row = 0 + for name, resolution_value in resolutions_data.items(): + try: + name_item = QTableWidgetItem(str(name)) + res_item = QTableWidgetItem(str(resolution_value)) + + # Make items editable for Phase 1 (actual editing will be improved in Phase 3) + name_item.setFlags(name_item.flags() | Qt.ItemIsEditable) + res_item.setFlags(res_item.flags() | Qt.ItemIsEditable) + + table.setItem(row, 0, name_item) + table.setItem(row, 1, res_item) + except Exception as e: + print(f"Error populating resolution row for '{name}': {e}") + # Optionally add a row indicating error + table.setItem(row, 0, QTableWidgetItem(str(name))) + table.setItem(row, 1, QTableWidgetItem(f"Error: {e}")) + row += 1 + + + def populate_map_bit_depth_rules_table(self, table: QTableWidget, rules_data: dict): + """Populates the map bit depth rules table.""" + table.setRowCount(len(rules_data)) + row = 0 + for map_type, rule in rules_data.items(): + table.setItem(row, 0, QTableWidgetItem(map_type)) + table.setItem(row, 1, QTableWidgetItem(str(rule))) # Rule (respect/force_8bit) + row += 1 + + + + +# Example usage (for testing the dialog independently) +if __name__ == '__main__': + # Use PySide6 instead of PyQt5 for consistency + from PySide6.QtWidgets import QApplication + import sys + + app = QApplication(sys.argv) + dialog = ConfigEditorDialog() + dialog.exec() # Use exec() for PySide6 QDialog + sys.exit(app.exec()) # Use exec() for PySide6 QApplication \ No newline at end of file diff --git a/gui/definitions_editor_dialog.py b/gui/definitions_editor_dialog.py new file mode 100644 index 0000000..26a1403 --- /dev/null +++ b/gui/definitions_editor_dialog.py @@ -0,0 +1,1288 @@ +import logging +from PySide6.QtWidgets import ( + QDialog, QVBoxLayout, QTabWidget, QWidget, QListWidget, QListWidgetItem, QPushButton, + QHBoxLayout, QLabel, QGroupBox, QDialogButtonBox, QFormLayout, + QTextEdit, QColorDialog, QInputDialog, QMessageBox, QFrame, QComboBox, + QLineEdit, QCheckBox, QAbstractItemView +) +from PySide6.QtGui import QColor, QPalette, QMouseEvent # Added QMouseEvent +from PySide6.QtCore import Qt, QEvent + +# Assuming load_asset_definitions, load_file_type_definitions, load_supplier_settings +# are in configuration.py at the root level. +# Adjust the import path if configuration.py is located elsewhere relative to this file. +# For example, if configuration.py is in the parent directory: +# from ..configuration import load_asset_definitions, load_file_type_definitions, load_supplier_settings +# Or if it's in the same directory (less likely for a root config file): +# from .configuration import ... +# Given the project structure, configuration.py is at the root. +import sys +import os +# Add project root to sys.path to allow direct import of configuration +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +try: + from configuration import ( + load_asset_definitions, save_asset_definitions, + load_file_type_definitions, save_file_type_definitions, + load_supplier_settings, save_supplier_settings + ) +except ImportError as e: + logging.error(f"Failed to import configuration functions: {e}. Ensure configuration.py is in the project root and accessible.") + # Provide dummy functions if import fails, so the UI can still be tested somewhat + def load_asset_definitions(): return {} + def save_asset_definitions(data): pass + def load_file_type_definitions(): return {} + def save_file_type_definitions(data): pass + def load_supplier_settings(): return {} + # def save_supplier_settings(data): pass + +logger = logging.getLogger(__name__) + +class DebugListWidget(QListWidget): + def mousePressEvent(self, event: QMouseEvent): # QMouseEvent needs to be imported from PySide6.QtGui + logger.info(f"DebugListWidget.mousePressEvent: pos={event.pos()}") + item = self.itemAt(event.pos()) + if item: + logger.info(f"DebugListWidget.mousePressEvent: Item under cursor: {item.text()}") + else: + logger.info("DebugListWidget.mousePressEvent: No item under cursor.") + super().mousePressEvent(event) + logger.info("DebugListWidget.mousePressEvent: super call finished.") + +class DefinitionsEditorDialog(QDialog): + def __init__(self, parent=None): + super().__init__(parent) + self.setWindowTitle("Definitions Editor") + self.setGeometry(200, 200, 800, 600) # x, y, width, height + + self.asset_type_data = {} + self.file_type_data = {} + self.supplier_data = {} + self.unsaved_changes = False # For unsaved changes tracking + self.asset_types_tab_page_for_filtering = None # For event filtering + + self._load_all_definitions() + + main_layout = QVBoxLayout(self) + + self.tab_widget = QTabWidget() + main_layout.addWidget(self.tab_widget) + + self._create_ui() # Creates and adds tabs to self.tab_widget + + self.button_box = QDialogButtonBox(QDialogButtonBox.Save | QDialogButtonBox.Cancel) + self.button_box.accepted.connect(self.save_definitions) + self.button_box.rejected.connect(self.reject) + main_layout.addWidget(self.button_box) + + self.setLayout(main_layout) + # self.tab_widget.installEventFilter(self) # Temporarily disable event filter on tab_widget for this test + # logger.info(f"Event filter on self.tab_widget ({self.tab_widget}) TEMPORARILY DISABLED for DebugListWidget test.") + + def _load_all_definitions(self): + logger.info("Loading all definitions...") + try: + self.asset_type_data = load_asset_definitions() + logger.info(f"Loaded {len(self.asset_type_data)} asset type definitions.") + except Exception as e: + logger.error(f"Failed to load asset type definitions: {e}") + self.asset_type_data = {} # Ensure it's an empty dict on failure + + try: + self.file_type_data = load_file_type_definitions() + logger.info(f"Loaded {len(self.file_type_data)} file type definitions.") + except Exception as e: + logger.error(f"Failed to load file type definitions: {e}") + self.file_type_data = {} + + try: + self.supplier_data = load_supplier_settings() + logger.info(f"Loaded {len(self.supplier_data)} supplier settings.") + except Exception as e: + logger.error(f"Failed to load supplier settings: {e}") + self.supplier_data = {} + logger.info("Finished loading definitions.") + + + def _create_ui(self): + self.tab_widget.addTab(self._create_asset_types_tab(), "Asset Type Definitions") + self.tab_widget.addTab(self._create_file_types_tab(), "File Type Definitions") + self.tab_widget.addTab(self._create_suppliers_tab(), "Supplier Settings") + + # Add a diagnostic button + self.diag_button = QPushButton("Test Select Item 2 (Asset)") + self.diag_button.clicked.connect(self._run_diag_selection) + # Assuming main_layout is accessible here or passed if _create_ui is part of __init__ + # If main_layout is self.layout() established in __init__ + if self.layout(): # Check if layout exists + self.layout().addWidget(self.diag_button) + else: + logger.error("Main layout not found for diagnostic button in _create_ui. Button not added.") + + + def _run_diag_selection(self): + logger.info("Diagnostic button clicked. Attempting to select second item in asset_type_list_widget.") + if hasattr(self, 'asset_type_list_widget') and self.asset_type_list_widget.count() > 1: + logger.info(f"Asset type list widget isEnabled: {self.asset_type_list_widget.isEnabled()}") # Check if enabled + logger.info(f"Asset type list widget signalsBlocked: {self.asset_type_list_widget.signalsBlocked()}") + + self.asset_type_list_widget.setFocus() # Explicitly set focus + logger.info(f"Attempted to set focus to asset_type_list_widget. Has focus: {self.asset_type_list_widget.hasFocus()}") + + item_to_select = self.asset_type_list_widget.item(1) # Select the second item (index 1) + if item_to_select: + logger.info(f"Programmatically selecting: {item_to_select.text()}") + self.asset_type_list_widget.setCurrentItem(item_to_select) + # Check if it's actually selected + if self.asset_type_list_widget.currentItem() == item_to_select: + logger.info(f"Programmatic selection successful. Current item is now: {self.asset_type_list_widget.currentItem().text()}") + else: + logger.warning("Programmatic selection FAILED. Current item did not change as expected.") + else: + logger.warning("Second item not found in asset_type_list_widget.") + elif hasattr(self, 'asset_type_list_widget'): + logger.warning("asset_type_list_widget has less than 2 items for diagnostic selection.") + else: + logger.warning("asset_type_list_widget not found for diagnostic selection.") + + def _create_tab_pane(self, title_singular, data_dict, list_widget_name): + tab_page = QWidget() + tab_page.setFocusPolicy(Qt.FocusPolicy.ClickFocus) + tab_layout = QHBoxLayout(tab_page) + + # Left Pane + left_pane_layout = QVBoxLayout() + + lbl_list_title = QLabel(f"{title_singular}s:") + left_pane_layout.addWidget(lbl_list_title) + + if list_widget_name == "asset_type_list_widget": + logger.info(f"Creating DebugListWidget for {list_widget_name}") + list_widget = DebugListWidget(self) # Pass parent + else: + list_widget = QListWidget(self) # Pass parent + + from PySide6.QtWidgets import QAbstractItemView + list_widget.setSelectionMode(QAbstractItemView.SingleSelection) + list_widget.setEnabled(True) + logger.info(f"For {list_widget_name}, SelectionMode set to SingleSelection, Enabled set to True.") + setattr(self, list_widget_name, list_widget) # e.g., self.asset_type_list_widget = list_widget + logger.info(f"Creating tab pane for {title_singular}, list_widget_name: {list_widget_name}") + logger.info(f"List widget instance for {list_widget_name}: {list_widget}") + + # Ensure no other event filters are active on the list_widget for this specific test + if list_widget_name == "asset_type_list_widget": + # If an event filter was installed on list_widget by a previous debug step via self.installEventFilter(list_widget), + # it would need to be removed here, or the logic installing it should be conditional. + # For now, we assume no other filter is on list_widget itself. + logger.info(f"Ensuring no stray event filter on DebugListWidget instance for {list_widget_name}.") + + if isinstance(data_dict, dict): + for key, value_dict in data_dict.items(): # Iterate over items for UserRole data + item = QListWidgetItem(key) + item.setData(Qt.UserRole, value_dict) # Store the whole dict + item.setFlags(item.flags() | Qt.ItemIsSelectable | Qt.ItemIsEnabled) # Explicitly set flags + list_widget.addItem(item) + else: + logger.warning(f"Data for {title_singular} is not a dictionary, cannot populate list.") + + left_pane_layout.addWidget(list_widget) + + buttons_layout = QHBoxLayout() + btn_add = QPushButton(f"Add {title_singular}") + btn_remove = QPushButton(f"Remove Selected {title_singular}") + + # Connections for these buttons will be specific to each tab type + if list_widget_name == "asset_type_list_widget": + btn_add.clicked.connect(self._add_asset_type) + btn_remove.clicked.connect(self._remove_asset_type) + # The event filter on asset_type_list_widget should be disabled for this test. + # Assuming the Debug mode task that set it up can be told to disable/remove it, + # or we ensure it's not re-added here if it was part of this method. + # For now, we just connect currentItemChanged directly. + list_widget.currentItemChanged.connect( + lambda current, previous, name=list_widget_name: + logger.info(f"LAMBDA: currentItemChanged for {name}. Current: {current.text() if current else 'None'}") + ) + list_widget.currentItemChanged.connect(self._display_asset_type_details) + logger.info(f"Connected currentItemChanged for {list_widget_name} to _display_asset_type_details AND diagnostic lambda.") + elif list_widget_name == "file_type_list_widget": + # For other list widgets, keep the previous event filter setup if it was specific, + # or remove if it was generic and now we only want DebugListWidget for assets. + # For this step, we are only changing asset_type_list_widget. + btn_add.clicked.connect(self._add_file_type) + btn_remove.clicked.connect(self._remove_file_type) + list_widget.currentItemChanged.connect( + lambda current, previous, name=list_widget_name: + logger.info(f"LAMBDA: currentItemChanged for {name}. Current: {current.text() if current else 'None'}") + ) + list_widget.currentItemChanged.connect(self._display_file_type_details) + logger.info(f"Connected currentItemChanged for {list_widget_name} to _display_file_type_details AND diagnostic lambda.") + elif list_widget_name == "supplier_list_widget": # Connections for Supplier tab + btn_add.clicked.connect(self._add_supplier) + btn_remove.clicked.connect(self._remove_supplier) + list_widget.currentItemChanged.connect( + lambda current, previous, name=list_widget_name: + logger.info(f"LAMBDA: currentItemChanged for {name}. Current: {current.text() if current else 'None'}") + ) + list_widget.currentItemChanged.connect(self._display_supplier_details) + logger.info(f"Connected currentItemChanged for {list_widget_name} to _display_supplier_details AND diagnostic lambda.") + + buttons_layout.addWidget(btn_add) + buttons_layout.addWidget(btn_remove) + left_pane_layout.addLayout(buttons_layout) + + tab_layout.addLayout(left_pane_layout, 1) # 1 part for left pane + + # Right Pane - This will be customized by specific tab creation methods + right_pane_widget = QWidget() # Create a generic widget to be returned + tab_layout.addWidget(right_pane_widget, 2) # 2 parts for right pane + + tab_page.setEnabled(True) # Explicitly enable the tab page widget + logger.info(f"Tab page for {title_singular} explicitly enabled.") + tab_page.setLayout(tab_layout) + return tab_page, right_pane_widget # Return the pane for customization + + def _create_asset_types_tab(self): + tab_page, right_pane_container = self._create_tab_pane("Asset Type", self.asset_type_data, "asset_type_list_widget") + self.asset_types_tab_page_for_filtering = tab_page # Store reference for event filter + # Ensure event filter on tab_page is also disabled if it was installed + # logger.info(f"Event filter on asset_types_tab_page ({tab_page}) should be disabled for DebugListWidget test.") + + # Customize the right pane for Asset Types + right_pane_groupbox = QGroupBox("Details for Selected Asset Type") + details_layout = QFormLayout(right_pane_groupbox) + + # Description + self.asset_description_edit = QTextEdit() + details_layout.addRow("Description:", self.asset_description_edit) + + # Color + color_layout = QHBoxLayout() + self.asset_color_swatch_label = QLabel() + self.asset_color_swatch_label.setFixedSize(20, 20) + self.asset_color_swatch_label.setAutoFillBackground(True) + self._update_color_swatch("#ffffff") # Default color + + btn_choose_color = QPushButton("Choose Color...") + btn_choose_color.clicked.connect(self._choose_asset_color) + color_layout.addWidget(self.asset_color_swatch_label) + color_layout.addWidget(btn_choose_color) + color_layout.addStretch() + details_layout.addRow("Color:", color_layout) + + # Examples + examples_group = QGroupBox("Examples") + examples_layout = QVBoxLayout(examples_group) + + self.asset_examples_list_widget = QListWidget() + examples_layout.addWidget(self.asset_examples_list_widget) + + example_buttons_layout = QHBoxLayout() + btn_add_example = QPushButton("Add Example") + btn_remove_example = QPushButton("Remove Selected Example") + btn_add_example.clicked.connect(self._add_asset_example) + btn_remove_example.clicked.connect(self._remove_asset_example) + example_buttons_layout.addWidget(btn_add_example) + example_buttons_layout.addWidget(btn_remove_example) + examples_layout.addLayout(example_buttons_layout) + + details_layout.addRow(examples_group) + + # Replace the generic right_pane_widget with our specific groupbox + # To do this, we need to find the layout of right_pane_container's parent (which is tab_layout) + # and replace the widget. + parent_layout = right_pane_container.parentWidget().layout() + if parent_layout: + parent_layout.replaceWidget(right_pane_container, right_pane_groupbox) + right_pane_container.deleteLater() # Remove the placeholder + + # Connect signals for editing + self.asset_description_edit.textChanged.connect(self._on_asset_detail_changed) + + # Initial population of list widget (if not already done by _create_tab_pane) + # and display details for the first item if any. + self._populate_asset_type_list() # Ensure data is loaded with UserRole + if self.asset_type_list_widget.count() > 0: + self.asset_type_list_widget.setCurrentRow(0) + # self._display_asset_type_details(self.asset_type_list_widget.currentItem()) # Already connected + + return tab_page + + def _populate_asset_type_list(self): + self.asset_type_list_widget.clear() + for key, asset_data_item in self.asset_type_data.items(): + item = QListWidgetItem(key) + # Ensure asset_data_item is a dictionary, if not, create a default one + if not isinstance(asset_data_item, dict): + logger.warning(f"Asset data for '{key}' is not a dict: {asset_data_item}. Using default.") + asset_data_item = {"description": str(asset_data_item), "color": "#ffffff", "examples": []} + + # Ensure essential keys exist + asset_data_item.setdefault('description', '') + asset_data_item.setdefault('color', '#ffffff') + asset_data_item.setdefault('examples', []) + + item.setData(Qt.UserRole, asset_data_item) + self.asset_type_list_widget.addItem(item) + + def _display_asset_type_details(self, current_item, previous_item=None): + logger.info(f"_display_asset_type_details called. Current: {current_item.text() if current_item else 'None'}, Previous: {previous_item.text() if previous_item else 'None'}") + if current_item: + logger.info(f"Current item text: {current_item.text()}") + logger.info(f"Current item data (UserRole): {current_item.data(Qt.UserRole)}") + else: + logger.info("Current item is None for asset_type_details.") + + try: + # Disconnect signals temporarily to prevent feedback loops during population + if hasattr(self, 'asset_description_edit'): + try: + self.asset_description_edit.textChanged.disconnect(self._on_asset_detail_changed) + logger.debug("Disconnected asset_description_edit.textChanged") + except TypeError: # Signal not connected + logger.debug("asset_description_edit.textChanged was not connected or already disconnected.") + pass + + if current_item: + asset_data = current_item.data(Qt.UserRole) + if not isinstance(asset_data, dict): # Should not happen if _populate is correct + logger.error(f"Invalid data for item {current_item.text()}. Expected dict, got {type(asset_data)}") + asset_data = {"description": "Error: Invalid data", "color": "#ff0000", "examples": []} + + self.asset_description_edit.setText(asset_data.get('description', '')) + + color_hex = asset_data.get('color', '#ffffff') + self._update_color_swatch(color_hex) + + self.asset_examples_list_widget.clear() + for example in asset_data.get('examples', []): + self.asset_examples_list_widget.addItem(example) + logger.debug(f"Populated details for {current_item.text()}") + else: + # Clear details if no item is selected + self.asset_description_edit.clear() + self._update_color_swatch("#ffffff") + self.asset_examples_list_widget.clear() + logger.debug("Cleared asset type details as no item is selected.") + + except Exception as e: + logger.error(f"Error in _display_asset_type_details: {e}", exc_info=True) + finally: + # Reconnect signals + if hasattr(self, 'asset_description_edit'): + try: + self.asset_description_edit.textChanged.connect(self._on_asset_detail_changed) + logger.debug("Reconnected asset_description_edit.textChanged") + except Exception as e: + logger.error(f"Failed to reconnect asset_description_edit.textChanged: {e}", exc_info=True) + logger.info("_display_asset_type_details finished.") + + def _update_color_swatch(self, color_hex): + if hasattr(self, 'asset_color_swatch_label'): + palette = self.asset_color_swatch_label.palette() + palette.setColor(QPalette.Window, QColor(color_hex)) + self.asset_color_swatch_label.setPalette(palette) + + def _choose_asset_color(self): + current_item = self.asset_type_list_widget.currentItem() + if not current_item: + return + + asset_data = current_item.data(Qt.UserRole) + initial_color = QColor(asset_data.get('color', '#ffffff')) + + color = QColorDialog.getColor(initial_color, self, "Choose Asset Type Color") + if color.isValid(): + color_hex = color.name() + self._update_color_swatch(color_hex) + asset_data['color'] = color_hex + current_item.setData(Qt.UserRole, asset_data) # Update data in item + self.unsaved_changes = True + # No need to call _on_asset_detail_changed explicitly for color, direct update is fine + + def _on_asset_detail_changed(self): + current_item = self.asset_type_list_widget.currentItem() + if not current_item: + return + + asset_data = current_item.data(Qt.UserRole) + if not isinstance(asset_data, dict): return # Should not happen + + # Update description + asset_data['description'] = self.asset_description_edit.toPlainText() + + # Examples are handled by their own add/remove buttons + # Color is handled by _choose_asset_color + + current_item.setData(Qt.UserRole, asset_data) # Save changes back to the item's data + self.unsaved_changes = True + + def _add_asset_type(self): + new_name, ok = QInputDialog.getText(self, "Add Asset Type", "Enter name for the new asset type:") + if ok and new_name: + if new_name in self.asset_type_data: + QMessageBox.warning(self, "Name Exists", f"An asset type named '{new_name}' already exists.") + return + + default_asset_type = { + "description": "", + "color": "#ffffff", + "examples": [] + } + self.asset_type_data[new_name] = default_asset_type + + item = QListWidgetItem(new_name) + item.setData(Qt.UserRole, default_asset_type) # Store a copy + self.asset_type_list_widget.addItem(item) + self.asset_type_list_widget.setCurrentItem(item) # Triggers _display_asset_type_details + logger.info(f"Added new asset type: {new_name}") + self.unsaved_changes = True + elif ok and not new_name: + QMessageBox.warning(self, "Invalid Name", "Asset type name cannot be empty.") + + def _remove_asset_type(self): + current_item = self.asset_type_list_widget.currentItem() + if not current_item: + QMessageBox.information(self, "No Selection", "Please select an asset type to remove.") + return + + asset_name = current_item.text() + reply = QMessageBox.question(self, "Confirm Removal", + f"Are you sure you want to remove the asset type '{asset_name}'?", + QMessageBox.Yes | QMessageBox.No, QMessageBox.No) + + if reply == QMessageBox.Yes: + if asset_name in self.asset_type_data: + del self.asset_type_data[asset_name] + + row = self.asset_type_list_widget.row(current_item) + self.asset_type_list_widget.takeItem(row) + logger.info(f"Removed asset type: {asset_name}") + self.unsaved_changes = True + + if self.asset_type_list_widget.count() > 0: + new_row_to_select = max(0, row - 1) if row > 0 else 0 + if self.asset_type_list_widget.count() > new_row_to_select: # Ensure new_row_to_select is valid + self.asset_type_list_widget.setCurrentRow(new_row_to_select) + else: # if list becomes empty or selection is out of bounds + self._display_asset_type_details(None, None) + else: + self._display_asset_type_details(None, None) # Clear details if list is empty + + def _add_asset_example(self): + current_asset_item = self.asset_type_list_widget.currentItem() + if not current_asset_item: + QMessageBox.information(self, "No Asset Type Selected", "Please select an asset type first.") + return + + new_example, ok = QInputDialog.getText(self, "Add Example", "Enter new example string:") + if ok and new_example: + asset_data = current_asset_item.data(Qt.UserRole) + if not isinstance(asset_data, dict) or 'examples' not in asset_data: + logger.error("Asset data is not a dict or 'examples' key is missing.") + QMessageBox.critical(self, "Error", "Internal data error for selected asset type.") + return + + if not isinstance(asset_data['examples'], list): # Ensure 'examples' is a list + asset_data['examples'] = [] + + asset_data['examples'].append(new_example) + current_asset_item.setData(Qt.UserRole, asset_data) # Update data in item + + self.asset_examples_list_widget.addItem(new_example) + logger.info(f"Added example '{new_example}' to asset type '{current_asset_item.text()}'") + self.unsaved_changes = True + elif ok and not new_example: + QMessageBox.warning(self, "Invalid Example", "Example string cannot be empty.") + + def _remove_asset_example(self): + current_asset_item = self.asset_type_list_widget.currentItem() + if not current_asset_item: + QMessageBox.information(self, "No Asset Type Selected", "Please select an asset type first.") + return + + current_example_item = self.asset_examples_list_widget.currentItem() + if not current_example_item: + QMessageBox.information(self, "No Example Selected", "Please select an example to remove.") + return + + example_text = current_example_item.text() + + # No confirmation needed as per typical list item removal, but can be added if desired. + # reply = QMessageBox.question(self, "Confirm Removal", + # f"Are you sure you want to remove the example '{example_text}'?", + # QMessageBox.Yes | QMessageBox.No, QMessageBox.No) + # if reply == QMessageBox.No: + # return + + asset_data = current_asset_item.data(Qt.UserRole) + if not isinstance(asset_data, dict) or 'examples' not in asset_data or not isinstance(asset_data['examples'], list): + logger.error("Asset data issue during example removal.") + QMessageBox.critical(self, "Error", "Internal data error for selected asset type.") + return + + try: + asset_data['examples'].remove(example_text) + current_asset_item.setData(Qt.UserRole, asset_data) # Update data in item + + row = self.asset_examples_list_widget.row(current_example_item) + self.asset_examples_list_widget.takeItem(row) + logger.info(f"Removed example '{example_text}' from asset type '{current_asset_item.text()}'") + self.unsaved_changes = True + except ValueError: + logger.warning(f"Example '{example_text}' not found in internal list for asset '{current_asset_item.text()}'. UI might be out of sync.") + # Still remove from UI if it was there + row = self.asset_examples_list_widget.row(current_example_item) + if row >=0: self.asset_examples_list_widget.takeItem(row) + + + def _update_file_type_color_swatch(self, color_hex, swatch_label): + if hasattr(self, swatch_label): # Check if the specific swatch label exists + palette = swatch_label.palette() + palette.setColor(QPalette.Window, QColor(color_hex)) + swatch_label.setPalette(palette) + + def _create_file_types_tab(self): + tab_page, right_pane_container = self._create_tab_pane("File Type", self.file_type_data, "file_type_list_widget") + + right_pane_groupbox = QGroupBox("Details for Selected File Type") + details_layout = QFormLayout(right_pane_groupbox) + + # Description + self.ft_description_edit = QTextEdit() + details_layout.addRow("Description:", self.ft_description_edit) + + # Color + ft_color_layout = QHBoxLayout() + self.ft_color_swatch_label = QLabel() + self.ft_color_swatch_label.setFixedSize(20, 20) + self.ft_color_swatch_label.setAutoFillBackground(True) + self._update_color_swatch_generic(self.ft_color_swatch_label, "#ffffff") # Default + + btn_ft_choose_color = QPushButton("Choose Color...") + btn_ft_choose_color.clicked.connect(self._choose_file_type_color) + ft_color_layout.addWidget(self.ft_color_swatch_label) + ft_color_layout.addWidget(btn_ft_choose_color) + ft_color_layout.addStretch() + details_layout.addRow("Color:", ft_color_layout) + + # Examples + ft_examples_group = QGroupBox("Examples") + ft_examples_layout = QVBoxLayout(ft_examples_group) + self.ft_examples_list_widget = QListWidget() + ft_examples_layout.addWidget(self.ft_examples_list_widget) + ft_example_buttons_layout = QHBoxLayout() + btn_ft_add_example = QPushButton("Add Example") + btn_ft_remove_example = QPushButton("Remove Selected Example") + btn_ft_add_example.clicked.connect(self._add_file_type_example) + btn_ft_remove_example.clicked.connect(self._remove_file_type_example) + ft_example_buttons_layout.addWidget(btn_ft_add_example) + ft_example_buttons_layout.addWidget(btn_ft_remove_example) + ft_examples_layout.addLayout(ft_example_buttons_layout) + details_layout.addRow(ft_examples_group) + + # Standard Type + self.ft_standard_type_edit = QLineEdit() + details_layout.addRow("Standard Type:", self.ft_standard_type_edit) + + # Bit Depth Rule + self.ft_bit_depth_combo = QComboBox() + self.ft_bit_depth_combo.addItems(["respect", "force_8bit", "force_16bit"]) + details_layout.addRow("Bit Depth Rule:", self.ft_bit_depth_combo) + + # Is Grayscale + self.ft_is_grayscale_check = QCheckBox("Is Grayscale") + details_layout.addRow(self.ft_is_grayscale_check) # No label for checkbox itself + + # Keybind + self.ft_keybind_edit = QLineEdit() + self.ft_keybind_edit.setMaxLength(1) # Basic validation + details_layout.addRow("Keybind:", self.ft_keybind_edit) + + parent_layout = right_pane_container.parentWidget().layout() + if parent_layout: + parent_layout.replaceWidget(right_pane_container, right_pane_groupbox) + right_pane_container.deleteLater() + + # Connect signals for editing + self.ft_description_edit.textChanged.connect(self._on_file_type_detail_changed) + self.ft_standard_type_edit.textChanged.connect(self._on_file_type_detail_changed) + self.ft_bit_depth_combo.currentIndexChanged.connect(self._on_file_type_detail_changed) + self.ft_is_grayscale_check.stateChanged.connect(self._on_file_type_detail_changed) + self.ft_keybind_edit.textChanged.connect(self._on_file_type_detail_changed) + + self._populate_file_type_list() + if self.file_type_list_widget.count() > 0: + self.file_type_list_widget.setCurrentRow(0) + # _display_file_type_details is connected to currentItemChanged + + return tab_page + + def _populate_file_type_list(self): + self.file_type_list_widget.clear() + for key, ft_data_item in self.file_type_data.items(): + item = QListWidgetItem(key) + if not isinstance(ft_data_item, dict): + logger.warning(f"File type data for '{key}' is not a dict: {ft_data_item}. Using default.") + ft_data_item = { + "description": str(ft_data_item), "color": "#ffffff", "examples": [], + "standard_type": "", "bit_depth_rule": "respect", + "is_grayscale": False, "keybind": "" + } + + # Ensure all essential keys exist with defaults + ft_data_item.setdefault('description', '') + ft_data_item.setdefault('color', '#ffffff') + ft_data_item.setdefault('examples', []) + ft_data_item.setdefault('standard_type', '') + ft_data_item.setdefault('bit_depth_rule', 'respect') + ft_data_item.setdefault('is_grayscale', False) + ft_data_item.setdefault('keybind', '') + + item.setData(Qt.UserRole, ft_data_item) + self.file_type_list_widget.addItem(item) + + def _display_file_type_details(self, current_item, previous_item=None): + logger.info(f"_display_file_type_details called. Current: {current_item.text() if current_item else 'None'}, Previous: {previous_item.text() if previous_item else 'None'}") + if current_item: + logger.info(f"Current item text: {current_item.text()}") + logger.info(f"Current item data (UserRole): {current_item.data(Qt.UserRole)}") + else: + logger.info("Current item is None for file_type_details.") + + try: + # Disconnect signals temporarily + logger.debug("Disconnecting file type detail signals...") + try: self.ft_description_edit.textChanged.disconnect(self._on_file_type_detail_changed) + except TypeError: pass + try: self.ft_standard_type_edit.textChanged.disconnect(self._on_file_type_detail_changed) + except TypeError: pass + try: self.ft_bit_depth_combo.currentIndexChanged.disconnect(self._on_file_type_detail_changed) + except TypeError: pass + try: self.ft_is_grayscale_check.stateChanged.disconnect(self._on_file_type_detail_changed) + except TypeError: pass + try: self.ft_keybind_edit.textChanged.disconnect(self._on_file_type_detail_changed) + except TypeError: pass + logger.debug("Finished disconnecting file type detail signals.") + + if current_item: + ft_data = current_item.data(Qt.UserRole) + if not isinstance(ft_data, dict): + logger.error(f"Invalid data for file type item {current_item.text()}. Expected dict, got {type(ft_data)}") + ft_data = { + "description": "Error: Invalid data", "color": "#ff0000", "examples": [], + "standard_type": "error", "bit_depth_rule": "respect", + "is_grayscale": False, "keybind": "X" + } + + self.ft_description_edit.setText(ft_data.get('description', '')) + self._update_color_swatch_generic(self.ft_color_swatch_label, ft_data.get('color', '#ffffff')) + + self.ft_examples_list_widget.clear() + for example in ft_data.get('examples', []): + self.ft_examples_list_widget.addItem(example) + + self.ft_standard_type_edit.setText(ft_data.get('standard_type', '')) + + bdr_index = self.ft_bit_depth_combo.findText(ft_data.get('bit_depth_rule', 'respect')) + if bdr_index != -1: + self.ft_bit_depth_combo.setCurrentIndex(bdr_index) + else: + self.ft_bit_depth_combo.setCurrentIndex(0) # Default to 'respect' + + self.ft_is_grayscale_check.setChecked(ft_data.get('is_grayscale', False)) + self.ft_keybind_edit.setText(ft_data.get('keybind', '')) + logger.debug(f"Populated details for file type {current_item.text()}") + else: + # Clear details if no item is selected + self.ft_description_edit.clear() + self._update_color_swatch_generic(self.ft_color_swatch_label, "#ffffff") + self.ft_examples_list_widget.clear() + self.ft_standard_type_edit.clear() + self.ft_bit_depth_combo.setCurrentIndex(0) + self.ft_is_grayscale_check.setChecked(False) + self.ft_keybind_edit.clear() + logger.debug("Cleared file type details as no item is selected.") + + except Exception as e: + logger.error(f"Error in _display_file_type_details: {e}", exc_info=True) + finally: + # Reconnect signals + logger.debug("Reconnecting file type detail signals...") + try: + self.ft_description_edit.textChanged.connect(self._on_file_type_detail_changed) + self.ft_standard_type_edit.textChanged.connect(self._on_file_type_detail_changed) + self.ft_bit_depth_combo.currentIndexChanged.connect(self._on_file_type_detail_changed) + self.ft_is_grayscale_check.stateChanged.connect(self._on_file_type_detail_changed) + self.ft_keybind_edit.textChanged.connect(self._on_file_type_detail_changed) + logger.debug("Finished reconnecting file type detail signals.") + except Exception as e: + logger.error(f"Failed to reconnect file type detail signals: {e}", exc_info=True) + logger.info("_display_file_type_details finished.") + + def _update_color_swatch_generic(self, swatch_label, color_hex): + """Generic color swatch update for any QLabel.""" + if swatch_label: # Check if the swatch label exists and is passed correctly + palette = swatch_label.palette() + palette.setColor(QPalette.Window, QColor(color_hex)) + swatch_label.setPalette(palette) + swatch_label.update() # Ensure the label repaints + + # --- File Type action methods --- + def _add_file_type(self): + new_id, ok = QInputDialog.getText(self, "Add File Type", "Enter ID for the new file type (e.g., MAP_ALB):") + if ok and new_id: + new_id = new_id.strip() # Remove leading/trailing whitespace + if not new_id: # Check if empty after strip + QMessageBox.warning(self, "Invalid ID", "File type ID cannot be empty.") + return + if new_id in self.file_type_data: + QMessageBox.warning(self, "ID Exists", f"A file type with ID '{new_id}' already exists.") + return + + default_file_type = { + "description": "", + "color": "#ffffff", + "examples": [], + "standard_type": "", + "bit_depth_rule": "respect", + "is_grayscale": False, + "keybind": "" + } + self.file_type_data[new_id] = default_file_type + + item = QListWidgetItem(new_id) + item.setData(Qt.UserRole, default_file_type.copy()) # Store a copy for the item + self.file_type_list_widget.addItem(item) + self.file_type_list_widget.setCurrentItem(item) # Triggers _display_file_type_details + logger.info(f"Added new file type: {new_id}") + self.unsaved_changes = True + elif ok and not new_id.strip(): # Also catch if user entered only spaces and pressed OK + QMessageBox.warning(self, "Invalid ID", "File type ID cannot be empty.") + + def _remove_file_type(self): + current_item = self.file_type_list_widget.currentItem() + if not current_item: + QMessageBox.information(self, "No Selection", "Please select a file type to remove.") + return + + file_type_id = current_item.text() + reply = QMessageBox.question(self, "Confirm Removal", + f"Are you sure you want to remove the file type '{file_type_id}'?", + QMessageBox.Yes | QMessageBox.No, QMessageBox.No) + + if reply == QMessageBox.Yes: + if file_type_id in self.file_type_data: + del self.file_type_data[file_type_id] + + row = self.file_type_list_widget.row(current_item) + self.file_type_list_widget.takeItem(row) + logger.info(f"Removed file type: {file_type_id}") + self.unsaved_changes = True + + if self.file_type_list_widget.count() > 0: + new_row_to_select = max(0, row - 1) if row > 0 else 0 + if self.file_type_list_widget.count() > new_row_to_select: + self.file_type_list_widget.setCurrentRow(new_row_to_select) + else: # if list becomes empty or selection is out of bounds + self._display_file_type_details(None, None) # Clear details + else: + self._display_file_type_details(None, None) # Clear details if list is empty + + def _choose_file_type_color(self): + current_item = self.file_type_list_widget.currentItem() + if not current_item: + return + + ft_data = current_item.data(Qt.UserRole) + if not isinstance(ft_data, dict): # Should not happen + logger.error("File type item data is not a dict in _choose_file_type_color.") + return + + initial_color = QColor(ft_data.get('color', '#ffffff')) + + color = QColorDialog.getColor(initial_color, self, "Choose File Type Color") + if color.isValid(): + color_hex = color.name() + self._update_color_swatch_generic(self.ft_color_swatch_label, color_hex) + ft_data['color'] = color_hex + current_item.setData(Qt.UserRole, ft_data) # Update data in item + self.unsaved_changes = True + + def _add_file_type_example(self): + current_ft_item = self.file_type_list_widget.currentItem() + if not current_ft_item: + QMessageBox.information(self, "No File Type Selected", "Please select a file type first.") + return + + new_example, ok = QInputDialog.getText(self, "Add File Type Example", "Enter new example string (e.g., _alb.png, .exr):") + if ok and new_example: + new_example = new_example.strip() + if not new_example: + QMessageBox.warning(self, "Invalid Example", "Example string cannot be empty.") + return + + ft_data = current_ft_item.data(Qt.UserRole) + if not isinstance(ft_data, dict) or 'examples' not in ft_data: + logger.error("File type data is not a dict or 'examples' key is missing.") + QMessageBox.critical(self, "Error", "Internal data error for selected file type.") + return + + if not isinstance(ft_data['examples'], list): # Ensure 'examples' is a list + ft_data['examples'] = [] + + if new_example in ft_data['examples']: + QMessageBox.information(self, "Example Exists", f"The example '{new_example}' already exists for this file type.") + return + + ft_data['examples'].append(new_example) + current_ft_item.setData(Qt.UserRole, ft_data) # Update data in item + + self.ft_examples_list_widget.addItem(new_example) + logger.info(f"Added example '{new_example}' to file type '{current_ft_item.text()}'") + self.unsaved_changes = True + elif ok and not new_example.strip(): + QMessageBox.warning(self, "Invalid Example", "Example string cannot be empty.") + + def _remove_file_type_example(self): + current_ft_item = self.file_type_list_widget.currentItem() + if not current_ft_item: + QMessageBox.information(self, "No File Type Selected", "Please select a file type first.") + return + + current_example_item = self.ft_examples_list_widget.currentItem() + if not current_example_item: + QMessageBox.information(self, "No Example Selected", "Please select an example to remove.") + return + + example_text = current_example_item.text() + + ft_data = current_ft_item.data(Qt.UserRole) + if not isinstance(ft_data, dict) or 'examples' not in ft_data or not isinstance(ft_data['examples'], list): + logger.error("File type data issue during example removal.") + QMessageBox.critical(self, "Error", "Internal data error for selected file type.") + return + + try: + ft_data['examples'].remove(example_text) + current_ft_item.setData(Qt.UserRole, ft_data) # Update data in item + + row = self.ft_examples_list_widget.row(current_example_item) + self.ft_examples_list_widget.takeItem(row) + logger.info(f"Removed example '{example_text}' from file type '{current_ft_item.text()}'") + self.unsaved_changes = True + except ValueError: + logger.warning(f"Example '{example_text}' not found in internal list for file type '{current_ft_item.text()}'. UI might be out of sync.") + row = self.ft_examples_list_widget.row(current_example_item) + if row >=0: self.ft_examples_list_widget.takeItem(row) + + def _on_file_type_detail_changed(self): + current_item = self.file_type_list_widget.currentItem() + if not current_item: + return + + ft_data = current_item.data(Qt.UserRole) + if not isinstance(ft_data, dict): + logger.error("File type item data is not a dict in _on_file_type_detail_changed.") + return + + # Update based on which widget triggered (or update all) + ft_data['description'] = self.ft_description_edit.toPlainText() + ft_data['standard_type'] = self.ft_standard_type_edit.text() + ft_data['bit_depth_rule'] = self.ft_bit_depth_combo.currentText() + ft_data['is_grayscale'] = self.ft_is_grayscale_check.isChecked() + + # Keybind validation (force uppercase) + keybind_text = self.ft_keybind_edit.text() + if keybind_text: # MaxLength(1) is already set + # Disconnect to prevent recursive call during setText + try: self.ft_keybind_edit.textChanged.disconnect(self._on_file_type_detail_changed) + except TypeError: pass + self.ft_keybind_edit.setText(keybind_text.upper()) + # Reconnect + self.ft_keybind_edit.textChanged.connect(self._on_file_type_detail_changed) + ft_data['keybind'] = keybind_text.upper() + else: + ft_data['keybind'] = '' + + current_item.setData(Qt.UserRole, ft_data) + logger.debug(f"File type '{current_item.text()}' data updated: {ft_data}") + self.unsaved_changes = True + # --- End Placeholder methods --- + + def _create_suppliers_tab(self): + tab_page, right_pane_container = self._create_tab_pane("Supplier", self.supplier_data, "supplier_list_widget") + + right_pane_groupbox = QGroupBox("Details for Selected Supplier") + details_layout = QFormLayout(right_pane_groupbox) + + # Normal Map Type + self.supplier_normal_map_type_combo = QComboBox() + self.supplier_normal_map_type_combo.addItems(["OpenGL", "DirectX"]) + details_layout.addRow("Normal Map Type:", self.supplier_normal_map_type_combo) + + # Replace the generic right_pane_widget + parent_layout = right_pane_container.parentWidget().layout() + if parent_layout: + parent_layout.replaceWidget(right_pane_container, right_pane_groupbox) + right_pane_container.deleteLater() + + # Connect signals for editing + self.supplier_normal_map_type_combo.currentIndexChanged.connect(self._on_supplier_detail_changed) + + # Initial population and display + self._populate_supplier_list() + if self.supplier_list_widget.count() > 0: + self.supplier_list_widget.setCurrentRow(0) + # _display_supplier_details is connected to currentItemChanged + + return tab_page + + def _populate_supplier_list(self): + self.supplier_list_widget.clear() + for key, sup_data_item in self.supplier_data.items(): + item = QListWidgetItem(key) + if not isinstance(sup_data_item, dict): + logger.warning(f"Supplier data for '{key}' is not a dict: {sup_data_item}. Using default.") + sup_data_item = {"normal_map_type": "OpenGL"} + sup_data_item.setdefault('normal_map_type', 'OpenGL') # Ensure key exists + item.setData(Qt.UserRole, sup_data_item) + self.supplier_list_widget.addItem(item) + + def _display_supplier_details(self, current_item, previous_item=None): + logger.info(f"_display_supplier_details called. Current: {current_item.text() if current_item else 'None'}, Previous: {previous_item.text() if previous_item else 'None'}") + if current_item: + logger.info(f"Current item text: {current_item.text()}") + logger.info(f"Current item data (UserRole): {current_item.data(Qt.UserRole)}") + else: + logger.info("Current item is None for supplier_details.") + + try: + # Disconnect signals temporarily + if hasattr(self, 'supplier_normal_map_type_combo'): + try: + self.supplier_normal_map_type_combo.currentIndexChanged.disconnect(self._on_supplier_detail_changed) + logger.debug("Disconnected supplier_normal_map_type_combo.currentIndexChanged") + except TypeError: + logger.debug("supplier_normal_map_type_combo.currentIndexChanged was not connected or already disconnected.") + pass + + if current_item: + supplier_name = current_item.text() + supplier_data = self.supplier_data.get(supplier_name) + + if not isinstance(supplier_data, dict): + logger.error(f"Invalid data for supplier item {supplier_name}. Expected dict, got {type(supplier_data)}") + item_data_role = current_item.data(Qt.UserRole) + if isinstance(item_data_role, dict): + supplier_data = item_data_role + else: + supplier_data = {"normal_map_type": "OpenGL"} + + normal_map_type = supplier_data.get('normal_map_type', 'OpenGL') + nmt_index = self.supplier_normal_map_type_combo.findText(normal_map_type) + if nmt_index != -1: + self.supplier_normal_map_type_combo.setCurrentIndex(nmt_index) + else: + self.supplier_normal_map_type_combo.setCurrentIndex(0) + logger.debug(f"Populated details for supplier {current_item.text()}") + else: + # Clear details if no item is selected + if hasattr(self, 'supplier_normal_map_type_combo'): + self.supplier_normal_map_type_combo.setCurrentIndex(0) + logger.debug("Cleared supplier details as no item is selected.") + + except Exception as e: + logger.error(f"Error in _display_supplier_details: {e}", exc_info=True) + finally: + # Reconnect signals + if hasattr(self, 'supplier_normal_map_type_combo'): + try: + self.supplier_normal_map_type_combo.currentIndexChanged.connect(self._on_supplier_detail_changed) + logger.debug("Reconnected supplier_normal_map_type_combo.currentIndexChanged") + except Exception as e: + logger.error(f"Failed to reconnect supplier_normal_map_type_combo.currentIndexChanged: {e}", exc_info=True) + logger.info("_display_supplier_details finished.") + + def _on_supplier_detail_changed(self): + current_item = self.supplier_list_widget.currentItem() + if not current_item: + return + + supplier_name = current_item.text() + if supplier_name not in self.supplier_data: + logger.error(f"Supplier '{supplier_name}' not found in self.supplier_data during detail change.") + return # Or create it, but that might be unexpected here + + # Ensure the entry in self.supplier_data is a dictionary + if not isinstance(self.supplier_data[supplier_name], dict): + self.supplier_data[supplier_name] = {} # Initialize if it's not a dict + + new_normal_map_type = self.supplier_normal_map_type_combo.currentText() + self.supplier_data[supplier_name]['normal_map_type'] = new_normal_map_type + + # Update the item's UserRole data as well to keep it in sync + current_item.setData(Qt.UserRole, self.supplier_data[supplier_name].copy()) + + logger.debug(f"Supplier '{supplier_name}' normal_map_type updated to: {new_normal_map_type}") + self.unsaved_changes = True + + def _add_supplier(self): + new_name, ok = QInputDialog.getText(self, "Add Supplier", "Enter name for the new supplier:") + if ok and new_name: + new_name = new_name.strip() + if not new_name: + QMessageBox.warning(self, "Invalid Name", "Supplier name cannot be empty.") + return + if new_name in self.supplier_data: + QMessageBox.warning(self, "Name Exists", f"A supplier named '{new_name}' already exists.") + return + + default_supplier_settings = {"normal_map_type": "OpenGL"} + self.supplier_data[new_name] = default_supplier_settings + + item = QListWidgetItem(new_name) + item.setData(Qt.UserRole, default_supplier_settings.copy()) # Store a copy + self.supplier_list_widget.addItem(item) + self.supplier_list_widget.setCurrentItem(item) # Triggers display + logger.info(f"Added new supplier: {new_name}") + self.unsaved_changes = True + elif ok and not new_name.strip(): + QMessageBox.warning(self, "Invalid Name", "Supplier name cannot be empty.") + + def _remove_supplier(self): + current_item = self.supplier_list_widget.currentItem() + if not current_item: + QMessageBox.information(self, "No Selection", "Please select a supplier to remove.") + return + + supplier_name = current_item.text() + reply = QMessageBox.question(self, "Confirm Removal", + f"Are you sure you want to remove the supplier '{supplier_name}'?", + QMessageBox.Yes | QMessageBox.No, QMessageBox.No) + + if reply == QMessageBox.Yes: + if supplier_name in self.supplier_data: + del self.supplier_data[supplier_name] + + row = self.supplier_list_widget.row(current_item) + self.supplier_list_widget.takeItem(row) + logger.info(f"Removed supplier: {supplier_name}") + self.unsaved_changes = True + + # Select another item or clear details + if self.supplier_list_widget.count() > 0: + new_row_to_select = max(0, row - 1) if row > 0 else 0 + if self.supplier_list_widget.count() > new_row_to_select: + self.supplier_list_widget.setCurrentRow(new_row_to_select) + else: + self._display_supplier_details(None, None) + else: + self._display_supplier_details(None, None) # Clear details if list is empty + + def save_definitions(self): + logger.info("Attempting to save definitions...") + try: + # --- Asset Type Definitions --- + # Ensure self.asset_type_data is consistent with the QListWidget items. + # All edits should have updated the item's UserRole data. + # Add/Remove operations update self.asset_type_data directly. + # This loop ensures any in-place modifications to item data (like description, color) + # are reflected in the self.asset_type_data before saving. + + current_keys_in_list = set() + if hasattr(self, 'asset_type_list_widget'): # Check if the widget exists + for i in range(self.asset_type_list_widget.count()): + item = self.asset_type_list_widget.item(i) + key = item.text() + current_keys_in_list.add(key) + # Update self.asset_type_data with the (potentially modified) UserRole data + item_data = item.data(Qt.UserRole) + if isinstance(item_data, dict): + self.asset_type_data[key] = item_data + else: + logger.warning(f"Item '{key}' in asset_type_list_widget has non-dict UserRole data: {type(item_data)}. Skipping update for this item in self.asset_type_data.") + + # Remove any keys from self.asset_type_data that are no longer in the list + # (should be handled by _remove_asset_type, but this is a safeguard) + keys_to_remove_from_dict = set(self.asset_type_data.keys()) - current_keys_in_list + for key in keys_to_remove_from_dict: + logger.info(f"Removing orphaned key '{key}' from self.asset_type_data before saving.") + del self.asset_type_data[key] + + save_asset_definitions(self.asset_type_data) + logger.info("Asset Type definitions saved successfully.") + + # --- File Type Definitions --- + if hasattr(self, 'file_type_data') and hasattr(self, 'file_type_list_widget'): + current_ft_keys_in_list = set() + for i in range(self.file_type_list_widget.count()): + item = self.file_type_list_widget.item(i) + key = item.text() + current_ft_keys_in_list.add(key) + item_data = item.data(Qt.UserRole) + if isinstance(item_data, dict): + self.file_type_data[key] = item_data + else: + logger.warning(f"Item '{key}' in file_type_list_widget has non-dict UserRole data: {type(item_data)}. Skipping.") + + keys_to_remove_ft = set(self.file_type_data.keys()) - current_ft_keys_in_list + for key in keys_to_remove_ft: + logger.info(f"Removing orphaned key '{key}' from self.file_type_data before saving.") + del self.file_type_data[key] + + save_file_type_definitions(self.file_type_data) + logger.info("File Type definitions saved successfully.") + else: + logger.info("File type data or list widget not found, skipping save for file types.") + + # --- Supplier Settings --- + if hasattr(self, 'supplier_data') and hasattr(self, 'supplier_list_widget'): + current_s_keys_in_list = set() + for i in range(self.supplier_list_widget.count()): + item = self.supplier_list_widget.item(i) + key = item.text() + current_s_keys_in_list.add(key) + item_data = item.data(Qt.UserRole) + if isinstance(item_data, dict): + self.supplier_data[key] = item_data # Ensure self.supplier_data is up-to-date + else: + logger.warning(f"Item '{key}' in supplier_list_widget has non-dict UserRole data: {type(item_data)}. Skipping update for this item in self.supplier_data.") + + keys_to_remove_s = set(self.supplier_data.keys()) - current_s_keys_in_list + for key in keys_to_remove_s: + logger.info(f"Removing orphaned key '{key}' from self.supplier_data before saving.") + del self.supplier_data[key] + + save_supplier_settings(self.supplier_data) + logger.info("Supplier settings saved successfully.") + else: + logger.info("Supplier data or list widget not found, skipping save for suppliers.") + + + QMessageBox.information(self, "Save Successful", "Definitions saved successfully.") + self.unsaved_changes = False # Reset flag + self.accept() # Close dialog on successful save + + except Exception as e: + logger.error(f"Failed to save definitions: {e}", exc_info=True) + QMessageBox.critical(self, "Save Error", f"Could not save definitions: {e}") + # Optionally, do not close the dialog on error by removing self.accept() or calling self.reject() + + def reject(self): + if self.unsaved_changes: + reply = QMessageBox.question(self, "Unsaved Changes", + "You have unsaved changes. Are you sure you want to cancel?", + QMessageBox.Yes | QMessageBox.No, QMessageBox.No) + if reply == QMessageBox.No: + return # Do not close + super().reject() # Proceed with closing + + def closeEvent(self, event): + if self.unsaved_changes: + reply = QMessageBox.question(self, "Unsaved Changes", + "You have unsaved changes. Are you sure you want to close?", + QMessageBox.Yes | QMessageBox.No, QMessageBox.No) + if reply == QMessageBox.Yes: + event.accept() + else: + event.ignore() + else: + event.accept() + + def eventFilter(self, watched, event: QEvent): # Renamed from mouse_event_filter + event_type = event.type() + + if watched == self.tab_widget: + # Construct a more identifiable name for the tab widget in logs + tab_widget_name_for_log = self.tab_widget.objectName() if self.tab_widget.objectName() else watched.__class__.__name__ + prefix = f"EventFilter (QTabWidget '{tab_widget_name_for_log}'):" + + if event_type == QEvent.MouseButtonPress or event_type == QEvent.MouseButtonRelease: + event_name = "Press" if event_type == QEvent.MouseButtonPress else "Release" + + # Ensure event has position method (it's a QMouseEvent) + if hasattr(event, 'position') and hasattr(event, 'globalPosition') and hasattr(event, 'button'): + log_line = (f"{prefix} MouseButton{event_name} " + f"global_pos={event.globalPosition().toPoint()}, " + f"widget_pos={event.position().toPoint()}, " + f"button={event.button()}, accepted={event.isAccepted()}") + logger.info(log_line) + + current_page = self.tab_widget.currentWidget() + if current_page: + # event.position() is relative to self.tab_widget (the watched object) + tab_widget_event_pos_float = event.position() # QPointF + tab_widget_event_pos = tab_widget_event_pos_float.toPoint() # QPoint + + # Map event position from tab_widget coordinates to global, then to page coordinates + global_pos = self.tab_widget.mapToGlobal(tab_widget_event_pos) + page_event_pos = current_page.mapFromGlobal(global_pos) + + is_over_page = current_page.rect().contains(page_event_pos) + page_name_for_log = current_page.objectName() if current_page.objectName() else current_page.__class__.__name__ + + logger.info(f"{prefix} Event mapped to page '{page_name_for_log}' coords: {page_event_pos}. " + f"Page rect: {current_page.rect()}. Is over page: {is_over_page}") + + if is_over_page: + logger.info(f"{prefix} Event IS OVER CURRENT PAGE. " + f"Current event.isAccepted(): {event.isAccepted()}. " + f"Returning False from filter to allow propagation to QTabWidget's default handling.") + # Returning False means this filter does not stop the event. + # The event will be sent to self.tab_widget.event() for its default handling, + # which should then propagate to children if appropriate. + return False + else: + logger.info(f"{prefix} Event is NOT over current page (likely on tab bar). Allowing default QTabWidget handling.") + else: + logger.info(f"{prefix} No current page for tab_widget during mouse event.") + else: + logger.warning(f"{prefix} MouseButton{event_name} received, but event object lacks expected QMouseEvent attributes.") + + # Example: Log other event types if needed for debugging, but keep it concise + # elif event_type == QEvent.Enter: + # logger.debug(f"{prefix} Enter event") + # elif event_type == QEvent.Leave: + # logger.debug(f"{prefix} Leave event") + # elif event_type == QEvent.FocusIn: + # logger.debug(f"{prefix} FocusIn event") + # elif event_type == QEvent.FocusOut: + # logger.debug(f"{prefix} FocusOut event") + + # For other watched objects (if any were installed on), or for events on self.tab_widget + # that were not explicitly handled (e.g., not mouse press/release over page), + # call the base class implementation. + return super().eventFilter(watched, event) + +if __name__ == '__main__': + # This is for testing the dialog independently + from PyQt5.QtWidgets import QApplication + import sys + + # Setup basic logging for testing + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s') + + # Create dummy config files if they don't exist for testing + config_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'config')) + os.makedirs(config_dir, exist_ok=True) + + asset_types_path = os.path.join(config_dir, 'asset_type_definitions.json') + file_types_path = os.path.join(config_dir, 'file_type_definitions.json') + suppliers_path = os.path.join(config_dir, 'suppliers.json') + + if not os.path.exists(asset_types_path): + with open(asset_types_path, 'w') as f: + f.write('{"GenericModel": {"description": "A generic 3D model"}, "TextureSet": {"description": "A set of PBR textures"}}') + if not os.path.exists(file_types_path): + with open(file_types_path, 'w') as f: + f.write('{".fbx": {"description": "Filmbox format"}, ".png": {"description": "Portable Network Graphics"}}') + if not os.path.exists(suppliers_path): + with open(suppliers_path, 'w') as f: + f.write('{"Poliigon": {"api_key": "dummy_key"}, "Local": {}}') + + app = QApplication(sys.argv) + dialog = DefinitionsEditorDialog() + dialog.show() + sys.exit(app.exec_()) \ No newline at end of file diff --git a/gui/delegates.py b/gui/delegates.py new file mode 100644 index 0000000..d40765d --- /dev/null +++ b/gui/delegates.py @@ -0,0 +1,240 @@ +from pathlib import Path +from PySide6.QtWidgets import QStyledItemDelegate, QLineEdit, QComboBox +from PySide6.QtCore import Qt, QModelIndex +from configuration import Configuration, ConfigurationError, load_base_config # Keep load_base_config for SupplierSearchDelegate +from PySide6.QtWidgets import QListWidgetItem + +import json +import logging +import os +from PySide6.QtWidgets import QCompleter + +log = logging.getLogger(__name__) +SUPPLIERS_CONFIG_PATH = "config/suppliers.json" + +class LineEditDelegate(QStyledItemDelegate): + """Delegate for editing string values using a QLineEdit.""" + def createEditor(self, parent, option, index): + editor = QLineEdit(parent) + return editor + + def setEditorData(self, editor: QLineEdit, index: QModelIndex): + # Use EditRole to get the raw data suitable for editing. + value = index.model().data(index, Qt.EditRole) + editor.setText(str(value) if value is not None else "") + + def setModelData(self, editor: QLineEdit, model, index: QModelIndex): + value = editor.text() + # Pass the potentially modified text back to the model's setData. + model.setData(index, value, Qt.EditRole) + + def updateEditorGeometry(self, editor, option, index): + editor.setGeometry(option.rect) + + +class ComboBoxDelegate(QStyledItemDelegate): + """ + Delegate for editing string values from a predefined list using a QComboBox. + Determines the list source based on column index by accessing the + UnifiedViewModel directly. + """ + # REMOVED main_window parameter + def __init__(self, parent=None): + super().__init__(parent) + # REMOVED self.main_window store + + def createEditor(self, parent, option, index: QModelIndex): + editor = QComboBox(parent) + column = index.column() + model = index.model() + + # Add a "clear" option first, associating None with it. + editor.addItem("---", None) # UserData = None + + # Populate based on column by accessing the model's cached keys + items_keys = [] # Default to empty list + + # --- Get keys directly from the UnifiedViewModel --- + # Check if the model is the correct type and has the attributes + if hasattr(model, '_asset_type_keys') and hasattr(model, '_file_type_keys'): + try: + # Use column constants from the model if available + COL_ASSET_TYPE = getattr(model, 'COL_ASSET_TYPE', 3) # Default fallback + COL_ITEM_TYPE = getattr(model, 'COL_ITEM_TYPE', 4) # Default fallback + + if column == COL_ASSET_TYPE: + items_keys = model._asset_type_keys # Use cached keys + elif column == COL_ITEM_TYPE: + items_keys = model._file_type_keys # Use cached keys + + except Exception as e: + log.error(f"Error getting keys from UnifiedViewModel in ComboBoxDelegate: {e}") + items_keys = [] # Fallback on error + else: + log.warning("ComboBoxDelegate: Model is not a UnifiedViewModel or is missing key attributes (_asset_type_keys, _file_type_keys). Dropdown may be empty.") + # --- End key retrieval from model --- + + # REMOVED the entire block that loaded Configuration based on main_window preset + + if items_keys: + for item_key in sorted(items_keys): # Sort keys alphabetically for consistency + # Add item with the key string itself as text and UserData + editor.addItem(item_key, item_key) + else: + # If the delegate is incorrectly applied to another column, + # it will just have the "---" option. + pass + + return editor + + def setEditorData(self, editor: QComboBox, index: QModelIndex): + # Get the current string value (or None) from the model via EditRole. + value = index.model().data(index, Qt.EditRole) # This should be a string or None + + idx = -1 + if value is not None: + # Find the index corresponding to the string value. + idx = editor.findText(value) + else: + # If the model value is None, find the "---" item. + idx = editor.findData(None) # Find the item with UserData == None + + # Set the current index, defaulting to 0 ("---") if not found. + editor.setCurrentIndex(idx if idx != -1 else 0) + + + def setModelData(self, editor: QComboBox, model, index: QModelIndex): + # Get the UserData associated with the currently selected item. + # This will be the string value or None (for the "---" option). + value = editor.currentData() # This is either the string or None + # Pass this string value or None back to the model's setData. + model.setData(index, value, Qt.EditRole) + + def updateEditorGeometry(self, editor, option, index): + editor.setGeometry(option.rect) + +class SupplierSearchDelegate(QStyledItemDelegate): + """ + Delegate for editing supplier names using a QLineEdit with auto-completion. + Loads known suppliers from config/suppliers.json and allows adding new ones. + """ + def __init__(self, parent=None): + super().__init__(parent) + self.known_suppliers = self._load_suppliers() + + def _load_suppliers(self): + """Loads the list of known suppliers from the JSON config file.""" + try: + with open(SUPPLIERS_CONFIG_PATH, 'r') as f: + suppliers = json.load(f) + if isinstance(suppliers, list): + # Ensure all items are strings + return sorted([str(s) for s in suppliers if isinstance(s, str)]) + else: + log.warning(f"'{SUPPLIERS_CONFIG_PATH}' does not contain a valid list. Starting fresh.") + return [] + except FileNotFoundError: + log.info(f"'{SUPPLIERS_CONFIG_PATH}' not found. Starting with an empty supplier list.") + return [] + except json.JSONDecodeError: + log.error(f"Error decoding JSON from '{SUPPLIERS_CONFIG_PATH}'. Starting fresh.", exc_info=True) + return [] + except Exception as e: + log.error(f"An unexpected error occurred loading '{SUPPLIERS_CONFIG_PATH}': {e}", exc_info=True) + return [] + + def _save_suppliers(self): + """Saves the current list of known suppliers back to the JSON config file.""" + try: + # Ensure the directory exists (though write_to_file handled initial creation) + os.makedirs(os.path.dirname(SUPPLIERS_CONFIG_PATH), exist_ok=True) + with open(SUPPLIERS_CONFIG_PATH, 'w') as f: + json.dump(self.known_suppliers, f, indent=4) # Save sorted list with indentation + log.debug(f"Successfully saved updated supplier list to '{SUPPLIERS_CONFIG_PATH}'.") + except IOError as e: + log.error(f"Could not write to '{SUPPLIERS_CONFIG_PATH}': {e}", exc_info=True) + except Exception as e: + log.error(f"An unexpected error occurred saving '{SUPPLIERS_CONFIG_PATH}': {e}", exc_info=True) + + + def createEditor(self, parent, option, index): + """Creates the QLineEdit editor with a QCompleter.""" + editor = QLineEdit(parent) + completer = QCompleter(self.known_suppliers, editor) + completer.setCaseSensitivity(Qt.CaseInsensitive) + completer.setFilterMode(Qt.MatchContains) # More flexible matching + completer.setCompletionMode(QCompleter.PopupCompletion) # Standard popup + editor.setCompleter(completer) + return editor + + def setEditorData(self, editor: QLineEdit, index: QModelIndex): + """Sets the editor's initial data from the model.""" + # Use EditRole as defined in the model's data() method for supplier + value = index.model().data(index, Qt.EditRole) + editor.setText(str(value) if value is not None else "") + + def setModelData(self, editor: QLineEdit, model, index: QModelIndex): + """Commits the editor's data back to the model and handles new suppliers.""" + final_text = editor.text().strip() + value_to_set = final_text if final_text else None # Set None if empty after stripping + + # Set data in the model first + model.setData(index, value_to_set, Qt.EditRole) + + # Add new supplier if necessary + if final_text and final_text not in self.known_suppliers: + log.info(f"Adding new supplier '{final_text}' to known list.") + self.known_suppliers.append(final_text) + self.known_suppliers.sort() # Keep the list sorted + + # Update the completer's model immediately + completer = editor.completer() + if completer: + completer.model().setStringList(self.known_suppliers) + + # Save the updated list back to the file + self._save_suppliers() + + def updateEditorGeometry(self, editor, option, index): + """Ensures the editor widget is placed correctly.""" + editor.setGeometry(option.rect) +class ItemTypeSearchDelegate(QStyledItemDelegate): + """ + Delegate for editing item types using a QLineEdit with auto-completion. + Loads known item types from the provided list. + """ + def __init__(self, item_type_keys: list[str] | None = None, parent=None): + super().__init__(parent) + self.item_type_keys = item_type_keys if item_type_keys else [] + log.debug(f"ItemTypeSearchDelegate initialized with {len(self.item_type_keys)} keys: {self.item_type_keys}") + + def createEditor(self, parent, option, index: QModelIndex): + """Creates the QLineEdit editor with a QCompleter.""" + editor = QLineEdit(parent) + # Use the keys passed during initialization + completer = QCompleter(self.item_type_keys, editor) + completer.setCaseSensitivity(Qt.CaseInsensitive) + completer.setFilterMode(Qt.MatchContains) + completer.setCompletionMode(QCompleter.PopupCompletion) + editor.setCompleter(completer) + return editor + + def setEditorData(self, editor: QLineEdit, index: QModelIndex): + """Sets the editor's initial data from the model.""" + # Use EditRole as defined in the model's data() method for item type override + value = index.model().data(index, Qt.EditRole) + editor.setText(str(value) if value is not None else "") + + def setModelData(self, editor: QLineEdit, model, index: QModelIndex): + """Commits the editor's data back to the model.""" + final_text = editor.text().strip() + value_to_set = final_text if final_text else None # Set None if empty after stripping + + # Set data in the model + # The model's setData handles updating the override and item_type + model.setData(index, value_to_set, Qt.EditRole) + # DO NOT add to a persistent list or save back to config + + def updateEditorGeometry(self, editor, option, index): + """Ensures the editor widget is placed correctly.""" + editor.setGeometry(option.rect) \ No newline at end of file diff --git a/gui/llm_editor_widget.py b/gui/llm_editor_widget.py new file mode 100644 index 0000000..48377b0 --- /dev/null +++ b/gui/llm_editor_widget.py @@ -0,0 +1,350 @@ +# gui/llm_editor_widget.py +import json +import logging +import copy # Added for deepcopy +from PySide6.QtWidgets import ( + QWidget, QVBoxLayout, QTabWidget, QPlainTextEdit, QGroupBox, + QHBoxLayout, QPushButton, QFormLayout, QLineEdit, QDoubleSpinBox, + QSpinBox, QMessageBox, QTextEdit +) +from PySide6.QtCore import Slot as pyqtSlot, Signal as pyqtSignal # Use PySide6 equivalents + +# Assuming configuration module exists and has relevant functions later +from configuration import save_llm_config, ConfigurationError +# For now, define path directly for initial structure +LLM_CONFIG_PATH = "config/llm_settings.json" + +logger = logging.getLogger(__name__) + +class LLMEditorWidget(QWidget): + """ + Widget for editing LLM settings stored in config/llm_settings.json. + """ + settings_saved = pyqtSignal() # Signal emitted when settings are successfully saved + + def __init__(self, parent=None): + super().__init__(parent) + self._unsaved_changes = False + self.original_llm_settings = {} # Initialize original_llm_settings + self._init_ui() + self._connect_signals() + self.save_button.setEnabled(False) # Initially disabled + + def _init_ui(self): + """Initialize the user interface components.""" + main_layout = QVBoxLayout(self) + + # --- Main Tab Widget --- + self.tab_widget = QTabWidget() + main_layout.addWidget(self.tab_widget) + + # --- Tab 1: Prompt Settings --- + self.tab_prompt = QWidget() + prompt_layout = QVBoxLayout(self.tab_prompt) + self.tab_widget.addTab(self.tab_prompt, "Prompt Settings") + + self.prompt_editor = QPlainTextEdit() + self.prompt_editor.setPlaceholderText("Enter the main LLM predictor prompt here...") + prompt_layout.addWidget(self.prompt_editor) + + # Examples GroupBox + examples_groupbox = QGroupBox("Examples") + examples_layout = QVBoxLayout(examples_groupbox) + prompt_layout.addWidget(examples_groupbox) + + self.examples_tab_widget = QTabWidget() + self.examples_tab_widget.setTabsClosable(True) + examples_layout.addWidget(self.examples_tab_widget) + + example_button_layout = QHBoxLayout() + examples_layout.addLayout(example_button_layout) + + self.add_example_button = QPushButton("Add Example") + example_button_layout.addWidget(self.add_example_button) + + self.delete_example_button = QPushButton("Delete Current Example") + example_button_layout.addWidget(self.delete_example_button) + example_button_layout.addStretch() + + + # --- Tab 2: API Settings --- + self.tab_api = QWidget() + api_layout = QFormLayout(self.tab_api) + self.tab_widget.addTab(self.tab_api, "API Settings") + + self.endpoint_url_edit = QLineEdit() + api_layout.addRow("Endpoint URL:", self.endpoint_url_edit) + + self.api_key_edit = QLineEdit() + self.api_key_edit.setEchoMode(QLineEdit.Password) + api_layout.addRow("API Key:", self.api_key_edit) + + self.model_name_edit = QLineEdit() + api_layout.addRow("Model Name:", self.model_name_edit) + + self.temperature_spinbox = QDoubleSpinBox() + self.temperature_spinbox.setRange(0.0, 2.0) + self.temperature_spinbox.setSingleStep(0.1) + self.temperature_spinbox.setDecimals(2) + api_layout.addRow("Temperature:", self.temperature_spinbox) + + self.timeout_spinbox = QSpinBox() + self.timeout_spinbox.setRange(1, 600) + self.timeout_spinbox.setSuffix(" s") + api_layout.addRow("Request Timeout:", self.timeout_spinbox) + + # --- Save Button --- + save_button_layout = QHBoxLayout() + main_layout.addLayout(save_button_layout) + save_button_layout.addStretch() + self.save_button = QPushButton("Save LLM Settings") + save_button_layout.addWidget(self.save_button) + + self.setLayout(main_layout) + + def _connect_signals(self): + """Connect signals to slots.""" + self.save_button.clicked.connect(self._save_settings) + + self.prompt_editor.textChanged.connect(self._mark_unsaved) + self.endpoint_url_edit.textChanged.connect(self._mark_unsaved) + self.api_key_edit.textChanged.connect(self._mark_unsaved) + self.model_name_edit.textChanged.connect(self._mark_unsaved) + self.temperature_spinbox.valueChanged.connect(self._mark_unsaved) + self.timeout_spinbox.valueChanged.connect(self._mark_unsaved) + + self.add_example_button.clicked.connect(self._add_example_tab) + self.delete_example_button.clicked.connect(self._delete_current_example_tab) + self.examples_tab_widget.tabCloseRequested.connect(self._remove_example_tab) + + # Note: Connecting textChanged for example editors needs to happen + # when the tabs/editors are created (in load_settings and _add_example_tab) + + @pyqtSlot() + def load_settings(self): + """Load settings from the JSON file and populate the UI.""" + logger.info(f"Attempting to load LLM settings from {LLM_CONFIG_PATH}") + self.setEnabled(True) # Enable widget before trying to load + + # Clear previous examples + while self.examples_tab_widget.count() > 0: + self.examples_tab_widget.removeTab(0) + + try: + with open(LLM_CONFIG_PATH, 'r', encoding='utf-8') as f: + settings = json.load(f) + self.original_llm_settings = copy.deepcopy(settings) # Store a deep copy + + # Populate Prompt Settings + self.prompt_editor.setPlainText(settings.get("llm_predictor_prompt", "")) + + # Populate Examples + examples = settings.get("llm_predictor_examples", []) + for i, example in enumerate(examples): + try: + example_text = json.dumps(example, indent=4) + example_editor = QTextEdit() + example_editor.setPlainText(example_text) + example_editor.textChanged.connect(self._mark_unsaved) + self.examples_tab_widget.addTab(example_editor, f"Example {i+1}") + except TypeError as e: + logger.error(f"Error formatting example {i+1}: {e}. Skipping.") + QMessageBox.warning(self, "Load Error", f"Could not format example {i+1}. It might be invalid.\nError: {e}") + + + # Populate API Settings + self.endpoint_url_edit.setText(settings.get("llm_endpoint_url", "")) + self.api_key_edit.setText(settings.get("llm_api_key", "")) # Consider security implications + self.model_name_edit.setText(settings.get("llm_model_name", "")) + self.temperature_spinbox.setValue(settings.get("llm_temperature", 0.7)) + self.timeout_spinbox.setValue(settings.get("llm_request_timeout", 120)) + + logger.info("LLM settings loaded successfully.") + + except FileNotFoundError: + logger.warning(f"LLM settings file not found: {LLM_CONFIG_PATH}. Using defaults.") + QMessageBox.warning(self, "Load Error", + f"LLM settings file not found:\n{LLM_CONFIG_PATH}\n\nNew settings will be created if you save.") + # Reset to defaults (optional, or leave fields empty) + self.prompt_editor.clear() + self.endpoint_url_edit.clear() + self.api_key_edit.clear() + self.model_name_edit.clear() + self.temperature_spinbox.setValue(0.7) + self.timeout_spinbox.setValue(120) + self.original_llm_settings = {} # Start with empty original settings if file not found + + except json.JSONDecodeError as e: + logger.error(f"Error decoding JSON from {LLM_CONFIG_PATH}: {e}") + QMessageBox.critical(self, "Load Error", + f"Failed to parse LLM settings file:\n{LLM_CONFIG_PATH}\n\nError: {e}\n\nPlease check the file for syntax errors. Editor will be disabled.") + self.setEnabled(False) # Disable editor on critical load error + self.original_llm_settings = {} # Reset original settings on JSON error + + except Exception as e: # Catch other potential errors during loading/populating + logger.error(f"An unexpected error occurred loading LLM settings: {e}", exc_info=True) + QMessageBox.critical(self, "Load Error", + f"An unexpected error occurred while loading settings:\n{e}\n\nEditor will be disabled.") + self.setEnabled(False) + self.original_llm_settings = {} # Reset original settings on other errors + + + # Reset unsaved changes flag and disable save button after loading + self.save_button.setEnabled(False) + self._unsaved_changes = False + + @pyqtSlot() + def _mark_unsaved(self): + """Mark settings as having unsaved changes and enable the save button.""" + if not self._unsaved_changes: + self._unsaved_changes = True + self.save_button.setEnabled(True) + logger.debug("Unsaved changes marked.") + + @pyqtSlot() + def _save_settings(self): + """Gather data from UI, save to JSON file, and handle errors.""" + logger.info("Attempting to save LLM settings...") + + # 1.a. Load Current Target File + target_file_content = {} + try: + with open(LLM_CONFIG_PATH, 'r', encoding='utf-8') as f: + target_file_content = json.load(f) + except FileNotFoundError: + logger.info(f"{LLM_CONFIG_PATH} not found. Will create a new one.") + target_file_content = {} # Start with an empty dict if file doesn't exist + except json.JSONDecodeError as e: + logger.error(f"Error decoding existing {LLM_CONFIG_PATH}: {e}. Starting with an empty config for save.") + QMessageBox.warning(self, "Warning", + f"Could not parse existing LLM settings file ({LLM_CONFIG_PATH}).\n" + f"Any pre-existing settings in that file might be overwritten if you save now.\nError: {e}") + target_file_content = {} # Start fresh if current file is corrupt + + # 1.b. Gather current UI settings into current_llm_settings + current_llm_settings = {} + parsed_examples = [] + has_errors = False # For example parsing + + current_llm_settings["llm_endpoint_url"] = self.endpoint_url_edit.text().strip() + current_llm_settings["llm_api_key"] = self.api_key_edit.text() # Keep as is + current_llm_settings["llm_model_name"] = self.model_name_edit.text().strip() + current_llm_settings["llm_temperature"] = self.temperature_spinbox.value() + current_llm_settings["llm_request_timeout"] = self.timeout_spinbox.value() + current_llm_settings["llm_predictor_prompt"] = self.prompt_editor.toPlainText().strip() + + for i in range(self.examples_tab_widget.count()): + example_editor = self.examples_tab_widget.widget(i) + if isinstance(example_editor, QTextEdit): + example_text = example_editor.toPlainText().strip() + if not example_text: + continue + try: + parsed_example = json.loads(example_text) + parsed_examples.append(parsed_example) + except json.JSONDecodeError as e: + has_errors = True + tab_name = self.examples_tab_widget.tabText(i) + logger.warning(f"Invalid JSON in '{tab_name}': {e}. Skipping example.") + QMessageBox.warning(self, "Invalid Example", + f"The content in '{tab_name}' is not valid JSON and will not be saved.\n\nError: {e}\n\nPlease correct it or remove the tab.") + else: + logger.warning(f"Widget at index {i} in examples tab is not a QTextEdit. Skipping.") + + if has_errors: + logger.warning("LLM settings not saved due to invalid JSON in examples.") + return + + current_llm_settings["llm_predictor_examples"] = parsed_examples + + # 1.c. Identify Changes and Update Target File Content + changed_settings_count = 0 + for key, current_value in current_llm_settings.items(): + original_value = self.original_llm_settings.get(key) + + # Special handling for lists (e.g., examples) - direct comparison works + # For other types, direct comparison also works. + # This includes new keys present in current_llm_settings but not in original_llm_settings + if key not in self.original_llm_settings or current_value != original_value: + target_file_content[key] = current_value + logger.debug(f"Setting '{key}' changed or added. Old: '{original_value}', New: '{current_value}'") + changed_settings_count +=1 + + if changed_settings_count == 0 and self._unsaved_changes: + logger.info("Save called, but no actual changes detected compared to original loaded settings.") + # If _unsaved_changes was true, it means UI interaction happened, + # but values might have been reverted to original. + # We still proceed to save target_file_content as it might contain + # values from a file that was modified externally since last load. + # Or, if the file didn't exist, it will now be created with current UI values. + + # 1.d. Save Updated Content + try: + save_llm_config(target_file_content) # Save the potentially modified target_file_content + QMessageBox.information(self, "Save Successful", f"LLM settings saved to:\n{LLM_CONFIG_PATH}") + + # Update original_llm_settings to reflect the newly saved state + self.original_llm_settings = copy.deepcopy(target_file_content) + + self.save_button.setEnabled(False) + self._unsaved_changes = False + self.settings_saved.emit() + logger.info("LLM settings saved successfully.") + + except ConfigurationError as e: + logger.error(f"Failed to save LLM settings: {e}") + QMessageBox.critical(self, "Save Error", f"Could not save LLM settings.\n\nError: {e}") + self.save_button.setEnabled(True) # Keep save enabled + self._unsaved_changes = True + except Exception as e: + logger.error(f"An unexpected error occurred during LLM settings save: {e}", exc_info=True) + QMessageBox.critical(self, "Save Error", f"An unexpected error occurred while saving settings:\n{e}") + self.save_button.setEnabled(True) # Keep save enabled + self._unsaved_changes = True + + # --- Example Management Slots --- + @pyqtSlot() + def _add_example_tab(self): + """Add a new, empty tab for an LLM example.""" + logger.debug("Adding new example tab.") + new_example_editor = QTextEdit() + new_example_editor.setPlaceholderText("Enter example JSON here...") + new_example_editor.textChanged.connect(self._mark_unsaved) + + # Determine the next example number + next_example_num = self.examples_tab_widget.count() + 1 + index = self.examples_tab_widget.addTab(new_example_editor, f"Example {next_example_num}") + self.examples_tab_widget.setCurrentIndex(index) # Focus the new tab + new_example_editor.setFocus() # Focus the editor within the tab + + self._mark_unsaved() # Mark changes since we added a tab + + @pyqtSlot() + def _delete_current_example_tab(self): + """Delete the currently selected example tab.""" + current_index = self.examples_tab_widget.currentIndex() + if current_index != -1: # Check if a tab is selected + logger.debug(f"Deleting current example tab at index {current_index}.") + self._remove_example_tab(current_index) # Reuse the remove logic + else: + logger.debug("Delete current example tab called, but no tab is selected.") + + @pyqtSlot(int) + def _remove_example_tab(self, index): + """Remove the example tab at the given index.""" + if 0 <= index < self.examples_tab_widget.count(): + widget_to_remove = self.examples_tab_widget.widget(index) + self.examples_tab_widget.removeTab(index) + if widget_to_remove: + # Disconnect signals if necessary, though Python's GC should handle it + # widget_to_remove.textChanged.disconnect(self._mark_unsaved) # Optional cleanup + widget_to_remove.deleteLater() # Ensure proper cleanup of the widget + logger.debug(f"Removed example tab at index {index}.") + + # Renumber subsequent tabs + for i in range(index, self.examples_tab_widget.count()): + self.examples_tab_widget.setTabText(i, f"Example {i+1}") + + self._mark_unsaved() # Mark changes since we removed a tab + else: + logger.warning(f"Attempted to remove example tab at invalid index {index}.") \ No newline at end of file diff --git a/gui/llm_interaction_handler.py b/gui/llm_interaction_handler.py new file mode 100644 index 0000000..b67cf24 --- /dev/null +++ b/gui/llm_interaction_handler.py @@ -0,0 +1,417 @@ +import os +import json +import logging +from pathlib import Path + +from PySide6.QtCore import QObject, Signal, QThread, Slot, QTimer + +# --- Backend Imports --- +# Assuming these might be needed based on MainWindow's usage +try: + from configuration import ConfigurationError # Keep error class + from .llm_prediction_handler import LLMPredictionHandler # Backend handler + from rule_structure import SourceRule # For signal emission type hint +except ImportError as e: + logging.getLogger(__name__).critical(f"Failed to import backend modules for LLMInteractionHandler: {e}") + LLMPredictionHandler = None + ConfigurationError = Exception + SourceRule = None # Define as None if import fails + +log = logging.getLogger(__name__) +# Define config file paths relative to this handler's location +CONFIG_DIR = Path(__file__).parent.parent / "config" +APP_SETTINGS_PATH = CONFIG_DIR / "app_settings.json" +LLM_SETTINGS_PATH = CONFIG_DIR / "llm_settings.json" + +class LLMInteractionHandler(QObject): + """ + Handles the logic for interacting with the LLM prediction service, + including managing the queue, thread, and communication. + """ + # Signals to communicate results/status back to MainWindow or other components + llm_prediction_ready = Signal(str, list) # input_path, List[SourceRule] + llm_prediction_error = Signal(str, str) # input_path, error_message + llm_status_update = Signal(str) # status_message + llm_processing_state_changed = Signal(bool) # is_processing (True when busy, False when idle) + + def __init__(self, main_window_ref, parent=None): + """ + Initializes the handler. + + Args: + main_window_ref: A reference to the MainWindow instance for accessing + shared components like status bar or models if needed. + parent: The parent QObject. + """ + super().__init__(parent) + self.main_window = main_window_ref # Store reference if needed for status updates etc. + self.llm_processing_queue = [] # Unified queue for initial adds and re-interpretations + self.llm_prediction_thread = None + self.llm_prediction_handler = None + self._is_processing = False # Internal flag to track processing state + + def _set_processing_state(self, processing: bool): + """Updates the internal processing state and emits a signal.""" + if self._is_processing != processing: + self._is_processing = processing + log.debug(f"LLM Handler processing state changed to: {processing}") + self.llm_processing_state_changed.emit(processing) + + def force_reset_state(self): + """Forces the processing state to False. Use with caution.""" + log.warning("Forcing LLMInteractionHandler state reset.") + if self.llm_prediction_thread and self.llm_prediction_thread.isRunning(): + log.warning("Force reset called while thread is running. Attempting to stop thread.") + # Attempt graceful shutdown first + self.llm_prediction_thread.quit() + if not self.llm_prediction_thread.wait(500): # Wait 0.5 sec + log.warning("LLM thread did not quit gracefully after force reset. Terminating.") + self.llm_prediction_thread.terminate() + self.llm_prediction_thread.wait() # Wait after terminate + self.llm_prediction_thread = None + self.llm_prediction_handler = None + self._set_processing_state(False) + # Do NOT clear the queue here, let the user decide via Clear Queue button + + @Slot(str, list) + def queue_llm_request(self, input_path: str, file_list: list | None): + """Adds a request to the LLM processing queue.""" + log.debug(f"Queueing LLM request for '{input_path}'. Current queue size: {len(self.llm_processing_queue)}") + # Avoid duplicates? Check if already in queue + is_in_queue = any(item[0] == input_path for item in self.llm_processing_queue) + if not is_in_queue: + self.llm_processing_queue.append((input_path, file_list)) + log.info(f"Added '{input_path}' to LLM queue. New size: {len(self.llm_processing_queue)}") + # If not currently processing, start the queue + if not self._is_processing: + # Use QTimer.singleShot to avoid immediate processing if called rapidly + QTimer.singleShot(0, self._process_next_llm_item) + else: + log.debug(f"Skipping duplicate add to LLM queue for: {input_path}") + + @Slot(list) + def queue_llm_requests_batch(self, requests: list[tuple[str, list | None]]): + """Adds multiple requests to the LLM processing queue.""" + added_count = 0 + log.debug(f"Queueing batch. Current queue content: {self.llm_processing_queue}") + for input_path, file_list in requests: + is_in_queue = any(item[0] == input_path for item in self.llm_processing_queue) + if not is_in_queue: + self.llm_processing_queue.append((input_path, file_list)) + added_count += 1 + else: + log.debug(f"Skipping duplicate add to LLM queue for: {input_path}") + + if added_count > 0: + log.info(f"Added {added_count} requests to LLM queue. New size: {len(self.llm_processing_queue)}") + if not self._is_processing: + QTimer.singleShot(0, self._process_next_llm_item) + + # --- Methods to be moved from MainWindow --- + + @Slot() + def _reset_llm_thread_references(self): + """Resets LLM thread and handler references after the thread finishes.""" + log.debug("--> Entered LLMInteractionHandler._reset_llm_thread_references") + log.debug("Resetting LLM prediction thread and handler references.") + self.llm_prediction_thread = None + self.llm_prediction_handler = None + # --- Process next item now that the previous thread is fully finished --- + log.debug("Previous LLM thread finished. Setting processing state to False.") + self._set_processing_state(False) # Mark processing as finished + # The next item will be processed when _handle_llm_result or _handle_llm_error + # calls _process_next_llm_item after popping the completed item. + log.debug("<-- Exiting LLMInteractionHandler._reset_llm_thread_references") + + + def _start_llm_prediction(self, input_path_str: str, file_list: list = None): + """ + Sets up and starts the LLMPredictionHandler in a separate thread. + Emits signals for results, errors, or status updates. + If file_list is not provided, it will be extracted. + """ + log.debug(f"Attempting to start LLM prediction for: {input_path_str}") + # Extract file list if not provided (needed for re-interpretation calls) + if file_list is None: + log.debug(f"File list not provided for {input_path_str}, extracting...") + if hasattr(self.main_window, '_extract_file_list'): + file_list = self.main_window._extract_file_list(input_path_str) + if file_list is None: + error_msg = f"Failed to extract file list for {input_path_str} in _start_llm_prediction." + log.error(error_msg) + self.llm_status_update.emit(f"Error extracting files for {os.path.basename(input_path_str)}") + self.llm_prediction_error.emit(input_path_str, error_msg) # Signal error + return # Stop if extraction failed + else: + error_msg = f"MainWindow reference does not have _extract_file_list method." + log.error(error_msg) + self.llm_status_update.emit(f"Internal Error: Cannot extract files for {os.path.basename(input_path_str)}") + self.llm_prediction_error.emit(input_path_str, error_msg) + return # Stop + + input_path_obj = Path(input_path_str) # Still needed for basename + + if not file_list: + error_msg = f"LLM Error: No files found/extracted for {input_path_str}" + log.error(error_msg) + self.llm_status_update.emit(f"LLM Error: No files found for {input_path_obj.name}") + self.llm_prediction_error.emit(input_path_str, error_msg) + return + + # --- Load Required Settings Directly --- + llm_settings = {} + try: + log.debug(f"Loading LLM settings from: {LLM_SETTINGS_PATH}") + with open(LLM_SETTINGS_PATH, 'r') as f: + llm_data = json.load(f) + # Extract required fields with defaults + llm_settings['endpoint_url'] = llm_data.get('llm_endpoint_url') + llm_settings['api_key'] = llm_data.get('llm_api_key') # Can be None + llm_settings['model_name'] = llm_data.get('llm_model_name', 'local-model') + llm_settings['temperature'] = llm_data.get('llm_temperature', 0.5) + llm_settings['request_timeout'] = llm_data.get('llm_request_timeout', 120) + llm_settings['predictor_prompt'] = llm_data.get('llm_predictor_prompt', '') + llm_settings['examples'] = llm_data.get('llm_examples', []) + + log.debug(f"Loading App settings from: {APP_SETTINGS_PATH}") + with open(APP_SETTINGS_PATH, 'r') as f: + app_data = json.load(f) + # Extract required fields + llm_settings['asset_type_definitions'] = app_data.get('ASSET_TYPE_DEFINITIONS', {}) + llm_settings['file_type_definitions'] = app_data.get('FILE_TYPE_DEFINITIONS', {}) + + # Validate essential settings + if not llm_settings['endpoint_url']: + raise ValueError("LLM endpoint URL is missing in llm_settings.json") + if not llm_settings['predictor_prompt']: + raise ValueError("LLM predictor prompt is missing in llm_settings.json") + + log.debug("LLM and App settings loaded successfully for LLMInteractionHandler.") + + except FileNotFoundError as e: + error_msg = f"LLM Error: Configuration file not found: {e.filename}" + log.critical(error_msg) + self.llm_status_update.emit("LLM Error: Cannot load configuration file.") + self.llm_prediction_error.emit(input_path_str, error_msg) + return + except json.JSONDecodeError as e: + error_msg = f"LLM Error: Failed to parse configuration file: {e}" + log.critical(error_msg) + self.llm_status_update.emit("LLM Error: Cannot parse configuration file.") + self.llm_prediction_error.emit(input_path_str, error_msg) + return + except ValueError as e: # Catch validation errors + error_msg = f"LLM Error: Invalid configuration - {e}" + log.critical(error_msg) + self.llm_status_update.emit("LLM Error: Invalid configuration.") + self.llm_prediction_error.emit(input_path_str, error_msg) + return + except Exception as e: # Catch other potential errors + error_msg = f"LLM Error: Unexpected error loading configuration: {e}" + log.critical(error_msg, exc_info=True) + self.llm_status_update.emit("LLM Error: Cannot load application configuration.") + self.llm_prediction_error.emit(input_path_str, error_msg) + return + + # --- Wrap thread/handler setup and start in try...except --- + try: + # --- Check if Handler Class is Available --- + if LLMPredictionHandler is None: + # Raise ValueError to be caught below + raise ValueError("LLMPredictionHandler class not available.") + + # --- Clean up previous thread/handler if necessary --- + # (Keep this cleanup logic as it handles potential stale threads) + if self.llm_prediction_thread and self.llm_prediction_thread.isRunning(): + log.warning("Warning: Previous LLM prediction thread still running when trying to start new one. Attempting cleanup.") + if self.llm_prediction_handler: + if hasattr(self.llm_prediction_handler, 'cancel'): + self.llm_prediction_handler.cancel() + self.llm_prediction_thread.quit() + if not self.llm_prediction_thread.wait(1000): # Wait 1 sec + log.warning("LLM thread did not quit gracefully. Forcing termination.") + self.llm_prediction_thread.terminate() + self.llm_prediction_thread.wait() # Wait after terminate + self.llm_prediction_thread = None + self.llm_prediction_handler = None + + log.info(f"Starting LLM prediction thread for source: {input_path_str} with {len(file_list)} files.") + self.llm_status_update.emit(f"Starting LLM interpretation for {input_path_obj.name}...") + + # --- Create Thread and Handler --- + self.llm_prediction_thread = QThread(self) # Parent thread to self + # Pass the loaded settings dictionary + self.llm_prediction_handler = LLMPredictionHandler(input_path_str, file_list, llm_settings) + self.llm_prediction_handler.moveToThread(self.llm_prediction_thread) + + # Connect signals from handler to *internal* slots or directly emit signals + self.llm_prediction_handler.prediction_ready.connect(self._handle_llm_result) + self.llm_prediction_handler.prediction_error.connect(self._handle_llm_error) + self.llm_prediction_handler.status_update.connect(self.llm_status_update) # Pass status through + + # Connect thread signals + self.llm_prediction_thread.started.connect(self.llm_prediction_handler.run) + # Clean up thread and handler when finished + self.llm_prediction_thread.finished.connect(self._reset_llm_thread_references) + self.llm_prediction_thread.finished.connect(self.llm_prediction_handler.deleteLater) + self.llm_prediction_thread.finished.connect(self.llm_prediction_thread.deleteLater) + # Also ensure thread quits when handler signals completion/error + self.llm_prediction_handler.prediction_ready.connect(self.llm_prediction_thread.quit) + self.llm_prediction_handler.prediction_error.connect(self.llm_prediction_thread.quit) + + # TODO: Add a logging.debug statement at the very beginning of LLMPredictionHandler.run() + # to confirm if the method is being reached. Example: + # log.debug(f"--> Entered LLMPredictionHandler.run() for {self.input_path}") + + self.llm_prediction_thread.start() + log.debug(f"LLM prediction thread start() called for {input_path_str}. Is running: {self.llm_prediction_thread.isRunning()}") + # Log success *after* start() is called successfully + log.debug(f"Successfully initiated LLM prediction thread for {input_path_str}.") + + except Exception as e: + # --- Handle errors during setup/start --- + log.exception(f"Critical error during LLM thread setup/start for {input_path_str}: {e}") + error_msg = f"Error initializing LLM task for {input_path_obj.name}: {e}" + self.llm_status_update.emit(error_msg) + self.llm_prediction_error.emit(input_path_str, error_msg) # Signal the error + + # --- Crucially, reset processing state if setup fails --- + log.warning("Resetting processing state due to thread setup/start error.") + self._set_processing_state(False) + + # Clean up potentially partially created objects + if self.llm_prediction_handler: + self.llm_prediction_handler.deleteLater() + self.llm_prediction_handler = None + if self.llm_prediction_thread: + if self.llm_prediction_thread.isRunning(): + self.llm_prediction_thread.quit() + self.llm_prediction_thread.wait(500) + self.llm_prediction_thread.terminate() # Force if needed + self.llm_prediction_thread.wait() + self.llm_prediction_thread.deleteLater() + self.llm_prediction_thread = None + + # Do NOT automatically try the next item here, as the error might be persistent. + # Let the error signal handle popping the item and trying the next one. + # The error signal (_handle_llm_error) will pop the item and call _process_next_llm_item. + + + def is_processing(self) -> bool: + """Safely checks if the LLM prediction thread is currently running.""" + # Use the internal flag, which is more reliable than checking thread directly + # due to potential race conditions during cleanup. + # The thread check can be a fallback. + is_running_flag = self._is_processing + # Also check thread as a safeguard, though the flag should be primary + try: + is_thread_alive = self.llm_prediction_thread is not None and self.llm_prediction_thread.isRunning() + if is_running_flag != is_thread_alive: + # This might indicate the flag wasn't updated correctly, log it. + log.warning(f"LLM Handler processing flag ({is_running_flag}) mismatch with thread state ({is_thread_alive}). Flag is primary.") + return is_running_flag + except RuntimeError: + log.debug("is_processing: Caught RuntimeError checking isRunning (thread likely deleted).") + # If thread died unexpectedly, the flag might be stale. Reset it. + if self._is_processing: + self._set_processing_state(False) + return False + + + def _process_next_llm_item(self): + """Processes the next directory in the unified LLM processing queue.""" + log.debug(f"--> Entered _process_next_llm_item. Queue size: {len(self.llm_processing_queue)}") + + if self.is_processing(): + log.info("LLM processing already running. Waiting for current item to finish.") + # Do not pop from queue if already running, wait for _reset_llm_thread_references to call this again + return + + if not self.llm_processing_queue: + log.info("LLM processing queue is empty. Finishing.") + self.llm_status_update.emit("LLM processing complete.") + self._set_processing_state(False) # Ensure state is set to idle + log.debug("<-- Exiting _process_next_llm_item (queue empty)") + return + + # Set state to busy *before* starting + self._set_processing_state(True) + + # Get next item *without* removing it yet + next_item = self.llm_processing_queue[0] # Peek at the first item + next_dir, file_list = next_item # Unpack the tuple + + # --- Update Status/Progress --- + total_in_queue_now = len(self.llm_processing_queue) + status_msg = f"LLM Processing {os.path.basename(next_dir)} ({total_in_queue_now} remaining)..." + self.llm_status_update.emit(status_msg) + log.info(status_msg) + + # --- Start Prediction (which might fail) --- + try: + # Pass the potentially None file_list. _start_llm_prediction handles extraction if needed. + self._start_llm_prediction(next_dir, file_list=file_list) + # --- DO NOT pop item here. Item is popped in _handle_llm_result or _handle_llm_error --- + except Exception as e: + # This block now catches errors from _start_llm_prediction itself + log.exception(f"Error occurred *during* _start_llm_prediction call for {next_dir}: {e}") + error_msg = f"Error starting LLM for {os.path.basename(next_dir)}: {e}" + self.llm_status_update.emit(error_msg) + self.llm_prediction_error.emit(next_dir, error_msg) # Signal the error + # --- Remove the failed item from the queue --- + try: + failed_item = self.llm_processing_queue.pop(0) + log.warning(f"Removed failed item {failed_item} from LLM queue due to start error.") + except IndexError: + log.error("Attempted to pop failed item from already empty LLM queue after start error.") + # --- Attempt to process the *next* item --- + # Reset processing state since this one failed *before* the thread finished signal could + self._set_processing_state(False) + # Use QTimer.singleShot to avoid deep recursion + QTimer.singleShot(100, self._process_next_llm_item) # Try next item after a short delay + + # --- Internal Slots to Handle Results/Errors from LLMPredictionHandler --- + @Slot(str, list) + def _handle_llm_result(self, input_path: str, source_rules: list): + """Internal slot to receive results, pop item, and emit the public signal.""" + log.debug(f"LLM Handler received result for {input_path}. Removing from queue and emitting llm_prediction_ready.") + # Remove the completed item from the queue + try: + # Find and remove the item by input_path + self.llm_processing_queue = [item for item in self.llm_processing_queue if item[0] != input_path] + log.debug(f"Removed '{input_path}' from LLM queue after successful prediction. New size: {len(self.llm_processing_queue)}") + except Exception as e: + log.error(f"Error removing '{input_path}' from LLM queue after success: {e}") + + self.llm_prediction_ready.emit(input_path, source_rules) + + # Process the next item in the queue + QTimer.singleShot(0, self._process_next_llm_item) + + @Slot(str, str) + def _handle_llm_error(self, input_path: str, error_message: str): + """Internal slot to receive errors, pop item, and emit the public signal.""" + log.debug(f"LLM Handler received error for {input_path}: {error_message}. Removing from queue and emitting llm_prediction_error.") + # Remove the failed item from the queue + try: + # Find and remove the item by input_path + self.llm_processing_queue = [item for item in self.llm_processing_queue if item[0] != input_path] + log.debug(f"Removed '{input_path}' from LLM queue after error. New size: {len(self.llm_processing_queue)}") + except Exception as e: + log.error(f"Error removing '{input_path}' from LLM queue after error: {e}") + + self.llm_prediction_error.emit(input_path, error_message) + + # Process the next item in the queue + QTimer.singleShot(0, self._process_next_llm_item) + + def clear_queue(self): + """Clears the LLM processing queue.""" + log.info(f"Clearing LLM processing queue ({len(self.llm_processing_queue)} items).") + self.llm_processing_queue.clear() + # TODO: Should we also attempt to cancel any *currently* running LLM task? + # This might be complex. For now, just clears the queue of pending items. + if self.is_processing(): + log.warning("LLM queue cleared, but a task is currently running. It will complete.") + else: + self.llm_status_update.emit("LLM queue cleared.") \ No newline at end of file diff --git a/gui/llm_prediction_handler.py b/gui/llm_prediction_handler.py new file mode 100644 index 0000000..f995927 --- /dev/null +++ b/gui/llm_prediction_handler.py @@ -0,0 +1,397 @@ +import os +import json +import requests +import re +import logging +from pathlib import Path +from PySide6.QtCore import QObject, Slot +# Removed Signal, QThread as they are handled by BasePredictionHandler or caller +from typing import List, Dict, Any + +# Assuming rule_structure defines SourceRule, AssetRule, FileRule etc. +# Adjust the import path if necessary based on project structure +from rule_structure import SourceRule, AssetRule, FileRule + +# Assuming configuration loads app_settings.json +# Adjust the import path if necessary +# Removed Configuration import +from .base_prediction_handler import BasePredictionHandler + +log = logging.getLogger(__name__) + +class LLMPredictionHandler(BasePredictionHandler): + """ + Handles the interaction with an LLM for predicting asset structures + based on a directory's file list. Inherits from BasePredictionHandler. + """ + # Signals (prediction_ready, prediction_error, status_update) are inherited + + # Changed 'config: Configuration' to 'settings: dict' + def __init__(self, input_source_identifier: str, file_list: list, settings: dict, parent: QObject = None): + """ + Initializes the LLM handler. + + Args: + input_source_identifier: The unique identifier for the input source (e.g., file path). + file_list: A list of *relative* file paths extracted from the input source. + (LLM expects relative paths based on the prompt template). + settings: A dictionary containing required LLM and App settings. + parent: The parent QObject. + """ + super().__init__(input_source_identifier, parent) + # input_source_identifier is stored by the base class as self.input_source_identifier + self.file_list = file_list + self.settings = settings + # Access LLM settings via self.settings['key'] + # _is_running and _is_cancelled are handled by the base class + + # The run() and cancel() slots are provided by the base class. + # We only need to implement the core logic in _perform_prediction. + + def _perform_prediction(self) -> List[SourceRule]: + """ + Performs the LLM prediction by preparing the prompt, calling the LLM, + and parsing the response. Implements the abstract method from BasePredictionHandler. + + Returns: + A list containing a single SourceRule object based on the LLM response, + or an empty list if prediction fails or yields no results. + + Raises: + ValueError: If required settings (like endpoint URL or prompt template) are missing. + ConnectionError: If the LLM API call fails due to network issues or timeouts. + Exception: For other errors during prompt preparation, API call, or parsing. + """ + log.debug(f"--> Entered LLMPredictionHandler._perform_prediction() for {self.input_source_identifier}") + log.info(f"Performing LLM prediction for: {self.input_source_identifier}") + base_name = Path(self.input_source_identifier).name + + if not self.file_list: + log.warning(f"No files provided for LLM prediction for {self.input_source_identifier}. Returning empty list.") + self.status_update.emit(f"No files found for {base_name}.") + return [] # Return empty list, not an error + + # Check for cancellation before preparing prompt + if self._is_cancelled: + log.info("LLM prediction cancelled before preparing prompt.") + return [] + + # --- Prepare Prompt --- + self.status_update.emit(f"Preparing LLM input for {base_name}...") + try: + prompt = self._prepare_prompt(self.file_list) + except Exception as e: + log.exception("Error preparing LLM prompt.") + raise ValueError(f"Error preparing LLM prompt: {e}") from e # Re-raise for base handler + + if self._is_cancelled: + log.info("LLM prediction cancelled after preparing prompt.") + return [] + + # --- Call LLM --- + self.status_update.emit(f"Calling LLM for {base_name}...") + try: + llm_response_json_str = self._call_llm(prompt) + except Exception as e: + log.exception("Error calling LLM API.") + # Re-raise potentially specific errors (ConnectionError, ValueError) or a generic one + raise RuntimeError(f"Error calling LLM: {e}") from e + + if self._is_cancelled: + log.info("LLM prediction cancelled after calling LLM.") + return [] + + # --- Parse Response --- + self.status_update.emit(f"Parsing LLM response for {base_name}...") + try: + predicted_rules = self._parse_llm_response(llm_response_json_str) + except Exception as e: + log.exception("Error parsing LLM response.") + raise ValueError(f"Error parsing LLM response: {e}") from e # Re-raise for base handler + + if self._is_cancelled: + log.info("LLM prediction cancelled after parsing response.") + return [] + + log.info(f"LLM prediction finished successfully for '{self.input_source_identifier}'.") + # The base class run() method will emit prediction_ready with these results + return predicted_rules + + + # --- Helper Methods (Keep these internal to this class) --- + + def _prepare_prompt(self, relative_file_list: List[str]) -> str: + """ + Prepares the full prompt string to send to the LLM using stored settings. + """ + prompt_template = self.settings.get('predictor_prompt') + if not prompt_template: + raise ValueError("LLM predictor prompt template content is empty or missing in settings.") + + + asset_defs = json.dumps(self.settings.get('asset_type_definitions', {}), indent=4) + # Combine file type defs and examples (assuming structure from Configuration class) + file_type_defs_combined = {} + file_type_defs = self.settings.get('file_type_definitions', {}) + for key, definition in file_type_defs.items(): + # Add examples if they exist within the definition structure + file_type_defs_combined[key] = { + "description": definition.get("description", ""), + "examples": definition.get("examples", []) + } + file_defs = json.dumps(file_type_defs_combined, indent=4) + examples = json.dumps(self.settings.get('examples', []), indent=2) + + # Format *relative* file list as a single string with newlines + file_list_str = "\n".join(relative_file_list) + + prompt = prompt_template.replace('{ASSET_TYPE_DEFINITIONS}', asset_defs) + prompt = prompt.replace('{FILE_TYPE_DEFINITIONS}', file_defs) + prompt = prompt.replace('{EXAMPLE_INPUT_OUTPUT_PAIRS}', examples) + prompt = prompt.replace('{FILE_LIST}', file_list_str) + + return prompt + + def _call_llm(self, prompt: str) -> str: + """ + Calls the configured LLM API endpoint with the prepared prompt. + + Args: + prompt: The complete prompt string. + + Returns: + The content string from the LLM response, expected to be JSON. + + Raises: + ConnectionError: If the request fails due to network issues or timeouts. + ValueError: If the endpoint URL is not configured or the response is invalid. + requests.exceptions.RequestException: For other request-related errors. + """ + endpoint_url = self.settings.get('endpoint_url') + if not endpoint_url: + raise ValueError("LLM endpoint URL is not configured in settings.") + + headers = { + "Content-Type": "application/json", + } + api_key = self.settings.get('api_key') + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + # Construct payload based on OpenAI Chat Completions format + payload = { + "model": self.settings.get('model_name', 'local-model'), + "messages": [{"role": "user", "content": prompt}], + "temperature": self.settings.get('temperature', 0.5), + # Ensure the LLM is instructed to return JSON in the prompt itself + } + + print(f"--- Calling LLM API: {endpoint_url} ---") + + # Note: Exceptions raised here (Timeout, RequestException, ValueError) + # will be caught by the _perform_prediction method's handler. + + response = requests.post( + endpoint_url, + headers=headers, + json=payload, + timeout=self.settings.get('request_timeout', 120) + ) + response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx) + + response_data = response.json() + + # Extract content - structure depends on the API (OpenAI format assumed) + if "choices" in response_data and len(response_data["choices"]) > 0: + message = response_data["choices"][0].get("message", {}) + content = message.get("content") + if content: + # The content itself should be the JSON string we asked for + log.debug("--- LLM Response Content Extracted Successfully ---") + return content.strip() + else: + raise ValueError("LLM response missing 'content' in choices[0].message.") + else: + raise ValueError("LLM response missing 'choices' array or it's empty.") + + def _parse_llm_response(self, llm_response_json_str: str) -> List[SourceRule]: + """ + Parses the LLM's JSON response string (new two-part format) into a + list containing a single SourceRule object. + Includes sanitization for comments and markdown fences. + """ + # Note: Exceptions (JSONDecodeError, ValueError) raised here + # will be caught by the _perform_prediction method's handler. + + # --- Sanitize Input String --- + clean_json_str = re.sub(r'/\*.*?\*/', '', llm_response_json_str.strip(), flags=re.DOTALL) + + # 2. Remove single-line // comments (handle potential URLs carefully) + # Only remove // if it's likely a comment (e.g., whitespace before it, + # or at the start of a line after stripping leading whitespace). + lines = clean_json_str.splitlines() + cleaned_lines = [] + for line in lines: + stripped_line = line.strip() + # Find the first // that isn't preceded by a : (to avoid breaking URLs like http://) + comment_index = -1 + search_start = 0 + while True: + idx = stripped_line.find('//', search_start) + if idx == -1: + break # No more // found + if idx == 0 or stripped_line[idx-1] != ':': + # Found a potential comment marker + # Check if it's inside quotes + in_quotes = False + quote_char = '' + for i in range(idx): + char = stripped_line[i] + if char in ('"', "'") and (i == 0 or stripped_line[i-1] != '\\'): # Handle escaped quotes + if not in_quotes: + in_quotes = True + quote_char = char + elif char == quote_char: + in_quotes = False + quote_char = '' + if not in_quotes: + comment_index = idx + break # Found valid comment marker + else: + # // is inside quotes, continue searching after it + search_start = idx + 2 + else: + # Found ://, likely a URL, continue searching after it + search_start = idx + 2 + + if comment_index != -1: + # Find the original position in the non-stripped line + original_comment_start = line.find(stripped_line[comment_index:]) + cleaned_lines.append(line[:original_comment_start].rstrip()) + else: + cleaned_lines.append(line) + clean_json_str = "\n".join(cleaned_lines) + + + # 3. Remove markdown code fences + clean_json_str = clean_json_str.strip() + if clean_json_str.startswith("```json"): + clean_json_str = clean_json_str[7:].strip() + if clean_json_str.endswith("```"): + clean_json_str = clean_json_str[:-3].strip() + + # 4. Remove tags (just in case) + clean_json_str = re.sub(r'.*?', '', clean_json_str, flags=re.DOTALL | re.IGNORECASE).strip() + + # --- Parse Sanitized JSON --- + try: + response_data = json.loads(clean_json_str) + except json.JSONDecodeError as e: + error_detail = f"Failed to decode LLM JSON response after sanitization: {e}\nSanitized Response Attempted:\n{clean_json_str}" + log.error(f"ERROR: {error_detail}") + raise ValueError(error_detail) + + # --- Validate Top-Level Structure --- + if not isinstance(response_data, dict): + raise ValueError("Invalid LLM response: Root element is not a JSON object.") + + if "individual_file_analysis" not in response_data or not isinstance(response_data["individual_file_analysis"], list): + raise ValueError("Invalid LLM response format: 'individual_file_analysis' key missing or not a list.") + + if "asset_group_classifications" not in response_data or not isinstance(response_data["asset_group_classifications"], dict): + raise ValueError("Invalid LLM response format: 'asset_group_classifications' key missing or not a dictionary.") + + # --- Prepare for Rule Creation --- + source_rule = SourceRule(input_path=self.input_source_identifier) + valid_asset_types = list(self.settings.get('asset_type_definitions', {}).keys()) + valid_file_types = list(self.settings.get('file_type_definitions', {}).keys()) + asset_rules_map: Dict[str, AssetRule] = {} # Maps group_name to AssetRule + + # --- Process Individual Files and Build Rules --- + for file_data in response_data["individual_file_analysis"]: + # Check for cancellation within the loop + if self._is_cancelled: + log.info("LLM prediction cancelled during response parsing (files).") + return [] + + if not isinstance(file_data, dict): + log.warning(f"Skipping invalid file data entry (not a dict): {file_data}") + continue + + file_path_rel = file_data.get("relative_file_path") + file_type = file_data.get("classified_file_type") + group_name = file_data.get("proposed_asset_group_name") # Can be string or null + + # --- Validate File Data --- + if not file_path_rel or not isinstance(file_path_rel, str): + log.warning(f"Missing or invalid 'relative_file_path' in file data: {file_data}. Skipping file.") + continue + + if not file_type or not isinstance(file_type, str): + log.warning(f"Missing or invalid 'classified_file_type' for file '{file_path_rel}'. Skipping file.") + continue + + # Handle FILE_IGNORE explicitly + if file_type == "FILE_IGNORE": + log.debug(f"Ignoring file as per LLM prediction: {file_path_rel}") + continue # Skip creating a rule for this file + + # Validate file_type against definitions + if file_type not in valid_file_types: + log.warning(f"Invalid predicted_file_type '{file_type}' for file '{file_path_rel}'. Defaulting to EXTRA.") + file_type = "EXTRA" + + # --- Handle Grouping and Asset Type --- + if not group_name or not isinstance(group_name, str): + log.warning(f"File '{file_path_rel}' has missing, null, or invalid 'proposed_asset_group_name' ({group_name}). Cannot assign to an asset. Skipping file.") + continue + + asset_type = response_data["asset_group_classifications"].get(group_name) + + if not asset_type: + log.warning(f"No classification found in 'asset_group_classifications' for group '{group_name}' (proposed for file '{file_path_rel}'). Skipping file.") + continue + + if asset_type not in valid_asset_types: + log.warning(f"Invalid asset_type '{asset_type}' found in 'asset_group_classifications' for group '{group_name}'. Skipping file '{file_path_rel}'.") + continue + + # --- Construct Absolute Path --- + try: + base_path = Path(self.input_source_identifier) + if base_path.is_file(): + base_path = base_path.parent + clean_rel_path = Path(file_path_rel.strip().replace('\\', '/')) + file_path_abs = str(base_path / clean_rel_path) + except Exception as path_e: + log.warning(f"Error constructing absolute path for '{file_path_rel}' relative to '{self.input_source_identifier}': {path_e}. Skipping file.") + continue + + # --- Get or Create Asset Rule --- + asset_rule = asset_rules_map.get(group_name) + if not asset_rule: + # Create new AssetRule if this is the first file for this group + log.debug(f"Creating new AssetRule for group '{group_name}' with type '{asset_type}'.") + asset_rule = AssetRule(asset_name=group_name, asset_type=asset_type) + source_rule.assets.append(asset_rule) + asset_rules_map[group_name] = asset_rule + + # --- Create and Add File Rule --- + file_rule = FileRule( + file_path=file_path_abs, + item_type=file_type, + item_type_override=file_type, # Initial override based on LLM + target_asset_name_override=group_name, + output_format_override=None, + resolution_override=None, + channel_merge_instructions={} + ) + asset_rule.files.append(file_rule) + log.debug(f"Added file '{file_path_rel}' (type: {file_type}) to asset '{group_name}'.") + + + # Log if no assets were created + if not source_rule.assets: + log.warning(f"LLM prediction for '{self.input_source_identifier}' resulted in zero valid assets after parsing.") + + return [source_rule] # Return list containing the single SourceRule diff --git a/gui/log_console_widget.py b/gui/log_console_widget.py new file mode 100644 index 0000000..9bbdc4a --- /dev/null +++ b/gui/log_console_widget.py @@ -0,0 +1,35 @@ +import logging +from PySide6.QtWidgets import ( + QWidget, QVBoxLayout, QTextEdit, QLabel, QSizePolicy +) +from PySide6.QtCore import Slot + +log = logging.getLogger(__name__) + +class LogConsoleWidget(QWidget): + """ + A dedicated widget to display log messages. + """ + def __init__(self, parent=None): + super().__init__(parent) + self._init_ui() + + def _init_ui(self): + """Initializes the UI elements for the log console.""" + layout = QVBoxLayout(self) + layout.setContentsMargins(0, 5, 0, 0) + + log_console_label = QLabel("Log Console:") + self.log_console_output = QTextEdit() + self.log_console_output.setReadOnly(True) + self.log_console_output.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) # Allow vertical expansion + + layout.addWidget(log_console_label) + layout.addWidget(self.log_console_output) + + self.setVisible(False) + + @Slot(str) + def _append_log_message(self, message): + self.log_console_output.append(message) + self.log_console_output.verticalScrollBar().setValue(self.log_console_output.verticalScrollBar().maximum()) diff --git a/gui/main_panel_widget.py b/gui/main_panel_widget.py new file mode 100644 index 0000000..638f43f --- /dev/null +++ b/gui/main_panel_widget.py @@ -0,0 +1,647 @@ +import functools +import sys +import os +import json +import logging +import time +from pathlib import Path +from functools import partial + +from PySide6.QtWidgets import QApplication +from PySide6.QtWidgets import ( + QWidget, QVBoxLayout, QHBoxLayout, QSplitter, QTableView, + QPushButton, QComboBox, QTableWidget, QTableWidgetItem, QHeaderView, + QProgressBar, QLabel, QFrame, QCheckBox, QSpinBox, QListWidget, QTextEdit, + QLineEdit, QMessageBox, QFileDialog, QInputDialog, QListWidgetItem, QTabWidget, + QFormLayout, QGroupBox, QAbstractItemView, QSizePolicy, QTreeView, QMenu +) +from PySide6.QtCore import Qt, Signal, Slot, QPoint, QModelIndex, QTimer +from PySide6.QtGui import QColor, QAction, QPalette, QClipboard, QGuiApplication + +from .delegates import LineEditDelegate, ComboBoxDelegate, SupplierSearchDelegate, ItemTypeSearchDelegate +from .unified_view_model import UnifiedViewModel + +from rule_structure import SourceRule, AssetRule, FileRule +import configuration +try: + from configuration import ConfigurationError, load_base_config +except ImportError: + ConfigurationError = Exception + load_base_config = None + class configuration: + PRESETS_DIR = "Presets" + +log = logging.getLogger(__name__) + +class MainPanelWidget(QWidget): + """ + Widget handling the main interaction panel: + - Output directory selection + - Asset preview/editing view (Unified View) + - Blender post-processing options + - Processing controls (Start, Cancel, Clear, LLM Re-interpret) + """ + # --- Signals Emitted by the Panel --- + + # Request to start the main processing job + process_requested = Signal(dict) + + cancel_requested = Signal() + + clear_queue_requested = Signal() + + llm_reinterpret_requested = Signal(list) + preset_reinterpret_requested = Signal(list, str) + + output_dir_changed = Signal(str) + + blender_settings_changed = Signal(bool, str, str) + + def __init__(self, unified_model: UnifiedViewModel, parent=None, file_type_keys: list[str] | None = None): + """ + Initializes the MainPanelWidget. + + Args: + unified_model: The shared UnifiedViewModel instance. + parent: The parent widget. + file_type_keys: A list of available file type names (keys from FILE_TYPE_DEFINITIONS). + """ + super().__init__(parent) + self.unified_model = unified_model + self.file_type_keys = file_type_keys if file_type_keys else [] + self.llm_processing_active = False + + script_dir = Path(__file__).parent + self.project_root = script_dir.parent + + self._setup_ui() + self._connect_signals() + + def _setup_ui(self): + """Sets up the UI elements for the panel.""" + main_layout = QVBoxLayout(self) + main_layout.setContentsMargins(5, 5, 5, 5) + + output_layout = QHBoxLayout() + self.output_dir_label = QLabel("Output Directory:") + self.output_path_edit = QLineEdit() + self.browse_output_button = QPushButton("Browse...") + output_layout.addWidget(self.output_dir_label) + output_layout.addWidget(self.output_path_edit, 1) + output_layout.addWidget(self.browse_output_button) + main_layout.addLayout(output_layout) + + if load_base_config: + try: + base_config = load_base_config() + output_base_dir_config = base_config.get('OUTPUT_BASE_DIR', '../Asset_Processor_Output') + default_output_dir = (self.project_root / output_base_dir_config).resolve() + self.output_path_edit.setText(str(default_output_dir)) + log.info(f"MainPanelWidget: Default output directory set to: {default_output_dir}") + except ConfigurationError as e: + log.error(f"MainPanelWidget: Error reading base configuration for default output directory: {e}") + self.output_path_edit.setText("") + except Exception as e: + log.exception(f"MainPanelWidget: Error setting default output directory: {e}") + self.output_path_edit.setText("") + else: + log.warning("MainPanelWidget: load_base_config not available to set default output path.") + self.output_path_edit.setText("") + + + self.unified_view = QTreeView() + self.unified_view.setModel(self.unified_model) + + lineEditDelegate = LineEditDelegate(self.unified_view) + # TODO: Revisit ComboBoxDelegate dependency + comboBoxDelegate = ComboBoxDelegate(self) + supplierSearchDelegate = SupplierSearchDelegate(self) + itemTypeSearchDelegate = ItemTypeSearchDelegate(self.file_type_keys, self) + + self.unified_view.setItemDelegateForColumn(UnifiedViewModel.COL_SUPPLIER, supplierSearchDelegate) + self.unified_view.setItemDelegateForColumn(UnifiedViewModel.COL_ASSET_TYPE, comboBoxDelegate) + self.unified_view.setItemDelegateForColumn(UnifiedViewModel.COL_TARGET_ASSET, lineEditDelegate) + self.unified_view.setItemDelegateForColumn(UnifiedViewModel.COL_ITEM_TYPE, itemTypeSearchDelegate) + self.unified_view.setItemDelegateForColumn(UnifiedViewModel.COL_NAME, lineEditDelegate) + + self.unified_view.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) + self.unified_view.setAlternatingRowColors(True) + self.unified_view.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows) + self.unified_view.setEditTriggers(QAbstractItemView.EditTrigger.DoubleClicked | QAbstractItemView.EditTrigger.SelectedClicked | QAbstractItemView.EditTrigger.EditKeyPressed) + self.unified_view.setSelectionMode(QAbstractItemView.SelectionMode.ExtendedSelection) + + header = self.unified_view.header() + header.setStretchLastSection(False) + header.setSectionResizeMode(UnifiedViewModel.COL_NAME, QHeaderView.ResizeMode.ResizeToContents) + header.setSectionResizeMode(UnifiedViewModel.COL_TARGET_ASSET, QHeaderView.ResizeMode.Stretch) + header.setSectionResizeMode(UnifiedViewModel.COL_SUPPLIER, QHeaderView.ResizeMode.ResizeToContents) + header.setSectionResizeMode(UnifiedViewModel.COL_ASSET_TYPE, QHeaderView.ResizeMode.ResizeToContents) + header.setSectionResizeMode(UnifiedViewModel.COL_ITEM_TYPE, QHeaderView.ResizeMode.ResizeToContents) + + self.unified_view.setContextMenuPolicy(Qt.ContextMenuPolicy.CustomContextMenu) + + self.unified_view.setDragEnabled(True) + self.unified_view.setAcceptDrops(True) + self.unified_view.setDropIndicatorShown(True) + self.unified_view.setDefaultDropAction(Qt.MoveAction) + self.unified_view.setDragDropMode(QAbstractItemView.InternalMove) + self.unified_view.setSelectionMode(QAbstractItemView.SelectionMode.ExtendedSelection) + + main_layout.addWidget(self.unified_view, 1) + + self.progress_bar = QProgressBar() + self.progress_bar.setValue(0) + self.progress_bar.setTextVisible(True) + self.progress_bar.setFormat("Idle") + main_layout.addWidget(self.progress_bar) + + blender_group = QGroupBox("Blender Post-Processing") + blender_layout = QVBoxLayout(blender_group) + + self.blender_integration_checkbox = QCheckBox("Run Blender Scripts After Processing") + self.blender_integration_checkbox.setToolTip("If checked, attempts to run create_nodegroups.py and create_materials.py in Blender.") + blender_layout.addWidget(self.blender_integration_checkbox) + + nodegroup_layout = QHBoxLayout() + nodegroup_layout.addWidget(QLabel("Nodegroup .blend:")) + self.nodegroup_blend_path_input = QLineEdit() + self.browse_nodegroup_blend_button = QPushButton("...") + self.browse_nodegroup_blend_button.setFixedWidth(30) + nodegroup_layout.addWidget(self.nodegroup_blend_path_input) + nodegroup_layout.addWidget(self.browse_nodegroup_blend_button) + blender_layout.addLayout(nodegroup_layout) + + materials_layout = QHBoxLayout() + materials_layout.addWidget(QLabel("Materials .blend:")) + self.materials_blend_path_input = QLineEdit() + self.browse_materials_blend_button = QPushButton("...") + self.browse_materials_blend_button.setFixedWidth(30) + materials_layout.addWidget(self.materials_blend_path_input) + materials_layout.addWidget(self.browse_materials_blend_button) + blender_layout.addLayout(materials_layout) + + if load_base_config: + try: + base_config = load_base_config() + default_ng_path = base_config.get('DEFAULT_NODEGROUP_BLEND_PATH', '') + default_mat_path = base_config.get('DEFAULT_MATERIALS_BLEND_PATH', '') + self.nodegroup_blend_path_input.setText(default_ng_path if default_ng_path else "") + self.materials_blend_path_input.setText(default_mat_path if default_mat_path else "") + except ConfigurationError as e: + log.error(f"MainPanelWidget: Error reading base configuration for default Blender paths: {e}") + except Exception as e: + log.error(f"MainPanelWidget: Error reading default Blender paths from config: {e}") + else: + log.warning("MainPanelWidget: load_base_config not available to set default Blender paths.") + + + self.nodegroup_blend_path_input.setEnabled(False) + self.browse_nodegroup_blend_button.setEnabled(False) + self.materials_blend_path_input.setEnabled(False) + self.browse_materials_blend_button.setEnabled(False) + + main_layout.addWidget(blender_group) + + bottom_controls_layout = QHBoxLayout() + self.overwrite_checkbox = QCheckBox("Overwrite Existing") + self.overwrite_checkbox.setToolTip("If checked, existing output folders for processed assets will be deleted and replaced.") + bottom_controls_layout.addWidget(self.overwrite_checkbox) + + self.workers_label = QLabel("Workers:") + self.workers_spinbox = QSpinBox() + default_workers = 1 + try: + cores = os.cpu_count() + if cores: default_workers = max(1, cores // 2) + except NotImplementedError: pass + self.workers_spinbox.setMinimum(1) + self.workers_spinbox.setMaximum(os.cpu_count() or 32) + self.workers_spinbox.setValue(default_workers) + self.workers_spinbox.setToolTip("Number of assets to process concurrently.") + bottom_controls_layout.addWidget(self.workers_label) + bottom_controls_layout.addWidget(self.workers_spinbox) + bottom_controls_layout.addStretch(1) + + + self.clear_queue_button = QPushButton("Clear Queue") + self.start_button = QPushButton("Start Processing") + self.cancel_button = QPushButton("Cancel") + self.cancel_button.setEnabled(False) + + bottom_controls_layout.addWidget(self.clear_queue_button) + bottom_controls_layout.addWidget(self.start_button) + bottom_controls_layout.addWidget(self.cancel_button) + main_layout.addLayout(bottom_controls_layout) + + def _connect_signals(self): + """Connect internal UI signals to slots or emit panel signals.""" + self.browse_output_button.clicked.connect(self._browse_for_output_directory) + self.output_path_edit.editingFinished.connect(self._on_output_path_changed) + + self.unified_view.customContextMenuRequested.connect(self._show_unified_view_context_menu) + + self.blender_integration_checkbox.toggled.connect(self._toggle_blender_controls) + self.browse_nodegroup_blend_button.clicked.connect(self._browse_for_nodegroup_blend) + self.browse_materials_blend_button.clicked.connect(self._browse_for_materials_blend) + self.nodegroup_blend_path_input.editingFinished.connect(self._emit_blender_settings_changed) + self.materials_blend_path_input.editingFinished.connect(self._emit_blender_settings_changed) + self.blender_integration_checkbox.toggled.connect(self._emit_blender_settings_changed) + + + self.clear_queue_button.clicked.connect(self.clear_queue_requested) + self.start_button.clicked.connect(self._on_start_processing_clicked) + self.cancel_button.clicked.connect(self.cancel_requested) + + + @Slot() + def _browse_for_output_directory(self): + """Opens a dialog to select the output directory.""" + current_path = self.output_path_edit.text() + if not current_path or not Path(current_path).is_dir(): + current_path = str(self.project_root) + + directory = QFileDialog.getExistingDirectory( + self, + "Select Output Directory", + current_path, + QFileDialog.Option.ShowDirsOnly | QFileDialog.Option.DontResolveSymlinks + ) + if directory: + self.output_path_edit.setText(directory) + self._on_output_path_changed() + + @Slot() + def _on_output_path_changed(self): + """Emits the output_dir_changed signal.""" + self.output_dir_changed.emit(self.output_path_edit.text()) + + @Slot(bool) + def _toggle_blender_controls(self, checked): + """Enable/disable Blender path inputs based on the checkbox state.""" + self.nodegroup_blend_path_input.setEnabled(checked) + self.browse_nodegroup_blend_button.setEnabled(checked) + self.materials_blend_path_input.setEnabled(checked) + self.browse_materials_blend_button.setEnabled(checked) + + def _browse_for_blend_file(self, line_edit_widget: QLineEdit): + """Opens a dialog to select a .blend file and updates the line edit.""" + current_path = line_edit_widget.text() + start_dir = str(Path(current_path).parent) if current_path and Path(current_path).exists() else str(self.project_root) + + file_path, _ = QFileDialog.getOpenFileName( + self, + "Select Blender File", + start_dir, + "Blender Files (*.blend);;All Files (*)" + ) + if file_path: + line_edit_widget.setText(file_path) + line_edit_widget.editingFinished.emit() + + @Slot() + def _browse_for_nodegroup_blend(self): + self._browse_for_blend_file(self.nodegroup_blend_path_input) + + @Slot() + def _browse_for_materials_blend(self): + self._browse_for_blend_file(self.materials_blend_path_input) + + @Slot() + def _emit_blender_settings_changed(self): + """Gathers current Blender settings and emits the blender_settings_changed signal.""" + enabled = self.blender_integration_checkbox.isChecked() + ng_path = self.nodegroup_blend_path_input.text() + mat_path = self.materials_blend_path_input.text() + self.blender_settings_changed.emit(enabled, ng_path, mat_path) + + @Slot() + def _on_start_processing_clicked(self): + """Gathers settings and emits the process_requested signal.""" + output_dir = self.output_path_edit.text().strip() + if not output_dir: + QMessageBox.warning(self, "Missing Output Directory", "Please select an output directory.") + return + + try: + Path(output_dir).mkdir(parents=True, exist_ok=True) + except Exception as e: + QMessageBox.warning(self, "Invalid Output Directory", f"Cannot use output directory:\n{output_dir}\n\nError: {e}") + return + + settings = { + "output_dir": output_dir, + "overwrite": self.overwrite_checkbox.isChecked(), + "workers": self.workers_spinbox.value(), + "blender_enabled": self.blender_integration_checkbox.isChecked(), + "nodegroup_blend_path": self.nodegroup_blend_path_input.text(), + "materials_blend_path": self.materials_blend_path_input.text() + } + self.process_requested.emit(settings) + + + def _get_unique_source_dirs_from_selection(self, selected_indexes: list[QModelIndex]) -> set[str]: + """ + Extracts unique, valid source directory/zip paths from the selected QModelIndex list. + Traverses up the model hierarchy to find the parent SourceRule for each index. + """ + unique_source_dirs = set() + model = self.unified_view.model() + if not model: + log.error("Unified view model not found.") + return unique_source_dirs + + processed_source_paths = set() + + for index in selected_indexes: + if not index.isValid(): + continue + + item_node = model.getItem(index) + source_rule_node = None + + source_rule_node = None + current_index = index + while current_index.isValid(): + current_item = model.getItem(current_index) + if isinstance(current_item, SourceRule): + source_rule_node = current_item + break + current_index = model.parent(current_index) # Move to the parent index + # If loop finishes without break, source_rule_node remains None + + if source_rule_node: + source_path = getattr(source_rule_node, 'input_path', None) + if source_path and source_path not in processed_source_paths: + source_path_obj = Path(source_path) + if source_path_obj.is_dir() or (source_path_obj.is_file() and source_path_obj.suffix.lower() == '.zip'): + log.debug(f"Identified source path for re-interpretation: {source_path}") + unique_source_dirs.add(source_path) + processed_source_paths.add(source_path) + else: + log.warning(f"Selected item's source path is not a directory or zip file: {source_path}") + elif not source_path: + log.warning(f"Parent SourceRule found for index {index.row()},{index.column()} but has no 'input_path' attribute.") + + else: + log.warning(f"Could not find parent SourceRule for selected index: {index.row()},{index.column()} (Node type: {type(item_node).__name__})") + + return unique_source_dirs + + @Slot() + def _on_llm_reinterpret_clicked(self): + """Gathers selected source paths and emits the llm_reinterpret_requested signal. (Triggered by context menu)""" + if self.llm_processing_active: + QMessageBox.warning(self, "Busy", "LLM processing is already in progress. Please wait.") + return + + selected_indexes = self.unified_view.selectionModel().selectedIndexes() + unique_source_dirs = self._get_unique_source_dirs_from_selection(selected_indexes) + + if not unique_source_dirs: + log.warning("No valid source directories found for selected items to re-interpret with LLM.") + # Optionally show status bar message via MainWindow reference if available + return + + log.info(f"Emitting llm_reinterpret_requested for {len(unique_source_dirs)} paths.") + self.llm_reinterpret_requested.emit(list(unique_source_dirs)) + + + @Slot(str, QModelIndex) + def _on_reinterpret_preset_selected(self, preset_name: str, index: QModelIndex): + """Handles the selection of a preset from the re-interpret context sub-menu.""" + log.info(f"Preset re-interpretation requested: Preset='{preset_name}', Index='{index.row()},{index.column()}'") + selected_indexes = self.unified_view.selectionModel().selectedIndexes() + unique_source_dirs = self._get_unique_source_dirs_from_selection(selected_indexes) + + if not unique_source_dirs: + log.warning("No valid source directories found for selected items to re-interpret with preset.") + # Optionally show status bar message via MainWindow reference if available + return + + log.info(f"Emitting preset_reinterpret_requested for {len(unique_source_dirs)} paths with preset '{preset_name}'.") + self.preset_reinterpret_requested.emit(list(unique_source_dirs), preset_name) + + + @Slot(QPoint) + def _show_unified_view_context_menu(self, point: QPoint): + """Shows the context menu for the unified view.""" + index = self.unified_view.indexAt(point) + if not index.isValid(): + return + + model = self.unified_view.model() + if not model: return + item_node = model.getItem(index) + + # Find the SourceRule node associated with the clicked index + source_rule_node = None + current_index = index + while current_index.isValid(): + current_item = model.getItem(current_index) + if isinstance(current_item, SourceRule): + source_rule_node = current_item + break + current_index = model.parent(current_index) # Move to the parent index + # If loop finishes without break, source_rule_node remains None + + menu = QMenu(self) + + if source_rule_node: # Only show if we clicked on or within a SourceRule item + reinterpet_menu = menu.addMenu("Re-interpret selected source") + + preset_names = [] + try: + presets_dir = configuration.PRESETS_DIR + if os.path.isdir(presets_dir): + for filename in os.listdir(presets_dir): + if filename.endswith(".json") and filename != "_template.json": + preset_name = os.path.splitext(filename)[0] + preset_names.append(preset_name) + preset_names.sort() + else: + log.warning(f"Presets directory not found or not a directory: {presets_dir}") + except Exception as e: + log.exception(f"Error listing presets in {configuration.PRESETS_DIR}: {e}") + + if preset_names: + for preset_name in preset_names: + preset_action = QAction(preset_name, self) + preset_action.triggered.connect(functools.partial(self._on_reinterpret_preset_selected, preset_name, index)) + reinterpet_menu.addAction(preset_action) + else: + no_presets_action = QAction("No presets found", self) + no_presets_action.setEnabled(False) + reinterpet_menu.addAction(no_presets_action) + + + reinterpet_menu.addSeparator() + llm_action = QAction("LLM", self) + llm_action.triggered.connect(self._on_llm_reinterpret_clicked) + llm_action.setEnabled(not self.llm_processing_active) + reinterpet_menu.addAction(llm_action) + + menu.addSeparator() + + if source_rule_node: # Check again if it's a source item for this action + copy_llm_example_action = QAction("Copy LLM Example to Clipboard", self) + copy_llm_example_action.setToolTip("Copies a JSON structure representing the input files and predicted output, suitable for LLM examples.") + copy_llm_example_action.triggered.connect(lambda: self._copy_llm_example_to_clipboard(source_rule_node)) + menu.addAction(copy_llm_example_action) + + + if not menu.isEmpty(): + menu.exec(self.unified_view.viewport().mapToGlobal(point)) + + @Slot(SourceRule) + def _copy_llm_example_to_clipboard(self, source_rule_node: SourceRule | None): + """Copies a JSON structure for the given SourceRule node to the clipboard.""" + if not source_rule_node: + log.warning(f"No SourceRule node provided to copy LLM example.") + return + + source_rule: SourceRule = source_rule_node + log.info(f"Attempting to generate LLM example JSON for source: {source_rule.input_path}") + + all_file_paths = [] + predicted_assets_data = [] + + for asset_rule in source_rule.assets: + asset_files_data = [] + for file_rule in asset_rule.files: + if file_rule.file_path: + all_file_paths.append(file_rule.file_path) + asset_files_data.append({ + "file_path": file_rule.file_path, + "predicted_file_type": file_rule.item_type or "UNKNOWN" + }) + asset_files_data.sort(key=lambda x: x['file_path']) + predicted_assets_data.append({ + "suggested_asset_name": asset_rule.asset_name or "UnnamedAsset", + "predicted_asset_type": asset_rule.asset_type or "UNKNOWN", + "files": asset_files_data + }) + + predicted_assets_data.sort(key=lambda x: x['suggested_asset_name']) + all_file_paths.sort() + + if not all_file_paths: + log.warning(f"No file paths found for source: {source_rule.input_path}. Cannot generate example.") + # Cannot show status bar message here + return + + llm_example = { + "input": "\n".join(all_file_paths), + "output": {"predicted_assets": predicted_assets_data} + } + + try: + json_string = json.dumps(llm_example, indent=2) + clipboard = QGuiApplication.clipboard() + if clipboard: + clipboard.setText(json_string) + log.info(f"Copied LLM example JSON to clipboard for source: {source_rule.input_path}") + else: + log.error("Failed to get system clipboard.") + except Exception as e: + log.exception(f"Error copying LLM example JSON to clipboard: {e}") + + + # --- Public Slots for MainWindow to Call --- + + @Slot(int, int) + def update_progress_bar(self, current_count, total_count): + """Updates the progress bar display.""" + if total_count > 0: + percentage = int((current_count / total_count) * 100) + log.debug(f"Updating progress bar: current={current_count}, total={total_count}, calculated_percentage={percentage}") + self.progress_bar.setValue(percentage) + self.progress_bar.setFormat(f"%p% ({current_count}/{total_count})") + QApplication.processEvents() + else: + self.progress_bar.setValue(0) + self.progress_bar.setFormat("0/0") + + @Slot(str) + def set_progress_bar_text(self, text: str): + """Sets the text format of the progress bar.""" + self.progress_bar.setFormat(text) + if not "%" in text: + self.progress_bar.setValue(0) + + + @Slot(bool) + def set_controls_enabled(self, enabled: bool): + """Enables or disables controls within the panel.""" + self.output_path_edit.setEnabled(enabled) + self.browse_output_button.setEnabled(enabled) + self.unified_view.setEnabled(enabled) + self.overwrite_checkbox.setEnabled(enabled) + self.workers_spinbox.setEnabled(enabled) + self.clear_queue_button.setEnabled(enabled) + self.blender_integration_checkbox.setEnabled(enabled) + + # Start button is enabled only if controls are generally enabled AND preset mode is active (handled by MainWindow) + # Cancel button is enabled only when processing is active (handled by MainWindow) + # LLM button state depends on selection and LLM status (handled by _update_llm_reinterpret_button_state) + + # Blender path inputs depend on both 'enabled' and the checkbox state + blender_paths_enabled = enabled and self.blender_integration_checkbox.isChecked() + self.nodegroup_blend_path_input.setEnabled(blender_paths_enabled) + self.browse_nodegroup_blend_button.setEnabled(blender_paths_enabled) + self.materials_blend_path_input.setEnabled(blender_paths_enabled) + self.browse_materials_blend_button.setEnabled(blender_paths_enabled) + + + + @Slot(bool) + def set_start_button_enabled(self, enabled: bool): + """Sets the enabled state of the Start Processing button.""" + self.start_button.setEnabled(enabled) + + @Slot(str) + def set_start_button_text(self, text: str): + """Sets the text of the Start Processing button.""" + self.start_button.setText(text) + + @Slot(bool) + def set_cancel_button_enabled(self, enabled: bool): + """Sets the enabled state of the Cancel button.""" + self.cancel_button.setEnabled(enabled) + + @Slot(bool) + def set_llm_processing_status(self, active: bool): + """Informs the panel whether LLM processing is active (used for context menu state).""" + self.llm_processing_active = active + # No button state to update directly, but context menu will check this flag when built. + + # TODO: Add method to get current output path if needed by MainWindow before processing + def get_output_directory() -> str: + return self.output_path_edit.text().strip() + + # TODO: Add method to get current Blender settings if needed by MainWindow before processing + def get_blender_settings() -> dict: + return { + "enabled": self.blender_integration_checkbox.isChecked(), + "nodegroup_blend_path": self.nodegroup_blend_path_input.text(), + "materials_blend_path": self.materials_blend_path_input.text() + } + + # TODO: Add method to get current worker count if needed by MainWindow before processing + def get_worker_count() -> int: + return self.workers_spinbox.value() + + # TODO: Add method to get current overwrite setting if needed by MainWindow before processing + def get_overwrite_setting() -> bool: + return self.overwrite_checkbox.isChecked() + + def get_llm_source_preset_name() -> str | None: + """ + Placeholder for providing context to delegates. + Ideally, the required info (like last preset name) should be passed + from MainWindow when the delegate needs it, or the delegate's dependency + should be refactored. + """ + log.warning("MainPanelWidget.get_llm_source_preset_name called - needs proper implementation or refactoring.") + # This needs to get the info from MainWindow, perhaps via a signal/slot or passed reference. + # Returning None for now. + return None \ No newline at end of file diff --git a/gui/main_window.py b/gui/main_window.py index 587f652..257ec21 100644 --- a/gui/main_window.py +++ b/gui/main_window.py @@ -3,49 +3,63 @@ import os import json import logging import time +import zipfile from pathlib import Path -from functools import partial # For connecting signals with arguments - +from functools import partial log = logging.getLogger(__name__) log.info(f"sys.path: {sys.path}") from PySide6.QtWidgets import ( - QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QSplitter, # Added QSplitter - QPushButton, QComboBox, QTableWidget, QTableWidgetItem, QHeaderView, - QProgressBar, QLabel, QFrame, QCheckBox, QSpinBox, QListWidget, QTextEdit, # Added QListWidget, QTextEdit - QLineEdit, QMessageBox, QFileDialog, QInputDialog, QListWidgetItem, QTabWidget, # Added more widgets - QFormLayout, QGroupBox, QAbstractItemView, QSizePolicy, # Added more layout/widget items - QMenuBar, QMenu # Added for menu + QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QSplitter, QTableView, + QPushButton, QComboBox, QTableWidget, QTableWidgetItem, QHeaderView, QStackedWidget, + QProgressBar, QLabel, QFrame, QCheckBox, QSpinBox, QListWidget, QTextEdit, + QLineEdit, QMessageBox, QFileDialog, QInputDialog, QListWidgetItem, QTabWidget, + QFormLayout, QGroupBox, QAbstractItemView, QSizePolicy, + QMenuBar, QMenu, QTreeView ) -from PySide6.QtCore import Qt, QThread, Slot, Signal, QObject # Added Signal, QObject -from PySide6.QtGui import QColor, QAction, QPalette # Add QColor import, QAction, QPalette +from PySide6.QtCore import Qt, QThread, Slot, Signal, QObject, QModelIndex, QItemSelectionModel, QPoint, QTimer +from PySide6.QtGui import QColor, QAction, QPalette, QClipboard +from PySide6.QtGui import QKeySequence + +# --- Local GUI Imports --- +from .preset_editor_widget import PresetEditorWidget +from .llm_editor_widget import LLMEditorWidget +from .log_console_widget import LogConsoleWidget +from .main_panel_widget import MainPanelWidget + +from .definitions_editor_dialog import DefinitionsEditorDialog +# --- Backend Imports for Data Structures --- +from rule_structure import SourceRule, AssetRule, FileRule + # --- GUI Model Imports --- -from gui.preview_table_model import PreviewTableModel, PreviewSortFilterProxyModel +from gui.unified_view_model import UnifiedViewModel, CustomRoles +# Removed delegate imports, now handled by MainPanelWidget +from .prediction_handler import RuleBasedPredictionHandler +from .llm_interaction_handler import LLMInteractionHandler +from .asset_restructure_handler import AssetRestructureHandler -# --- Backend Imports --- + # --- Backend Imports --- script_dir = Path(__file__).parent project_root = script_dir.parent if str(project_root) not in sys.path: sys.path.insert(0, str(project_root)) try: - from configuration import Configuration, ConfigurationError - from asset_processor import AssetProcessor, AssetProcessingError - from gui.processing_handler import ProcessingHandler - from gui.prediction_handler import PredictionHandler - import config as core_config # Import the config module - # PresetEditorDialog is no longer needed + from configuration import Configuration, ConfigurationError, load_base_config + + except ImportError as e: print(f"ERROR: Failed to import backend modules: {e}") print(f"Ensure GUI is run from project root or backend modules are in PYTHONPATH.") Configuration = None - AssetProcessor = None - ProcessingHandler = None - PredictionHandler = None + load_base_config = None ConfigurationError = Exception + AssetProcessor = None + RuleBasedPredictionHandler = None AssetProcessingError = Exception + # --- Constants --- PRESETS_DIR = project_root / "presets" TEMPLATE_PATH = PRESETS_DIR / "_template.json" @@ -53,8 +67,7 @@ TEMPLATE_PATH = PRESETS_DIR / "_template.json" # Setup basic logging log = logging.getLogger(__name__) if not log.hasHandlers(): - # Set level back to INFO for normal operation - logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') # Reverted level and format + logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') # --- Custom Log Handler --- @@ -63,11 +76,11 @@ class QtLogHandler(logging.Handler, QObject): Custom logging handler that emits a Qt signal for each log record. Inherits from QObject to support signals. """ - log_record_received = Signal(str) # Signal emitting the formatted log string + log_record_received = Signal(str) def __init__(self, parent=None): logging.Handler.__init__(self) - QObject.__init__(self, parent) # Initialize QObject part + QObject.__init__(self, parent) def emit(self, record): """ @@ -80,533 +93,147 @@ class QtLogHandler(logging.Handler, QObject): self.handleError(record) -# --- Helper Functions (from PresetEditorDialog) --- -# NOTE: Consider moving these to a utils file if reused elsewhere - -def setup_list_widget_with_controls(parent_layout, label_text, attribute_name, instance): - """Adds a QListWidget with Add/Remove buttons to a layout.""" - list_widget = QListWidget() - list_widget.setAlternatingRowColors(True) - # Make items editable by default in the editor context - list_widget.setEditTriggers(QAbstractItemView.EditTrigger.DoubleClicked | QAbstractItemView.EditTrigger.SelectedClicked | QAbstractItemView.EditTrigger.EditKeyPressed) - instance.__setattr__(attribute_name, list_widget) # Store list widget on the instance - - add_button = QPushButton("+") - remove_button = QPushButton("-") - add_button.setFixedWidth(30) - remove_button.setFixedWidth(30) - - button_layout = QVBoxLayout() - button_layout.addWidget(add_button) - button_layout.addWidget(remove_button) - button_layout.addStretch() - - list_layout = QHBoxLayout() - list_layout.addWidget(list_widget) - list_layout.addLayout(button_layout) - - group_box = QGroupBox(label_text) - group_box_layout = QVBoxLayout(group_box) - group_box_layout.addLayout(list_layout) - - parent_layout.addWidget(group_box) - - # Connections (use the instance's methods) - add_button.clicked.connect(partial(instance._editor_add_list_item, list_widget)) - remove_button.clicked.connect(partial(instance._editor_remove_list_item, list_widget)) - list_widget.itemChanged.connect(instance._mark_editor_unsaved) # Mark unsaved on item edit - -def setup_table_widget_with_controls(parent_layout, label_text, attribute_name, columns, instance): - """Adds a QTableWidget with Add/Remove buttons to a layout.""" - table_widget = QTableWidget() - table_widget.setColumnCount(len(columns)) - table_widget.setHorizontalHeaderLabels(columns) - table_widget.setAlternatingRowColors(True) - instance.__setattr__(attribute_name, table_widget) # Store table widget - - add_button = QPushButton("+ Row") - remove_button = QPushButton("- Row") - - button_layout = QHBoxLayout() - button_layout.addStretch() - button_layout.addWidget(add_button) - button_layout.addWidget(remove_button) - - group_box = QGroupBox(label_text) - group_box_layout = QVBoxLayout(group_box) - group_box_layout.addWidget(table_widget) - group_box_layout.addLayout(button_layout) - - parent_layout.addWidget(group_box) - - # Connections (use the instance's methods) - add_button.clicked.connect(partial(instance._editor_add_table_row, table_widget)) - remove_button.clicked.connect(partial(instance._editor_remove_table_row, table_widget)) - table_widget.itemChanged.connect(instance._mark_editor_unsaved) # Mark unsaved on item edit - - -# --- Main Window Class --- - class MainWindow(QMainWindow): - # Signal emitted when presets change in the editor panel - presets_changed_signal = Signal() + start_prediction_signal = Signal(str, list, str) + start_backend_processing = Signal(list, dict) def __init__(self): super().__init__() self.setWindowTitle("Asset Processor Tool") - self.resize(1200, 700) # Increased default size + self.resize(1200, 700) # --- Internal State --- - self.current_asset_paths = set() # Store unique paths of assets added - - # --- Editor State --- - self.current_editing_preset_path = None - self.editor_unsaved_changes = False - self._is_loading_editor = False # Flag to prevent signals during load + self.current_asset_paths = set() + self._pending_predictions = set() + self._completed_predictions = set() + self._accumulated_rules = {} + self._source_file_lists = {} + # Removed the problematic instantiation of Configuration without a preset. + # self.config_manager will be set when a specific preset is loaded, + # or LLM settings will be loaded directly via load_base_config(). + self.config_manager = None + self.llm_processing_queue = [] + self._current_output_dir = "" + self._current_blender_settings = {} # --- Threading Setup --- - self.processing_thread = None - self.processing_handler = None self.prediction_thread = None self.prediction_handler = None + # LLM thread/handler are now managed by LLMInteractionHandler self.setup_threads() + # --- Instantiate Handlers --- + self.llm_interaction_handler = LLMInteractionHandler(self) + + # --- Main Layout with Splitter --- self.splitter = QSplitter(Qt.Orientation.Horizontal) self.setCentralWidget(self.splitter) + # --- Create Models --- + self.unified_model = UnifiedViewModel() + # --- Instantiate Handlers that depend on the model --- + self.restructure_handler = AssetRestructureHandler(self.unified_model, self) + # --- Create Panels --- - self.editor_panel = QWidget() - self.main_panel = QWidget() + self.preset_editor_widget = PresetEditorWidget() + self.llm_editor_widget = LLMEditorWidget() - self.splitter.addWidget(self.editor_panel) - self.splitter.addWidget(self.main_panel) + # --- Load File Type Definitions for Rule Editor --- + file_type_keys = [] + try: + base_cfg_data = load_base_config() + if base_cfg_data and "FILE_TYPE_DEFINITIONS" in base_cfg_data: + file_type_keys = list(base_cfg_data["FILE_TYPE_DEFINITIONS"].keys()) + log.info(f"Loaded {len(file_type_keys)} FILE_TYPE_DEFINITIONS keys for RuleEditor.") + else: + log.warning("FILE_TYPE_DEFINITIONS not found in base_config. RuleEditor item_type dropdown might be empty.") + except Exception as e: + log.exception(f"Error loading FILE_TYPE_DEFINITIONS for RuleEditor: {e}") - # --- Setup UI Elements for each panel --- - self.setup_editor_panel_ui() - self.setup_main_panel_ui() - self.setup_menu_bar() # Setup menu bar + # Instantiate MainPanelWidget, passing the model, self (MainWindow) for context, and file_type_keys + self.main_panel_widget = MainPanelWidget(self.unified_model, self, file_type_keys=file_type_keys) + self.log_console = LogConsoleWidget(self) + + # --- Create Left Pane with Static Selector and Stacked Editor --- + self.left_pane_widget = QWidget() + left_pane_layout = QVBoxLayout(self.left_pane_widget) + left_pane_layout.setContentsMargins(0, 0, 0, 0) + left_pane_layout.setSpacing(0) + + left_pane_layout.addWidget(self.preset_editor_widget.selector_container) + + self.editor_stack = QStackedWidget() + self.editor_stack.addWidget(self.preset_editor_widget.json_editor_container) + self.editor_stack.addWidget(self.llm_editor_widget) + left_pane_layout.addWidget(self.editor_stack) + + self.splitter.addWidget(self.left_pane_widget) + self.splitter.addWidget(self.main_panel_widget) + + # --- Setup UI Elements --- + # Main panel UI is handled internally by MainPanelWidget + self.setup_menu_bar() # --- Status Bar --- self.statusBar().showMessage("Ready") # --- Initial State --- - self._clear_editor() # Clear/disable editor fields initially - self._set_editor_enabled(False) # Disable editor initially - self.populate_presets() # Populate preset list - self.setup_logging_handler() # Setup the custom log handler + self.setup_logging_handler() - # --- Connect Editor Signals --- - self._connect_editor_change_signals() + # --- Connect Signals from PresetEditorWidget --- + self.preset_editor_widget.preset_selection_changed_signal.connect(self._on_preset_selection_changed) + # --- Connect Signals from MainPanelWidget --- + self.main_panel_widget.process_requested.connect(self._on_process_requested) + self.main_panel_widget.cancel_requested.connect(self._on_cancel_requested) + self.main_panel_widget.clear_queue_requested.connect(self._on_clear_queue_requested) + self.main_panel_widget.llm_reinterpret_requested.connect(self._delegate_llm_reinterpret) + self.main_panel_widget.output_dir_changed.connect(self._on_output_dir_changed) + self.main_panel_widget.blender_settings_changed.connect(self._on_blender_settings_changed) + + self.main_panel_widget.preset_reinterpret_requested.connect(self._on_preset_reinterpret_requested) + # --- Connect Signals from LLMInteractionHandler --- + self.llm_interaction_handler.llm_prediction_ready.connect(self._on_llm_prediction_ready_from_handler) + self.llm_interaction_handler.llm_prediction_error.connect(self._on_prediction_error) + self.llm_interaction_handler.llm_status_update.connect(self.show_status_message) + self.llm_interaction_handler.llm_processing_state_changed.connect(self._on_llm_processing_state_changed) + + # --- Connect Model Signals --- + self.unified_model.targetAssetOverrideChanged.connect(self.restructure_handler.handle_target_asset_override) + self.unified_model.assetNameChanged.connect(self.restructure_handler.handle_asset_name_changed) + # --- Connect LLM Editor Signals --- + self.llm_editor_widget.settings_saved.connect(self._on_llm_settings_saved) # --- Adjust Splitter --- - self.splitter.setSizes([400, 800]) # Initial size ratio + self.splitter.setSizes([400, 800]) - # --- UI Setup Methods --- - - def setup_editor_panel_ui(self): - """Sets up the UI elements for the left preset editor panel.""" - editor_layout = QVBoxLayout(self.editor_panel) - editor_layout.setContentsMargins(5, 5, 5, 5) # Reduce margins - - # --- Log Console Output (Initially Hidden) --- - self.log_console_widget = QWidget() - log_console_layout = QVBoxLayout(self.log_console_widget) - log_console_layout.setContentsMargins(0, 0, 0, 5) # Add some bottom margin - log_console_label = QLabel("Log Console:") - self.log_console_output = QTextEdit() - self.log_console_output.setReadOnly(True) - self.log_console_output.setMaximumHeight(150) # Limit initial height - self.log_console_output.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Maximum) - log_console_layout.addWidget(log_console_label) - log_console_layout.addWidget(self.log_console_output) - self.log_console_widget.setVisible(False) # Start hidden - editor_layout.addWidget(self.log_console_widget) # Add it at the top - - # Preset List and Controls - list_layout = QVBoxLayout() - list_layout.addWidget(QLabel("Presets:")) - self.editor_preset_list = QListWidget() - self.editor_preset_list.currentItemChanged.connect(self._load_selected_preset_for_editing) - list_layout.addWidget(self.editor_preset_list) - - list_button_layout = QHBoxLayout() - self.editor_new_button = QPushButton("New") - self.editor_delete_button = QPushButton("Delete") - self.editor_new_button.clicked.connect(self._new_preset) - self.editor_delete_button.clicked.connect(self._delete_selected_preset) - list_button_layout.addWidget(self.editor_new_button) - list_button_layout.addWidget(self.editor_delete_button) - list_layout.addLayout(list_button_layout) - editor_layout.addLayout(list_layout, 1) # Allow list to stretch - - # Editor Tabs - self.editor_tab_widget = QTabWidget() - self.editor_tab_general_naming = QWidget() - self.editor_tab_mapping_rules = QWidget() - self.editor_tab_widget.addTab(self.editor_tab_general_naming, "General & Naming") - self.editor_tab_widget.addTab(self.editor_tab_mapping_rules, "Mapping & Rules") - self._create_editor_general_tab() - self._create_editor_mapping_tab() - editor_layout.addWidget(self.editor_tab_widget, 3) # Allow tabs to stretch more - - # Save Buttons - save_button_layout = QHBoxLayout() - self.editor_save_button = QPushButton("Save") - self.editor_save_as_button = QPushButton("Save As...") - self.editor_save_button.setEnabled(False) # Disabled initially - self.editor_save_button.clicked.connect(self._save_current_preset) - self.editor_save_as_button.clicked.connect(self._save_preset_as) - save_button_layout.addStretch() - save_button_layout.addWidget(self.editor_save_button) - save_button_layout.addWidget(self.editor_save_as_button) - editor_layout.addLayout(save_button_layout) - - def _create_editor_general_tab(self): - """Creates the widgets and layout for the 'General & Naming' editor tab.""" - layout = QVBoxLayout(self.editor_tab_general_naming) - form_layout = QFormLayout() - form_layout.setFieldGrowthPolicy(QFormLayout.FieldGrowthPolicy.ExpandingFieldsGrow) - - # Basic Info - self.editor_preset_name = QLineEdit() - self.editor_supplier_name = QLineEdit() - self.editor_notes = QTextEdit() - self.editor_notes.setAcceptRichText(False) - self.editor_notes.setFixedHeight(60) - form_layout.addRow("Preset Name:", self.editor_preset_name) - form_layout.addRow("Supplier Name:", self.editor_supplier_name) - form_layout.addRow("Notes:", self.editor_notes) - layout.addLayout(form_layout) - - # Source Naming Group - naming_group = QGroupBox("Source File Naming Rules") - naming_layout_outer = QVBoxLayout(naming_group) - naming_layout_form = QFormLayout() - self.editor_separator = QLineEdit() - self.editor_separator.setMaxLength(1) - self.editor_spin_base_name_idx = QSpinBox() - self.editor_spin_base_name_idx.setMinimum(-1) - self.editor_spin_map_type_idx = QSpinBox() - self.editor_spin_map_type_idx.setMinimum(-1) - naming_layout_form.addRow("Separator:", self.editor_separator) - naming_layout_form.addRow("Base Name Index:", self.editor_spin_base_name_idx) - naming_layout_form.addRow("Map Type Index:", self.editor_spin_map_type_idx) - naming_layout_outer.addLayout(naming_layout_form) - # Gloss Keywords List - setup_list_widget_with_controls(naming_layout_outer, "Glossiness Keywords", "editor_list_gloss_keywords", self) - # Bit Depth Variants Table - setup_table_widget_with_controls(naming_layout_outer, "16-bit Variant Patterns", "editor_table_bit_depth_variants", ["Map Type", "Pattern"], self) - self.editor_table_bit_depth_variants.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.ResizeToContents) - self.editor_table_bit_depth_variants.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch) - layout.addWidget(naming_group) - - # Extra Files Group - setup_list_widget_with_controls(layout, "Move to 'Extra' Folder Patterns", "editor_list_extra_patterns", self) - - layout.addStretch(1) - - def _create_editor_mapping_tab(self): - """Creates the widgets and layout for the 'Mapping & Rules' editor tab.""" - layout = QVBoxLayout(self.editor_tab_mapping_rules) - - # Map Type Mapping Group - setup_table_widget_with_controls(layout, "Map Type Mapping (Standard Type <- Input Keywords)", "editor_table_map_type_mapping", ["Standard Type", "Input Keywords (comma-sep)"], self) - self.editor_table_map_type_mapping.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.ResizeToContents) - self.editor_table_map_type_mapping.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch) - - # Category Rules Group - category_group = QGroupBox("Asset Category Rules") - category_layout = QVBoxLayout(category_group) - setup_list_widget_with_controls(category_layout, "Model File Patterns", "editor_list_model_patterns", self) - setup_list_widget_with_controls(category_layout, "Decal Keywords", "editor_list_decal_keywords", self) - layout.addWidget(category_group) - - # Archetype Rules Group - setup_table_widget_with_controls(layout, "Archetype Rules", "editor_table_archetype_rules", ["Archetype Name", "Match Any (comma-sep)", "Match All (comma-sep)"], self) - self.editor_table_archetype_rules.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.ResizeToContents) - self.editor_table_archetype_rules.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch) - self.editor_table_archetype_rules.horizontalHeader().setSectionResizeMode(2, QHeaderView.ResizeMode.Stretch) - - layout.addStretch(1) - - def setup_main_panel_ui(self): - """Sets up the UI elements for the right main processing panel.""" - main_layout = QVBoxLayout(self.main_panel) - main_layout.setContentsMargins(5, 5, 5, 5) # Reduce margins - - # --- Output Directory Selection --- - output_layout = QHBoxLayout() - self.output_dir_label = QLabel("Output Directory:") - self.output_path_edit = QLineEdit() - # Make read-only for now, user must use Browse - # self.output_path_edit.setReadOnly(True) - self.browse_output_button = QPushButton("Browse...") - self.browse_output_button.clicked.connect(self._browse_for_output_directory) - output_layout.addWidget(self.output_dir_label) - output_layout.addWidget(self.output_path_edit, 1) - output_layout.addWidget(self.browse_output_button) - main_layout.addLayout(output_layout) - - # --- Set Initial Output Path --- + # --- Initialize Keybind Map --- + self.key_char_to_qt_key = { + 'C': Qt.Key_C, 'R': Qt.Key_R, 'N': Qt.Key_N, 'M': Qt.Key_M, + 'D': Qt.Key_D, 'E': Qt.Key_E, 'X': Qt.Key_X + } + self.qt_key_to_ftd_map = {} try: - output_base_dir_config = getattr(core_config, 'OUTPUT_BASE_DIR', '../Asset_Processor_Output') # Default if not found - # Resolve the path relative to the project root - default_output_dir = (project_root / output_base_dir_config).resolve() - self.output_path_edit.setText(str(default_output_dir)) - log.info(f"Default output directory set to: {default_output_dir}") + base_settings = load_base_config() + file_type_defs = base_settings.get('FILE_TYPE_DEFINITIONS', {}) + for ftd_key, ftd_value in file_type_defs.items(): + if isinstance(ftd_value, dict) and 'keybind' in ftd_value: + char_key = ftd_value['keybind'] + qt_key_val = self.key_char_to_qt_key.get(char_key) + if qt_key_val: + if qt_key_val not in self.qt_key_to_ftd_map: + self.qt_key_to_ftd_map[qt_key_val] = [] + # Ensure consistent order for toggleable types if they are defined together under one key + # For example, if 'R' maps to ROUGH then GLOSS, they should appear in that order. + # This relies on the order in app_settings.json and dict iteration (Python 3.7+). + self.qt_key_to_ftd_map[qt_key_val].append(ftd_key) + log.info(f"Loaded keybind map: {self.qt_key_to_ftd_map}") except Exception as e: - log.error(f"Error setting default output directory: {e}") - self.output_path_edit.setText("") # Clear on error - self.statusBar().showMessage(f"Error setting default output path: {e}", 5000) + log.error(f"Failed to load keybind configurations: {e}") + # self.qt_key_to_ftd_map will be empty, keybinds won't work. - # --- Drag and Drop Area --- - self.drag_drop_area = QFrame() - self.drag_drop_area.setFrameShape(QFrame.Shape.StyledPanel) - self.drag_drop_area.setFrameShadow(QFrame.Shadow.Sunken) - drag_drop_layout = QVBoxLayout(self.drag_drop_area) - drag_drop_label = QLabel("Drag and Drop Asset Files/Folders Here") - drag_drop_label.setAlignment(Qt.AlignmentFlag.AlignCenter) - drag_drop_layout.addWidget(drag_drop_label) - self.drag_drop_area.setMinimumHeight(100) - self.setAcceptDrops(True) # Main window handles drops initially - main_layout.addWidget(self.drag_drop_area) - self.drag_drop_area.setVisible(False) # Hide the specific visual drag/drop area - - # --- Preview Area (Table) --- - self.preview_label = QLabel("File Preview:") # Updated Label - self.preview_table = QTableWidget() # Keep QTableWidget for now, will replace with QTableView later - # Initialize models - self.preview_model = PreviewTableModel() - self.preview_proxy_model = PreviewSortFilterProxyModel() - self.preview_proxy_model.setSourceModel(self.preview_model) - - # Use the proxy model for the table view - # NOTE: QTableWidget is simpler but less flexible with models. - # For full model/view benefits (like multi-column sorting via proxy), - # we should ideally switch to QTableView. Sticking with QTableWidget for minimal change first. - # However, QTableWidget doesn't fully support QSortFilterProxyModel for sorting. - # Let's switch to QTableView now for proper model/proxy integration. - from PySide6.QtWidgets import QTableView # Import QTableView - - self.preview_table_view = QTableView() # Use QTableView instead of QTableWidget - self.preview_table_view.setModel(self.preview_proxy_model) # Set the proxy model - - # Set headers and resize modes using the model's headerData - # The model defines the columns and headers - header = self.preview_table_view.horizontalHeader() - # Set resize modes for detailed columns - header.setSectionResizeMode(self.preview_model.COL_STATUS, QHeaderView.ResizeMode.ResizeToContents) - header.setSectionResizeMode(self.preview_model.COL_PREDICTED_ASSET, QHeaderView.ResizeMode.ResizeToContents) # Fit - header.setSectionResizeMode(self.preview_model.COL_DETAILS, QHeaderView.ResizeMode.ResizeToContents) # Fit - header.setSectionResizeMode(self.preview_model.COL_ORIGINAL_PATH, QHeaderView.ResizeMode.ResizeToContents) # Fixed width (using ResizeToContents as closest) - header.setSectionResizeMode(self.preview_model.COL_ADDITIONAL_FILES, QHeaderView.ResizeMode.Stretch) # Stretch (Fit-If-Possible) - - # Hide the Predicted Output column - self.preview_table_view.setColumnHidden(self.preview_model.COL_PREDICTED_OUTPUT, True) - - # Set selection behavior and alternating colors - self.preview_table_view.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers) - self.preview_table_view.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows) - self.preview_table_view.setAlternatingRowColors(False) - - # Enable sorting via header clicks - self.preview_table_view.setSortingEnabled(True) - # Set default sort column (Status) - the proxy model's lessThan handles the custom order - self.preview_table_view.sortByColumn(self.preview_model.COL_STATUS, Qt.SortOrder.AscendingOrder) - - # Move columns to the desired order: Status, Predicted Asset, Details, Original Path, Additional Files - # Initial logical order: [0, 1, 2, 3(hidden), 4, 5] - # Initial visual order: [0, 1, 2, 3, 4, 5] (assuming no initial moves) - # Desired visual order: [0, 1, 4, 2, 5, 3(hidden)] - - # Move Predicted Asset (logical 1) to visual index 1 (already there) - - # Move Details (logical 4) to visual index 2 - header.moveSection(header.visualIndex(self.preview_model.COL_DETAILS), 2) - # Current visual: [0, 1, 4, 2, 3, 5] - - # Move Original Path (logical 2) to visual index 3 - header.moveSection(header.visualIndex(self.preview_model.COL_ORIGINAL_PATH), 3) - # Current visual: [0, 1, 4, 2, 3, 5] - Original Path is already at visual index 3 after moving Details - - # Move Additional Files (logical 5) to visual index 4 - header.moveSection(header.visualIndex(self.preview_model.COL_ADDITIONAL_FILES), 4) - # Current visual: [0, 1, 4, 2, 5, 3] - This looks correct. - - main_layout.addWidget(self.preview_label) - - # Add placeholder label for the preview area - self.preview_placeholder_label = QLabel("Please select a preset to view file predictions") - self.preview_placeholder_label.setAlignment(Qt.AlignmentFlag.AlignCenter) - self.preview_placeholder_label.setStyleSheet("QLabel { font-size: 16px; color: grey; }") # Optional styling - - # Add both the table view and the placeholder label to the layout - # We will manage their visibility later - main_layout.addWidget(self.preview_placeholder_label, 1) # Give it stretch factor - main_layout.addWidget(self.preview_table_view, 1) # Give it stretch factor - - # Initially hide the table view and show the placeholder - self.preview_table_view.setVisible(False) - self.preview_placeholder_label.setVisible(True) - - # Apply style sheet to remove borders and rounded corners - self.preview_table_view.setStyleSheet("QTableView { border: none; }") - - # --- Progress Bar --- - self.progress_bar = QProgressBar() - self.progress_bar.setValue(0) - self.progress_bar.setTextVisible(True) - main_layout.addWidget(self.progress_bar) - - # --- Blender Integration Controls --- - blender_group = QGroupBox("Blender Post-Processing") - blender_layout = QVBoxLayout(blender_group) - - self.blender_integration_checkbox = QCheckBox("Run Blender Scripts After Processing") - self.blender_integration_checkbox.setToolTip("If checked, attempts to run create_nodegroups.py and create_materials.py in Blender.") - blender_layout.addWidget(self.blender_integration_checkbox) - - # Nodegroup Blend Path - nodegroup_layout = QHBoxLayout() - nodegroup_layout.addWidget(QLabel("Nodegroup .blend:")) - self.nodegroup_blend_path_input = QLineEdit() - self.browse_nodegroup_blend_button = QPushButton("...") - self.browse_nodegroup_blend_button.setFixedWidth(30) - self.browse_nodegroup_blend_button.clicked.connect(self._browse_for_nodegroup_blend) - nodegroup_layout.addWidget(self.nodegroup_blend_path_input) - nodegroup_layout.addWidget(self.browse_nodegroup_blend_button) - blender_layout.addLayout(nodegroup_layout) - - # Materials Blend Path - materials_layout = QHBoxLayout() - materials_layout.addWidget(QLabel("Materials .blend:")) - self.materials_blend_path_input = QLineEdit() - self.browse_materials_blend_button = QPushButton("...") - self.browse_materials_blend_button.setFixedWidth(30) - self.browse_materials_blend_button.clicked.connect(self._browse_for_materials_blend) - materials_layout.addWidget(self.materials_blend_path_input) - materials_layout.addWidget(self.browse_materials_blend_button) - blender_layout.addLayout(materials_layout) - - # Initialize paths from config - try: - default_ng_path = getattr(core_config, 'DEFAULT_NODEGROUP_BLEND_PATH', '') - default_mat_path = getattr(core_config, 'DEFAULT_MATERIALS_BLEND_PATH', '') - self.nodegroup_blend_path_input.setText(default_ng_path if default_ng_path else "") - self.materials_blend_path_input.setText(default_mat_path if default_mat_path else "") - except Exception as e: - log.error(f"Error reading default Blender paths from config: {e}") - - # Disable Blender controls initially if checkbox is unchecked - self.nodegroup_blend_path_input.setEnabled(False) - self.browse_nodegroup_blend_button.setEnabled(False) - self.materials_blend_path_input.setEnabled(False) - self.browse_materials_blend_button.setEnabled(False) - self.blender_integration_checkbox.toggled.connect(self._toggle_blender_controls) - - main_layout.addWidget(blender_group) # Add the group box to the main layout - - # --- Bottom Controls --- - bottom_controls_layout = QHBoxLayout() - self.overwrite_checkbox = QCheckBox("Overwrite Existing") - self.overwrite_checkbox.setToolTip("If checked, existing output folders for processed assets will be deleted and replaced.") - bottom_controls_layout.addWidget(self.overwrite_checkbox) - - # self.disable_preview_checkbox = QCheckBox("Disable Detailed Preview") # REMOVED - Moved to View Menu - # self.disable_preview_checkbox.setToolTip("If checked, shows only the list of input assets instead of detailed file predictions.") - # self.disable_preview_checkbox.setChecked(False) # Default is detailed preview enabled - # self.disable_preview_checkbox.toggled.connect(self.update_preview) # Update preview when toggled - # bottom_controls_layout.addWidget(self.disable_preview_checkbox) - - # bottom_controls_layout.addSpacing(20) # Add some space # REMOVED - No longer needed after checkbox removal - - self.workers_label = QLabel("Workers:") - self.workers_spinbox = QSpinBox() - default_workers = 1 - try: - cores = os.cpu_count() - if cores: default_workers = max(1, cores // 2) - except NotImplementedError: pass - self.workers_spinbox.setMinimum(1) - self.workers_spinbox.setMaximum(os.cpu_count() or 32) - self.workers_spinbox.setValue(default_workers) - self.workers_spinbox.setToolTip("Number of assets to process concurrently.") - bottom_controls_layout.addWidget(self.workers_label) - bottom_controls_layout.addWidget(self.workers_spinbox) - bottom_controls_layout.addStretch(1) - self.clear_queue_button = QPushButton("Clear Queue") # Added Clear button - self.start_button = QPushButton("Start Processing") - self.cancel_button = QPushButton("Cancel") - self.cancel_button.setEnabled(False) - self.clear_queue_button.clicked.connect(self.clear_queue) # Connect signal - self.start_button.clicked.connect(self.start_processing) - self.cancel_button.clicked.connect(self.cancel_processing) - bottom_controls_layout.addWidget(self.clear_queue_button) # Add button to layout - bottom_controls_layout.addWidget(self.start_button) - bottom_controls_layout.addWidget(self.cancel_button) - main_layout.addLayout(bottom_controls_layout) - - # --- Preset Population and Handling --- - - def populate_presets(self): - """Scans presets dir and populates BOTH the editor list and processing combo.""" - log.debug("Populating preset list...") - # Store current list selection - current_list_item = self.editor_preset_list.currentItem() - current_list_selection_text = current_list_item.text() if current_list_item else None - - # Clear list - self.editor_preset_list.clear() - log.debug("Preset list cleared.") - - # Add the "Select a Preset" placeholder item - placeholder_item = QListWidgetItem("--- Select a Preset ---") - # Make it non-selectable and non-editable - placeholder_item.setFlags(placeholder_item.flags() & ~Qt.ItemFlag.ItemIsSelectable & ~Qt.ItemFlag.ItemIsEditable) - # Set unique data to identify the placeholder - placeholder_item.setData(Qt.ItemDataRole.UserRole, "__PLACEHOLDER__") - self.editor_preset_list.addItem(placeholder_item) - log.debug("Added '--- Select a Preset ---' placeholder item.") - - if not PRESETS_DIR.is_dir(): - msg = f"Error: Presets directory not found at {PRESETS_DIR}" - self.statusBar().showMessage(msg) - log.error(msg) - return - - # Exclude template file starting with _ - presets = sorted([f for f in PRESETS_DIR.glob("*.json") if f.is_file() and not f.name.startswith('_')]) - preset_names = [p.stem for p in presets] - - if not presets: - msg = "Warning: No presets found in presets directory." - self.statusBar().showMessage(msg) - log.warning(msg) - else: - # Populate List Widget (for editing) - for preset_path in presets: - item = QListWidgetItem(preset_path.stem) - item.setData(Qt.ItemDataRole.UserRole, preset_path) # Store full path - self.editor_preset_list.addItem(item) - - self.statusBar().showMessage(f"Loaded {len(presets)} presets.") - - # Try to restore list selection - # combo_index = self.preset_combo.findText(current_combo_selection) # REMOVED - # if combo_index != -1: # REMOVED - # self.preset_combo.setCurrentIndex(combo_index) # REMOVED - - # Do NOT attempt to restore list selection by default - self.statusBar().showMessage(f"Loaded {len(presets)} presets.") - - # Select the "Select a Preset" item by default - log.debug("Preset list populated. Selecting '--- Select a Preset ---' item.") - self.editor_preset_list.setCurrentItem(placeholder_item) # Select the placeholder item - - # --- Drag and Drop Event Handlers (Unchanged) --- def dragEnterEvent(self, event): if event.mimeData().hasUrls(): event.acceptProposedAction() else: event.ignore() @@ -619,203 +246,427 @@ class MainWindow(QMainWindow): self.add_input_paths(paths) else: event.ignore() + def _extract_file_list(self, input_path_str: str) -> list | None: + """Extracts a list of relative file paths from a directory or zip archive.""" + input_path = Path(input_path_str) + file_list = [] + try: + if input_path.is_dir(): + log.debug(f"Extracting files from directory: {input_path_str}") + for root, _, files in os.walk(input_path): + for file in files: + full_path = Path(root) / file + # Use POSIX paths for consistency + relative_path = full_path.relative_to(input_path).as_posix() + file_list.append(relative_path) + log.debug(f"Found {len(file_list)} files in directory.") + elif input_path.is_file() and input_path.suffix.lower() == '.zip': + log.debug(f"Extracting files from zip archive: {input_path_str}") + if not zipfile.is_zipfile(input_path): + log.warning(f"File is not a valid zip archive: {input_path_str}") + return None + with zipfile.ZipFile(input_path, 'r') as zip_ref: + # Filter out directory entries if any exist in the zip explicitly + file_list = [name for name in zip_ref.namelist() if not name.endswith('/')] + log.debug(f"Found {len(file_list)} files in zip archive.") + else: + log.warning(f"Input path is neither a directory nor a supported .zip file: {input_path_str}") + return None + return file_list + except FileNotFoundError: + log.error(f"File or directory not found during extraction: {input_path_str}") + self.statusBar().showMessage(f"Error: Input not found: {input_path.name}", 5000) + return None + except zipfile.BadZipFile: + log.error(f"Bad zip file encountered: {input_path_str}") + self.statusBar().showMessage(f"Error: Invalid zip file: {input_path.name}", 5000) + return None + except PermissionError: + log.error(f"Permission denied accessing: {input_path_str}") + self.statusBar().showMessage(f"Error: Permission denied for: {input_path.name}", 5000) + return None + except Exception as e: + log.exception(f"Unexpected error extracting files from {input_path_str}: {e}") + self.statusBar().showMessage(f"Error extracting files from: {input_path.name}", 5000) + return None + def add_input_paths(self, paths): + log.debug(f"--> Entered add_input_paths with paths: {paths}") if not hasattr(self, 'current_asset_paths'): self.current_asset_paths = set() added_count = 0 newly_added_paths = [] for p_str in paths: p = Path(p_str) if p.exists(): - supported_suffixes = ['.zip', '.rar', '.7z'] - if p.is_dir() or (p.is_file() and p.suffix.lower() in supported_suffixes): + # Only support directories and .zip files for now + if p.is_dir() or (p.is_file() and p.suffix.lower() == '.zip'): if p_str not in self.current_asset_paths: self.current_asset_paths.add(p_str) newly_added_paths.append(p_str) added_count += 1 - else: log.debug(f"Skipping duplicate asset path: {p_str}") # Changed print to log.debug - else: self.statusBar().showMessage(f"Invalid input (not dir or supported archive): {p.name}", 5000); log.warning(f"Invalid input: {p_str}") # Changed print to log.warning and updated message + else: log.debug(f"Skipping duplicate asset path: {p_str}") + else: self.statusBar().showMessage(f"Invalid input (not dir or .zip): {p.name}", 5000); log.warning(f"Invalid input (not dir or .zip): {p_str}") else: self.statusBar().showMessage(f"Input path not found: {p.name}", 5000); print(f"Input path not found: {p_str}") if added_count > 0: log.info(f"Added {added_count} new asset paths: {newly_added_paths}") self.statusBar().showMessage(f"Added {added_count} asset(s). Updating preview...", 3000) - # --- Auto-disable detailed preview if > 10 assets --- - preview_toggled = False - if hasattr(self, 'toggle_preview_action') and len(self.current_asset_paths) > 10: - if not self.toggle_preview_action.isChecked(): # Only check it if it's not already checked - log.info(f"Asset count ({len(self.current_asset_paths)}) > 10. Forcing simple preview.") - self.toggle_preview_action.setChecked(True) # This will trigger update_preview via its signal - preview_toggled = True + mode, selected_preset_text = self.preset_editor_widget.get_selected_preset_mode() - # Only call update_preview directly if the toggle wasn't triggered - # If in simple mode, we need to explicitly update the model with the simple list of paths - if hasattr(self, 'toggle_preview_action') and self.toggle_preview_action.isChecked(): - log.debug("Currently in simple preview mode. Updating model with simple paths.") - self.preview_model.set_data(list(self.current_asset_paths)) # Update model with simple list - self.statusBar().showMessage(f"Added {added_count} asset(s). Preview updated.", 3000) - # Only call update_preview if a preset is currently selected in the editor list - current_editor_item = self.editor_preset_list.currentItem() - if not preview_toggled and current_editor_item: - log.debug("Preset selected and not in simple mode. Triggering detailed preview update.") - self.update_preview() - elif not current_editor_item: - log.debug("No preset selected. Not triggering detailed preview update.") - self.statusBar().showMessage(f"Added {added_count} asset(s). Select a preset to update preview.", 3000) - - def _browse_for_output_directory(self): - """Opens a dialog to select the output directory.""" - current_path = self.output_path_edit.text() - if not current_path or not Path(current_path).is_dir(): - # Fallback to project root or home directory if current path is invalid - current_path = str(project_root) + if mode == "llm": + log.info(f"LLM Interpretation selected. Preparing LLM prediction for {len(newly_added_paths)} new paths.") + llm_requests_to_queue = [] + for input_path_str in newly_added_paths: + file_list = self._extract_file_list(input_path_str) + if file_list is not None: + log.info(f"Extracted {len(file_list)} files for LLM prediction from: {input_path_str}") + self._source_file_lists[input_path_str] = file_list + # Use the same pending set for now + self._pending_predictions.add(input_path_str) + llm_requests_to_queue.append((input_path_str, file_list)) + else: + log.warning(f"Skipping LLM prediction queuing for {input_path_str} due to extraction error.") + if llm_requests_to_queue: + log.info(f"Delegating {len(llm_requests_to_queue)} LLM requests to the handler.") + self.llm_interaction_handler.queue_llm_requests_batch(llm_requests_to_queue) + # The handler manages starting its own processing internally. + elif mode == "preset" and selected_preset_text: + log.info(f"Preset '{selected_preset_text}' selected. Triggering prediction for {len(newly_added_paths)} new paths.") + if self.prediction_thread and not self.prediction_thread.isRunning(): + log.debug("Starting prediction thread from add_input_paths.") + self.prediction_thread.start() + for input_path_str in newly_added_paths: + file_list = self._extract_file_list(input_path_str) + if file_list is not None: + log.debug(f"Extracted {len(file_list)} files for {input_path_str}. Emitting signal.") + log.info(f"VERIFY: Extracted file list for '{input_path_str}'. Count: {len(file_list)}. Emitting prediction signal.") + self._source_file_lists[input_path_str] = file_list + self._pending_predictions.add(input_path_str) + log.debug(f"Added '{input_path_str}' to pending predictions. Current pending: {self._pending_predictions}") + self.start_prediction_signal.emit(input_path_str, file_list, selected_preset_text) + else: + log.warning(f"Skipping prediction for {input_path_str} due to extraction error.") + elif mode == "placeholder": + log.info(f"Added {added_count} asset(s) while placeholder selected. Adding directories with file contents to view.") + rules_to_add = [] + for input_path_str in newly_added_paths: + input_path = Path(input_path_str) + if input_path.is_dir(): + log.debug(f"Processing directory in placeholder mode: {input_path_str}") + file_rules = [] + try: + for item_name in os.listdir(input_path): + item_path = input_path / item_name + if item_path.is_file(): + relative_path = item_name + file_rules.append(FileRule(file_path=relative_path, map_type="")) + log.debug(f" Found file: {relative_path}") + except OSError as e: + log.warning(f"Could not list directory contents for {input_path_str}: {e}") + # Optionally add the directory itself even if listing fails + continue - directory = QFileDialog.getExistingDirectory( - self, - "Select Output Directory", - current_path, - QFileDialog.Option.ShowDirsOnly | QFileDialog.Option.DontResolveSymlinks - ) - if directory: - self.output_path_edit.setText(directory) - log.info(f"User selected output directory: {directory}") + dummy_asset = AssetRule(asset_name="", asset_type="", files=file_rules) + source_rule = SourceRule(input_path=input_path_str, assets=[dummy_asset]) + rules_to_add.append(source_rule) + log.debug(f"Created SourceRule with {len(file_rules)} child files for directory: {input_path_str}") + + elif input_path.is_file() and input_path.suffix.lower() == '.zip': + log.debug(f"Processing zip file in placeholder mode (inspecting contents): {input_path_str}") + file_rules = [] + try: + if not zipfile.is_zipfile(input_path): + log.warning(f"File is not a valid zip archive: {input_path_str}") + self.statusBar().showMessage(f"Warning: Not a valid zip: {input_path.name}", 5000) + continue + + with zipfile.ZipFile(input_path, 'r') as zip_ref: + for name in zip_ref.namelist(): + # Filter out directory entries explicitly marked with '/' + if not name.endswith('/'): + file_rules.append(FileRule(file_path=name)) + log.debug(f" Found file in zip: {name}") + + # This structure allows the UnifiedViewModel to display it hierarchically + dummy_asset = AssetRule(asset_name="", asset_type="", files=file_rules) + source_rule = SourceRule(input_path=input_path_str, assets=[dummy_asset]) + rules_to_add.append(source_rule) + log.debug(f"Created SourceRule with {len(file_rules)} child files for zip archive: {input_path_str}") + + except zipfile.BadZipFile: + log.error(f"Bad zip file encountered: {input_path_str}") + self.statusBar().showMessage(f"Error: Invalid zip file: {input_path.name}", 5000) + continue + except FileNotFoundError: # Should ideally not happen due to earlier checks + log.error(f"File not found during zip processing: {input_path_str}") + self.statusBar().showMessage(f"Error: Input not found: {input_path.name}", 5000) + continue + except PermissionError: + log.error(f"Permission denied accessing zip: {input_path_str}") + self.statusBar().showMessage(f"Error: Permission denied for: {input_path.name}", 5000) + continue + except Exception as e: + log.exception(f"Unexpected error processing zip file {input_path_str}: {e}") + self.statusBar().showMessage(f"Error reading zip: {input_path.name}", 5000) + continue + else: + # This case should ideally not be reached due to earlier checks, but log just in case + log.warning(f"Skipping unexpected item type in placeholder mode: {input_path_str}") + + if rules_to_add: + try: + log.info(f"Updating model with {len(rules_to_add)} SourceRules (placeholder mode with directory contents).") + self.unified_model.update_rules_for_sources(rules_to_add) + if hasattr(self, 'main_panel_widget'): + self.main_panel_widget.unified_view.expandToDepth(1) + self.statusBar().showMessage(f"Added {len(rules_to_add)} item(s) to view. Select preset/LLM for details.", 3000) + except Exception as e: + log.exception(f"Error updating model with placeholder rules: {e}") + self.statusBar().showMessage(f"Error adding items to view: {e}", 5000) + else: + # This might happen if only non-dir/zip items were added or directory listing failed for all + self.statusBar().showMessage(f"Added {added_count} input(s), but no valid items to display in placeholder mode.", 5000) + + else: + log.error(f"Added {added_count} asset(s), but encountered unexpected preset mode: {mode}. Prediction not triggered.") + self.statusBar().showMessage(f"Added {added_count} asset(s). Error determining preset mode.", 3000) + + # The preview update is now triggered per-item via the signal emission above, + # and also when the preset selection changes (handled in update_preview). - # --- Processing Action Methods --- - def start_processing(self): - if self.processing_handler and self.processing_handler.is_running: - log.warning("Start clicked, but processing is already running.") - self.statusBar().showMessage("Processing is already in progress.", 3000) - return - if ProcessingHandler is None: - self.statusBar().showMessage("Error: Processing components not loaded.", 5000) - return + # --- Slots for Handling Requests from MainPanelWidget --- + + @Slot(dict) + def _on_process_requested(self, settings: dict): + """Handles the process_requested signal from the MainPanelWidget.""" + log.info(f"Received process request signal with settings: {settings}") + if not hasattr(self, 'current_asset_paths') or not self.current_asset_paths: self.statusBar().showMessage("No assets added to process.", 3000) return - input_paths = list(self.current_asset_paths) - if not input_paths: - self.statusBar().showMessage("No assets added to process.", 3000) - return - # --- Get preset from editor list --- - current_editor_item = self.editor_preset_list.currentItem() - # Check if the selected item is the placeholder - is_placeholder = current_editor_item and current_editor_item.data(Qt.ItemDataRole.UserRole) == "__PLACEHOLDER__" + mode, selected_preset_name = self.preset_editor_widget.get_selected_preset_mode() - if is_placeholder: - self.statusBar().showMessage("Please select a valid preset from the list on the left.", 3000) - log.warning("Start processing failed: Placeholder preset selected.") - return - # Existing logic to get selected_preset text and proceed - selected_preset = current_editor_item.text() if current_editor_item else None - overwrite = self.overwrite_checkbox.isChecked() - num_workers = self.workers_spinbox.value() - - # --- Get Output Directory from UI and Validate --- - output_dir_str = self.output_path_edit.text().strip() + output_dir_str = settings.get("output_dir") if not output_dir_str: self.statusBar().showMessage("Error: Output directory cannot be empty.", 5000) log.error("Start processing failed: Output directory field is empty.") return try: output_dir = Path(output_dir_str) - # Attempt to create the directory if it doesn't exist output_dir.mkdir(parents=True, exist_ok=True) - # Basic writability check (create and delete a temp file) - # Note: This isn't foolproof due to potential race conditions/permissions issues - # but provides a basic level of validation. temp_file = output_dir / f".writable_check_{time.time()}" temp_file.touch() temp_file.unlink() log.info(f"Using validated output directory: {output_dir_str}") + self._current_output_dir = output_dir_str except OSError as e: error_msg = f"Error creating/accessing output directory '{output_dir_str}': {e}" self.statusBar().showMessage(error_msg, 5000) log.error(error_msg) return - except Exception as e: # Catch other potential Path errors or unexpected issues + except Exception as e: error_msg = f"Invalid output directory path '{output_dir_str}': {e}" self.statusBar().showMessage(error_msg, 5000) log.error(error_msg) return - # --- End Output Directory Validation --- - log.info(f"Preparing to start processing {len(input_paths)} items to '{output_dir_str}'.") - self.set_controls_enabled(False) - self.cancel_button.setEnabled(True) - self.start_button.setText("Processing...") - self.progress_bar.setValue(0) - self.progress_bar.setFormat("%p%") - self.setup_threads() - if self.processing_thread and self.processing_handler: - try: self.processing_thread.started.disconnect() - except RuntimeError: pass - self.processing_thread.started.connect( - lambda: self.processing_handler.run_processing( - input_paths, selected_preset, output_dir_str, overwrite, num_workers, - # Pass Blender integration settings - run_blender=self.blender_integration_checkbox.isChecked(), - nodegroup_blend_path=self.nodegroup_blend_path_input.text(), - materials_blend_path=self.materials_blend_path_input.text(), - # Pass verbose setting - verbose=self.toggle_verbose_action.isChecked() - ) - ) - self.processing_thread.start() - log.info("Processing thread started.") - self.statusBar().showMessage(f"Processing {len(input_paths)} items...", 0) - else: - log.error("Failed to start processing: Thread or handler not initialized.") - self.statusBar().showMessage("Error: Failed to initialize processing thread.", 5000) - self.set_controls_enabled(True) - self.cancel_button.setEnabled(False) - self.start_button.setText("Start Processing") - - def cancel_processing(self): - if self.processing_handler and self.processing_handler.is_running: - log.info("Cancel button clicked. Requesting cancellation.") - self.statusBar().showMessage("Requesting cancellation...", 3000) - self.processing_handler.request_cancel() - self.cancel_button.setEnabled(False) - self.start_button.setText("Cancelling...") - else: - log.warning("Cancel clicked, but no processing is running.") - self.statusBar().showMessage("Nothing to cancel.", 3000) - - def clear_queue(self): - """Clears the current asset queue and the preview table.""" - if self.processing_handler and self.processing_handler.is_running: - self.statusBar().showMessage("Cannot clear queue while processing.", 3000) + try: + final_source_rules = self.unified_model.get_all_source_rules() + if not final_source_rules: + log.warning("No source rules found in the model. Nothing to process.") + self.statusBar().showMessage("No rules generated or assets added. Nothing to process.", 3000) + return + except AttributeError: + log.error("UnifiedViewModel does not have 'get_all_source_rules()' method.") + self.statusBar().showMessage("Error: Cannot retrieve rules from model.", 5000) return + except Exception as e: + log.exception(f"Error getting rules from model: {e}") + self.statusBar().showMessage(f"Error retrieving rules: {e}", 5000) + return + + log.info(f"Retrieved {len(final_source_rules)} SourceRule objects from the model.") + + filtered_source_rules = [] + for source_rule in final_source_rules: + has_valid_target = False + if hasattr(source_rule, 'assets') and source_rule.assets: + for asset_rule in source_rule.assets: + # Check if asset_name (Target Asset) is not None and not empty/whitespace + if asset_rule.asset_name and asset_rule.asset_name.strip(): + has_valid_target = True + # Found one valid target, no need to check others in this source + break + if has_valid_target: + filtered_source_rules.append(source_rule) + else: + log.info(f"Filtering out SourceRule '{source_rule.input_path}' as it has no assets with a Target Asset name.") + + if not filtered_source_rules: + log.warning("All SourceRules were filtered out. No items have a valid Target Asset. Nothing to process.") + self.statusBar().showMessage("No items have a Target Asset assigned. Nothing to process.", 5000) + self.set_controls_enabled(True) + self.main_panel_widget.set_start_button_text("Start Processing") + self.main_panel_widget.set_cancel_button_enabled(False) + self.main_panel_widget.set_progress_bar_text("Idle") + return + + log.info(f"Processing {len(filtered_source_rules)} SourceRule objects after filtering (originally {len(final_source_rules)}).") + + self.main_panel_widget.set_progress_bar_text("Waiting for processing start...") + self.statusBar().showMessage(f"Requested processing for {len(filtered_source_rules)} rule sets...", 0) + self.set_controls_enabled(False) + self.main_panel_widget.set_start_button_enabled(False) + self.main_panel_widget.set_start_button_text("Processing...") + self.main_panel_widget.set_cancel_button_enabled(True) + + processing_data = { + "output_dir": output_dir_str, + "overwrite": settings.get("overwrite", False), + "workers": settings.get("workers", 1), + "blender_enabled": settings.get("blender_enabled", False), + "nodegroup_blend_path": settings.get("nodegroup_blend_path", ""), + "materials_blend_path": settings.get("materials_blend_path", "") + } + log.info(f"Emitting start_backend_processing with {len(filtered_source_rules)} rules and settings: {processing_data}") + self.start_backend_processing.emit(filtered_source_rules, processing_data) + + @Slot() + def _on_cancel_requested(self): + """Handles the cancel_requested signal from the MainPanelWidget.""" + # TODO: Implement cancellation by signaling the App/main thread to stop the QThreadPool tasks + log.warning("Cancel requested, but cancellation logic needs reimplementation in main application.") + self.statusBar().showMessage("Cancellation request sent (implementation pending).", 3000) + # Optionally, re-enable controls immediately or wait for confirmation + + @Slot() + def _on_clear_queue_requested(self): + """Handles the clear_queue_requested signal from the MainPanelWidget.""" + # TODO: Check processing state via App/main thread if needed before clearing if hasattr(self, 'current_asset_paths') and self.current_asset_paths: log.info(f"Clearing asset queue ({len(self.current_asset_paths)} items).") self.current_asset_paths.clear() - self.preview_model.clear_data() # Clear the model data - self.statusBar().showMessage("Asset queue cleared.", 3000) + self.unified_model.clear_data() + if hasattr(self, 'main_panel_widget'): + self.main_panel_widget.set_start_button_enabled(False) + self._pending_predictions.clear() + self._accumulated_rules.clear() + self._source_file_lists.clear() + self.llm_interaction_handler.clear_queue() + log.info("Cleared accumulation state and delegated LLM queue clear.") + self.statusBar().showMessage("Asset queue and prediction state cleared.", 3000) + self.main_panel_widget.set_progress_bar_text("Idle") else: self.statusBar().showMessage("Asset queue is already empty.", 3000) + @Slot(list) + def _delegate_llm_reinterpret(self, source_paths: list): + """ + Slot to receive the llm_reinterpret_requested signal from MainPanelWidget + and delegate the request to the LLMInteractionHandler. + """ + log.info(f"Received LLM re-interpret request for {len(source_paths)} paths. Delegating to handler.") + + if not source_paths: + self.statusBar().showMessage("No valid source directories selected for re-interpretation.", 5000) + return + + # Check handler status before queueing (optional, handler manages internally) + if self.llm_interaction_handler.is_processing(): + QMessageBox.warning(self, "Busy", "LLM interpretation is already in progress. Request added to queue.") + # Proceed to queue anyway, handler manages the queue + + requests = [(path, None) for path in source_paths] + + self.llm_interaction_handler.queue_llm_requests_batch(requests) + # Status updates (like "Added X directories to queue") will come from the handler via signals + + @Slot(str) + def _on_output_dir_changed(self, path: str): + """Stores the output directory path when it changes in the panel.""" + self._current_output_dir = path + log.debug(f"MainWindow stored output directory: {path}") + + @Slot(bool, str, str) + def _on_blender_settings_changed(self, enabled: bool, ng_path: str, mat_path: str): + """Stores the Blender settings when they change in the panel.""" + self._current_blender_settings = { + "enabled": enabled, + "nodegroup_blend_path": ng_path, + "materials_blend_path": mat_path + } + log.debug(f"MainWindow stored Blender settings: {self._current_blender_settings}") + @Slot(list, str) + def _on_preset_reinterpret_requested(self, source_paths: list, preset_name: str): + """Handles the preset_reinterpret_requested signal from MainPanelWidget.""" + log.info(f"Received preset re-interpret request for {len(source_paths)} paths using preset '{preset_name}'.") + + if not source_paths: + self.statusBar().showMessage("No valid source directories selected for preset re-interpretation.", 5000) + return + + # Check if rule-based prediction is already running (optional, handler might manage internally) + # Note: QueuedConnection on the signal helps, but check anyway for immediate feedback/logging + # TODO: Add is_running() method to RuleBasedPredictionHandler if needed for this check - NOTE: is_running is a property now + if self.prediction_handler and hasattr(self.prediction_handler, 'is_running') and self.prediction_handler.is_running: + log.warning("Rule-based prediction is already running. Queuing re-interpretation request.") + # Proceed, relying on QueuedConnection + + if self.prediction_thread and not self.prediction_thread.isRunning(): + log.debug("Starting prediction thread for preset re-interpretation.") + self.prediction_thread.start() + elif not self.prediction_thread: + log.error("Prediction thread not initialized. Cannot perform preset re-interpretation.") + self.statusBar().showMessage("Error: Prediction system not ready.", 5000) + return + + + self.statusBar().showMessage(f"Starting re-interpretation for {len(source_paths)} item(s) using preset '{preset_name}'...", 0) + for input_path_str in source_paths: + self._pending_predictions.add(input_path_str) + self._completed_predictions.discard(input_path_str) + + # Update status in model (Requires update_status method in UnifiedViewModel) + try: + if hasattr(self.unified_model, 'update_status'): + self.unified_model.update_status(input_path_str, "Re-interpreting...") + else: + log.warning("UnifiedViewModel does not have 'update_status' method. Cannot update status visually.") + except Exception as e: + log.exception(f"Error calling unified_model.update_status for {input_path_str}: {e}") + + + file_list = self._extract_file_list(input_path_str) + if file_list is not None: + log.debug(f"Emitting start_prediction_signal for re-interpretation: Path='{input_path_str}', Preset='{preset_name}'") + self.start_prediction_signal.emit(input_path_str, file_list, preset_name) + else: + log.warning(f"Skipping re-interpretation for {input_path_str} due to extraction error.") + # Update status in model to reflect error (Requires update_status method) + try: + if hasattr(self.unified_model, 'update_status'): + self.unified_model.update_status(input_path_str, "Error extracting files") + else: + log.warning("UnifiedViewModel does not have 'update_status' method. Cannot update error status visually.") + except Exception as e: + log.exception(f"Error calling unified_model.update_status (error case) for {input_path_str}: {e}") + + self._handle_prediction_completion(input_path_str) - # --- Preview Update Method --- def update_preview(self): log.info(f"--> Entered update_preview. View Action exists: {hasattr(self, 'toggle_preview_action')}") - if hasattr(self, 'toggle_preview_action'): - log.info(f" Disable Preview Action checked: {self.toggle_preview_action.isChecked()}") - - # --- Preview Update Method --- - def update_preview(self): - thread_id = QThread.currentThread() # Get current thread object + log.debug(f"[{time.time():.4f}] ### LOG: Entering update_preview") + log.debug("--> Entered update_preview") + thread_id = QThread.currentThread() log.info(f"[{time.time():.4f}][T:{thread_id}] --> Entered update_preview. View Action exists: {hasattr(self, 'toggle_preview_action')}") if hasattr(self, 'toggle_preview_action'): log.info(f"[{time.time():.4f}][T:{thread_id}] Disable Preview Action checked: {self.toggle_preview_action.isChecked()}") - # Determine mode based on menu action - simple_mode_enabled = hasattr(self, 'toggle_preview_action') and self.toggle_preview_action.isChecked() - log.info(f"[{time.time():.4f}][T:{thread_id}] Determined simple_mode_enabled: {simple_mode_enabled}") - # --- Cancel Prediction if Running --- if self.prediction_handler and self.prediction_handler.is_running: log.warning(f"[{time.time():.4f}][T:{thread_id}] Prediction is running. Attempting to call prediction_handler.request_cancel()...") try: @@ -829,242 +680,136 @@ class MainWindow(QMainWindow): # Note: Cancellation is not immediate even if it existed. The thread would stop when it next checks the flag. # We proceed with updating the UI immediately. - # Set the model's mode - log.info(f"[{time.time():.4f}][T:{thread_id}] Calling preview_model.set_simple_mode({simple_mode_enabled})...") - self.preview_model.set_simple_mode(simple_mode_enabled) - log.info(f"[{time.time():.4f}][T:{thread_id}] Returned from preview_model.set_simple_mode({simple_mode_enabled}).") - # Configure the QTableView based on the mode - header = self.preview_table_view.horizontalHeader() - if simple_mode_enabled: - log.info(" Configuring QTableView for SIMPLE mode.") - # Hide detailed columns, show simple column - self.preview_table_view.setColumnHidden(self.preview_model.COL_STATUS, True) - self.preview_table_view.setColumnHidden(self.preview_model.COL_PREDICTED_ASSET, True) - self.preview_table_view.setColumnHidden(self.preview_model.COL_ORIGINAL_PATH, True) - self.preview_table_view.setColumnHidden(self.preview_model.COL_PREDICTED_OUTPUT, True) # Already hidden, but good practice - self.preview_table_view.setColumnHidden(self.preview_model.COL_DETAILS, True) - # Ensure the simple path column exists and is visible - if self.preview_model.columnCount() > self.preview_model.COL_SIMPLE_PATH: - self.preview_table_view.setColumnHidden(self.preview_model.COL_SIMPLE_PATH, False) # Show the simple path column - # Set resize mode for the single visible column - header.setSectionResizeMode(self.preview_model.COL_SIMPLE_PATH, QHeaderView.ResizeMode.Stretch) - else: - log.error(" Simple path column index out of bounds for model.") + log.debug(f"[{time.time():.4f}] ### LOG: Checking if prediction handler is running") + if self.prediction_handler and self.prediction_handler.is_running: + log.warning(f"[{time.time():.4f}] Preview update requested, but already running.") + log.debug(f"[{time.time():.4f}] ### LOG: Inside 'is_running' check") + # Removed the 'return' statement here to allow the signal to be emitted + # The rest of the logic should execute regardless of is_running state, + # though the handler itself should handle being called multiple times. + # A better fix might involve properly resetting is_running in the handler. - - # Disable sorting in simple mode (optional, but makes sense) - self.preview_table_view.setSortingEnabled(False) - - # Update status bar - if hasattr(self, 'current_asset_paths') and self.current_asset_paths: - self.statusBar().showMessage(f"Preview disabled. Showing {len(self.current_asset_paths)} input assets.", 3000) - else: - self.statusBar().showMessage("Preview disabled. No assets added.", 3000) - - # In simple mode, the model's data is derived from current_asset_paths. - # We need to ensure the model's simple data is up-to-date. - # The simplest way is to re-set the data, which will re-extract simple data. - # This might be slightly inefficient if only the mode changed, but safe. - # A more optimized approach would be to have a separate method in the model - # to just update the simple data from a list of paths. - # For now, let's re-set the data. - # --- REMOVED REDUNDANT set_data CALL --- - # The set_simple_mode(True) call above should be sufficient as the model - # already holds the simple data internally. This extra reset seems to cause instability. - # log.debug(" Simple mode enabled. Re-setting model data to trigger simple data update.") - # self.preview_model.set_data(list(self.current_asset_paths)) # Pass the list of paths - # --- END REMOVAL --- - - # Stop here, do not run PredictionHandler in simple mode - log.info(f"[{time.time():.4f}][T:{thread_id}] <-- Exiting update_preview (Simple Mode).") - return - else: - # --- Proceed with Detailed Preview --- - log.info(f"[{time.time():.4f}][T:{thread_id}] Configuring QTableView for DETAILED mode.") - # Show detailed columns, hide simple column - self.preview_table_view.setColumnHidden(self.preview_model.COL_STATUS, False) - self.preview_table_view.setColumnHidden(self.preview_model.COL_PREDICTED_ASSET, False) - self.preview_table_view.setColumnHidden(self.preview_model.COL_ORIGINAL_PATH, False) - self.preview_table_view.setColumnHidden(self.preview_model.COL_PREDICTED_OUTPUT, True) # Keep this hidden - self.preview_table_view.setColumnHidden(self.preview_model.COL_DETAILS, False) - # Ensure the simple path column exists and is hidden - if self.preview_model.columnCount() > self.preview_model.COL_SIMPLE_PATH: - self.preview_table_view.setColumnHidden(self.preview_model.COL_SIMPLE_PATH, True) # Hide the simple path column - else: - log.warning(" Simple path column index out of bounds for model when hiding.") - - - # Set resize modes for detailed columns - header.setSectionResizeMode(self.preview_model.COL_STATUS, QHeaderView.ResizeMode.ResizeToContents) - header.setSectionResizeMode(self.preview_model.COL_PREDICTED_ASSET, QHeaderView.ResizeMode.ResizeToContents) # Fit - header.setSectionResizeMode(self.preview_model.COL_DETAILS, QHeaderView.ResizeMode.ResizeToContents) # Fit - header.setSectionResizeMode(self.preview_model.COL_ORIGINAL_PATH, QHeaderView.ResizeMode.ResizeToContents) # Fixed width (using ResizeToContents as closest) - header.setSectionResizeMode(self.preview_model.COL_ADDITIONAL_FILES, QHeaderView.ResizeMode.Stretch) # Stretch (Fit-If-Possible) - - # Move columns to the desired order: Status, Predicted Asset, Details, Original Path, Additional Files - # Initial logical order: [0, 1, 2, 3(hidden), 4, 5] - # Initial visual order: [0, 1, 2, 3, 4, 5] (assuming no initial moves) - # Desired visual order: [0, 1, 4, 2, 5, 3(hidden)] - - # Move Predicted Asset (logical 1) to visual index 1 (already there) - - # Move Details (logical 4) to visual index 2 - header.moveSection(header.visualIndex(self.preview_model.COL_DETAILS), 2) - # Current visual: [0, 1, 4, 2, 3, 5] - - # Move Original Path (logical 2) to visual index 3 - header.moveSection(header.visualIndex(self.preview_model.COL_ORIGINAL_PATH), 3) - # Current visual: [0, 1, 4, 2, 3, 5] - Original Path is already at visual index 3 after moving Details - - # Move Additional Files (logical 5) to visual index 4 - header.moveSection(header.visualIndex(self.preview_model.COL_ADDITIONAL_FILES), 4) - # Current visual: [0, 1, 4, 2, 5, 3] - This looks correct. - - # Re-enable sorting for detailed mode - self.preview_table_view.setSortingEnabled(True) - # Reset sort order if needed (optional, proxy model handles default) - # self.preview_table_view.sortByColumn(self.preview_model.COL_STATUS, Qt.SortOrder.AscendingOrder) - - # --- Trigger Prediction Handler --- - if self.prediction_handler and self.prediction_handler.is_running: - log.warning(f"[{time.time():.4f}] Preview update requested, but already running.") - return - if PredictionHandler is None: + if RuleBasedPredictionHandler is None: + log.error("RuleBasedPredictionHandler not loaded. Cannot update preview.") self.statusBar().showMessage("Error: Prediction components not loaded.", 5000) return - # Get preset from editor list - current_editor_item = self.editor_preset_list.currentItem() + mode, selected_preset_name = self.preset_editor_widget.get_selected_preset_mode() - # Check if the selected item is the placeholder - is_placeholder = current_editor_item and current_editor_item.data(Qt.ItemDataRole.UserRole) == "__PLACEHOLDER__" + if mode == "placeholder": + log.debug("Update preview called with placeholder preset selected. Showing existing raw inputs (detailed view).") + # Model is always detailed now, no need to set simple mode + # Don't clear data here, _on_preset_selection_changed handles mode switch, + # and add_input_paths handles adding raw data if needed. + # If current_asset_paths is empty, the view will be empty anyway. + if not self.current_asset_paths: + self.unified_model.clear_data() - if is_placeholder: - log.debug("Update preview called with placeholder preset selected. Clearing preview.") - self.preview_model.clear_data() # Clear model if placeholder selected - self.statusBar().showMessage("Select a preset from the list on the left to update preview.", 3000) - # Ensure placeholder is visible and table is hidden - if hasattr(self, 'preview_placeholder_label') and hasattr(self, 'preview_table_view'): - self.preview_placeholder_label.setVisible(True) - self.preview_table_view.setVisible(False) - return # Stop prediction as no valid preset is selected + self.statusBar().showMessage("Select a preset or LLM to generate preview.", 3000) + return - # Existing logic to get selected_preset text and proceed - selected_preset = current_editor_item.text() if current_editor_item else None - if not selected_preset: - log.debug("Update preview called with no preset selected in the editor list.") - self.preview_model.clear_data() # Clear model if no preset selected - self.statusBar().showMessage("Select a preset from the list on the left to update preview.", 3000) - return - if not hasattr(self, 'current_asset_paths') or not self.current_asset_paths: - log.debug("Update preview called with no assets tracked.") - self.preview_model.clear_data() # Clear model if no assets - return - input_paths = list(self.current_asset_paths) - if not input_paths: - log.debug("Update preview called but no input paths derived.") - self.preview_model.clear_data() # Clear model if no paths - return + if not hasattr(self, 'current_asset_paths') or not self.current_asset_paths: + log.debug("Update preview called with no assets tracked.") + self.unified_model.clear_data() + return + input_paths = list(self.current_asset_paths) - log.info(f"[{time.time():.4f}] Requesting background preview update for {len(input_paths)} items, Preset='{selected_preset}'") - self.statusBar().showMessage(f"Updating preview for '{selected_preset}'...", 0) - # Clearing is handled by model's set_data now, no need to clear table view directly - self.setup_threads() # Ensure threads are ready - if self.prediction_thread and self.prediction_handler: - try: self.prediction_thread.started.disconnect() # Disconnect previous lambda if any - except RuntimeError: pass - # Connect the lambda to start the prediction - self.prediction_thread.started.connect( - lambda: self.prediction_handler.run_prediction(input_paths, selected_preset) - ) - log.debug(f"[{time.time():.4f}] Starting prediction thread...") - self.prediction_thread.start() - log.debug(f"[{time.time():.4f}] Prediction thread start requested.") + if mode == "llm": + log.info(f"[{time.time():.4f}] LLM mode selected. Preparing LLM prediction for {len(input_paths)} assets.") + self.statusBar().showMessage(f"Starting LLM interpretation for assets...", 0) + log.debug("Clearing accumulated rules and pending predictions for LLM batch.") + self._accumulated_rules.clear() + self._pending_predictions = set(input_paths) + self._completed_predictions.clear() + log.debug(f"Reset pending predictions for LLM batch: {self._pending_predictions}") + + llm_requests_to_queue = [] + if input_paths: + log.info(f"Preparing LLM prediction requests for {len(input_paths)} existing assets.") + for input_path_str in input_paths: + # Duplication check is handled by the handler's queue method + file_list = self._extract_file_list(input_path_str) + if file_list is not None: + log.debug(f"Extracted {len(file_list)} files for LLM prediction from existing asset: {input_path_str}") + # Store file list (still needed for context if prediction fails before handler starts?) + self._source_file_lists[input_path_str] = file_list + llm_requests_to_queue.append((input_path_str, file_list)) + else: + log.warning(f"Skipping LLM prediction queuing for existing asset {input_path_str} due to extraction error.") + self._pending_predictions.discard(input_path_str) + self.statusBar().showMessage(f"Error extracting files for {Path(input_path_str).name}", 5000) else: - log.error(f"[{time.time():.4f}][T:{thread_id}] Failed to start prediction: Thread or handler not initialized.") - self.statusBar().showMessage("Error: Failed to initialize prediction thread.", 5000) - log.info(f"[{time.time():.4f}][T:{thread_id}] <-- Exiting update_preview (Detailed Mode).") + log.warning("LLM selected, but no input paths currently in view to process.") + self.statusBar().showMessage("LLM selected, but no assets are loaded.", 3000) + + if llm_requests_to_queue: + log.info(f"Delegating {len(llm_requests_to_queue)} LLM requests to the handler from update_preview.") + self.llm_interaction_handler.queue_llm_requests_batch(llm_requests_to_queue) + # The handler manages starting its own processing internally. + # Do not return here; let the function exit normally after handling LLM case. + # The standard prediction path below will be skipped because mode is 'llm'. + + elif mode == "preset" and selected_preset_name: + log.info(f"[{time.time():.4f}] Requesting background preview update for {len(input_paths)} items using Preset='{selected_preset_name}'") + self.statusBar().showMessage(f"Updating preview for '{selected_preset_name}'...", 0) + + log.debug("Clearing accumulated rules for new standard preview batch.") + self._accumulated_rules.clear() + self._pending_predictions = set(input_paths) + log.debug(f"Reset pending standard predictions for batch: {self._pending_predictions}") + + if self.prediction_thread and self.prediction_handler: + self.prediction_thread.start() + log.debug(f"[{time.time():.4f}] Iterating through {len(input_paths)} paths to extract files and emit standard prediction signals.") + for input_path_str in input_paths: + file_list = self._extract_file_list(input_path_str) + if file_list is not None: + log.debug(f"[{time.time():.4f}] Emitting start_prediction_signal for: {input_path_str} with {len(file_list)} files.") + self.start_prediction_signal.emit(input_path_str, file_list, selected_preset_name) + else: + log.warning(f"[{time.time():.4f}] Skipping standard prediction signal for {input_path_str} due to extraction error.") + else: + log.error(f"[{time.time():.4f}][T:{thread_id}] Failed to trigger standard prediction: Thread or handler not initialized.") + self.statusBar().showMessage("Error: Failed to initialize standard prediction thread.", 5000) + + log.info(f"[{time.time():.4f}][T:{thread_id}] <-- Exiting update_preview.") - # --- Threading and Processing Control --- def setup_threads(self): - # Setup Processing Thread - if ProcessingHandler and self.processing_thread is None: - self.processing_thread = QThread(self) - self.processing_handler = ProcessingHandler() - self.processing_handler.moveToThread(self.processing_thread) - self.processing_handler.progress_updated.connect(self.update_progress_bar) - self.processing_handler.file_status_updated.connect(self.update_file_status) - self.processing_handler.processing_finished.connect(self.on_processing_finished) - self.processing_handler.status_message.connect(self.show_status_message) - self.processing_handler.processing_finished.connect(self.processing_thread.quit) - self.processing_handler.processing_finished.connect(self.processing_handler.deleteLater) - self.processing_thread.finished.connect(self.processing_thread.deleteLater) - self.processing_thread.finished.connect(self._reset_processing_thread_references) - log.debug("Processing thread and handler set up.") - elif not ProcessingHandler: - log.error("ProcessingHandler not available. Cannot set up processing thread.") - if hasattr(self, 'start_button'): - self.start_button.setEnabled(False) - self.start_button.setToolTip("Error: Backend processing components failed to load.") - # Setup Prediction Thread - if PredictionHandler and self.prediction_thread is None: + if RuleBasedPredictionHandler and self.prediction_thread is None: self.prediction_thread = QThread(self) - self.prediction_handler = PredictionHandler() + self.prediction_handler = RuleBasedPredictionHandler(input_source_identifier="", original_input_paths=[], preset_name="") self.prediction_handler.moveToThread(self.prediction_thread) - self.prediction_handler.prediction_results_ready.connect(self.on_prediction_results_ready) # Updated slot below - self.prediction_handler.prediction_finished.connect(self.on_prediction_finished) - self.prediction_handler.status_message.connect(self.show_status_message) - self.prediction_handler.prediction_finished.connect(self.prediction_thread.quit) - self.prediction_handler.prediction_finished.connect(self.prediction_handler.deleteLater) - self.prediction_thread.finished.connect(self.prediction_thread.deleteLater) - self.prediction_thread.finished.connect(self._reset_prediction_thread_references) - log.debug("Prediction thread and handler set up.") - elif not PredictionHandler: - log.error("PredictionHandler not available. Cannot set up prediction thread.") - @Slot() - def _reset_processing_thread_references(self): - log.debug("Resetting processing thread and handler references.") - self.processing_thread = None - self.processing_handler = None + self.start_prediction_signal.connect(self.prediction_handler.run_prediction, Qt.ConnectionType.QueuedConnection) + + self.prediction_handler.prediction_ready.connect(self._on_rule_hierarchy_ready) + self.prediction_handler.prediction_error.connect(self._on_prediction_error) + self.prediction_handler.status_update.connect(self.show_status_message) + + # Keep thread alive (no automatic quit/deleteLater for persistent handler) + log.debug("Rule-Based Prediction thread and handler set up to be persistent.") + self.prediction_thread.start() + elif not RuleBasedPredictionHandler: + log.error("RuleBasedPredictionHandler not available. Cannot set up prediction thread.") + + # LLM Thread setup is now handled internally by LLMInteractionHandler + @Slot() def _reset_prediction_thread_references(self): - log.debug("Resetting prediction thread and handler references.") - self.prediction_thread = None - self.prediction_handler = None + # This slot is no longer connected, but keep it for now in case needed later + log.debug("Resetting prediction thread and handler references (Slot disconnected).") + @Slot(int, int) def update_progress_bar(self, current_count, total_count): - if total_count > 0: - percentage = int((current_count / total_count) * 100) - self.progress_bar.setValue(percentage) - self.progress_bar.setFormat(f"%p% ({current_count}/{total_count})") - else: - self.progress_bar.setValue(0) - self.progress_bar.setFormat("0/0") + if hasattr(self, 'main_panel_widget'): + self.main_panel_widget.update_progress_bar(current_count, total_count) - # Slot for prediction results (Updated for new format and coloring) - @Slot(list) - def on_prediction_results_ready(self, results: list): - """Populates the preview table model with detailed prediction results.""" - thread_id = QThread.currentThread() # Get current thread object - log.info(f"[{time.time():.4f}][T:{thread_id}] --> Entered on_prediction_results_ready. Received {len(results)} file details.") - # Update the model with the new data - log.info(f"[{time.time():.4f}][T:{thread_id}] Calling preview_model.set_data()...") - self.preview_model.set_data(results) - log.info(f"[{time.time():.4f}][T:{thread_id}] Returned from preview_model.set_data().") - log.info(f"[{time.time():.4f}][T:{thread_id}] <-- Exiting on_prediction_results_ready.") - - @Slot() - def on_prediction_finished(self): - log.info(f"[{time.time():.4f}] --> Prediction finished signal received.") - # Optionally update status bar or re-enable controls if needed after prediction finishes - # (Controls are primarily managed by processing_finished, but prediction is a separate background task) - self.statusBar().showMessage("Preview updated.", 3000) + # Completion is handled by _on_rule_hierarchy_ready or _on_prediction_error @Slot(str, str, str) def update_file_status(self, input_path_str, status, message): # TODO: Update status bar or potentially find rows in table later @@ -1073,499 +818,119 @@ class MainWindow(QMainWindow): self.statusBar().showMessage(status_text, 5000) log.debug(f"Received file status update: {input_path_str} - {status}") + # TODO: This slot needs to be connected to a signal from the App/main thread + # indicating that all tasks in the QThreadPool are complete. @Slot(int, int, int) def on_processing_finished(self, processed_count, skipped_count, failed_count): + # This log message might be inaccurate until signal source is updated log.info(f"GUI received processing_finished signal: P={processed_count}, S={skipped_count}, F={failed_count}") self.set_controls_enabled(True) - self.cancel_button.setEnabled(False) - self.start_button.setText("Start Processing") + self.main_panel_widget.set_cancel_button_enabled(False) + self.main_panel_widget.set_start_button_text("Start Processing") + # Start button enabled state depends on preset mode, handled by _on_preset_selection_changed or set_controls_enabled + self.main_panel_widget.set_progress_bar_text(f"Finished: {processed_count} processed, {skipped_count} skipped, {failed_count} failed.") - @Slot(str, int) - def show_status_message(self, message, timeout_ms): - if timeout_ms > 0: self.statusBar().showMessage(message, timeout_ms) - else: self.statusBar().showMessage(message) + # Signature changed: Base class signal only emits message string + @Slot(str) + def show_status_message(self, message): + # Show message indefinitely until replaced + self.statusBar().showMessage(message) def set_controls_enabled(self, enabled: bool): - """Enables/disables input controls during processing.""" - # Main panel controls - self.start_button.setEnabled(enabled) + """Enables/disables input controls in relevant panels during processing.""" self.setAcceptDrops(enabled) - self.drag_drop_area.setEnabled(enabled) - # self.preview_table.setEnabled(enabled) # This was the old QTableWidget - self.preview_table_view.setEnabled(enabled) # Enable/disable the QTableView instead - # Editor panel controls (should generally be enabled unless processing) - self.editor_panel.setEnabled(enabled) # Enable/disable the whole panel - # Blender controls - self.blender_integration_checkbox.setEnabled(enabled) - # Only enable path inputs if checkbox is checked AND main controls are enabled - blender_paths_enabled = enabled and self.blender_integration_checkbox.isChecked() - self.nodegroup_blend_path_input.setEnabled(blender_paths_enabled) - self.browse_nodegroup_blend_button.setEnabled(blender_paths_enabled) - self.materials_blend_path_input.setEnabled(blender_paths_enabled) - self.browse_materials_blend_button.setEnabled(blender_paths_enabled) - - - @Slot(bool) - def _toggle_blender_controls(self, checked): - """Enable/disable Blender path inputs based on the checkbox state.""" - self.nodegroup_blend_path_input.setEnabled(checked) - self.browse_nodegroup_blend_button.setEnabled(checked) - self.materials_blend_path_input.setEnabled(checked) - self.browse_materials_blend_button.setEnabled(checked) - - def _browse_for_blend_file(self, line_edit_widget: QLineEdit): - """Opens a dialog to select a .blend file and updates the line edit.""" - current_path = line_edit_widget.text() - start_dir = str(Path(current_path).parent) if current_path and Path(current_path).exists() else str(project_root) - - file_path, _ = QFileDialog.getOpenFileName( - self, - "Select Blender File", - start_dir, - "Blender Files (*.blend);;All Files (*)" - ) - if file_path: - line_edit_widget.setText(file_path) - log.info(f"User selected blend file: {file_path}") - - def _browse_for_nodegroup_blend(self): - self._browse_for_blend_file(self.nodegroup_blend_path_input) - - def _browse_for_materials_blend(self): - self._browse_for_blend_file(self.materials_blend_path_input) - - # --- Preset Editor Methods (Adapted from PresetEditorDialog) --- - - def _editor_add_list_item(self, list_widget: QListWidget): - """Adds an editable item to the specified list widget in the editor.""" - text, ok = QInputDialog.getText(self, f"Add Item", "Enter value:") - if ok and text: - item = QListWidgetItem(text) - # item.setFlags(item.flags() | Qt.ItemFlag.ItemIsEditable) # Already editable by default - list_widget.addItem(item) - self._mark_editor_unsaved() - - def _editor_remove_list_item(self, list_widget: QListWidget): - """Removes the selected item from the specified list widget in the editor.""" - selected_items = list_widget.selectedItems() - if not selected_items: return - for item in selected_items: list_widget.takeItem(list_widget.row(item)) - self._mark_editor_unsaved() - - def _editor_add_table_row(self, table_widget: QTableWidget): - """Adds an empty row to the specified table widget in the editor.""" - row_count = table_widget.rowCount() - table_widget.insertRow(row_count) - for col in range(table_widget.columnCount()): table_widget.setItem(row_count, col, QTableWidgetItem("")) - self._mark_editor_unsaved() - - def _editor_remove_table_row(self, table_widget: QTableWidget): - """Removes the selected row(s) from the specified table widget in the editor.""" - selected_rows = sorted(list(set(index.row() for index in table_widget.selectedIndexes())), reverse=True) - if not selected_rows: - if table_widget.rowCount() > 0: selected_rows = [table_widget.rowCount() - 1] - else: return - for row in selected_rows: table_widget.removeRow(row) - self._mark_editor_unsaved() - - def _mark_editor_unsaved(self): - """Marks changes in the editor panel as unsaved.""" - if self._is_loading_editor: return - self.editor_unsaved_changes = True - self.editor_save_button.setEnabled(True) - preset_name = Path(self.current_editing_preset_path).name if self.current_editing_preset_path else 'New Preset' - self.setWindowTitle(f"Asset Processor Tool - {preset_name}*") - - def _connect_editor_change_signals(self): - """Connect signals from all editor widgets to mark_editor_unsaved.""" - self.editor_preset_name.textChanged.connect(self._mark_editor_unsaved) - self.editor_supplier_name.textChanged.connect(self._mark_editor_unsaved) - self.editor_notes.textChanged.connect(self._mark_editor_unsaved) - self.editor_separator.textChanged.connect(self._mark_editor_unsaved) - self.editor_spin_base_name_idx.valueChanged.connect(self._mark_editor_unsaved) - self.editor_spin_map_type_idx.valueChanged.connect(self._mark_editor_unsaved) - # List/Table widgets are connected via helper functions - - def _check_editor_unsaved_changes(self) -> bool: - """Checks for unsaved changes in the editor and prompts the user. Returns True if should cancel action.""" - if not self.editor_unsaved_changes: return False - reply = QMessageBox.question(self, "Unsaved Preset Changes", - "You have unsaved changes in the preset editor. Discard them?", - QMessageBox.StandardButton.Save | QMessageBox.StandardButton.Discard | QMessageBox.StandardButton.Cancel, - QMessageBox.StandardButton.Cancel) - if reply == QMessageBox.StandardButton.Save: return not self._save_current_preset() # Return True (cancel) if save fails - elif reply == QMessageBox.StandardButton.Discard: return False # Discarded, proceed - else: return True # Cancel action - - def _set_editor_enabled(self, enabled: bool): - """Enables or disables all editor widgets.""" - self.editor_tab_widget.setEnabled(enabled) - # Also enable/disable save buttons based on editor state, not just processing state - self.editor_save_button.setEnabled(enabled and self.editor_unsaved_changes) - self.editor_save_as_button.setEnabled(enabled) # Save As is always possible if editor is enabled - - def _clear_editor(self): - """Clears the editor fields and resets state.""" - self._is_loading_editor = True - self.editor_preset_name.clear() - self.editor_supplier_name.clear() - self.editor_notes.clear() - self.editor_separator.clear() - self.editor_spin_base_name_idx.setValue(0) - self.editor_spin_map_type_idx.setValue(1) - self.editor_list_gloss_keywords.clear() - self.editor_table_bit_depth_variants.setRowCount(0) - self.editor_list_extra_patterns.clear() - self.editor_table_map_type_mapping.setRowCount(0) - self.editor_list_model_patterns.clear() - self.editor_list_decal_keywords.clear() - self.editor_table_archetype_rules.setRowCount(0) - self.current_editing_preset_path = None - self.editor_unsaved_changes = False - self.editor_save_button.setEnabled(False) - self.setWindowTitle("Asset Processor Tool") # Reset window title - self._set_editor_enabled(False) - - # Ensure placeholder is visible and table is hidden when editor is cleared - if hasattr(self, 'preview_placeholder_label') and hasattr(self, 'preview_table_view'): - log.debug("Clearing editor. Showing placeholder, hiding table view.") - self.preview_placeholder_label.setVisible(True) - self.preview_table_view.setVisible(False) - - self._is_loading_editor = False - - def _populate_editor_from_data(self, preset_data: dict): - """Helper method to populate editor UI widgets from a preset data dictionary.""" - self._is_loading_editor = True - try: - self.editor_preset_name.setText(preset_data.get("preset_name", "")) - self.editor_supplier_name.setText(preset_data.get("supplier_name", "")) - self.editor_notes.setText(preset_data.get("notes", "")) - naming_data = preset_data.get("source_naming", {}) - self.editor_separator.setText(naming_data.get("separator", "_")) - indices = naming_data.get("part_indices", {}) - self.editor_spin_base_name_idx.setValue(indices.get("base_name", 0)) - self.editor_spin_map_type_idx.setValue(indices.get("map_type", 1)) - self.editor_list_gloss_keywords.clear() - self.editor_list_gloss_keywords.addItems(naming_data.get("glossiness_keywords", [])) - self.editor_table_bit_depth_variants.setRowCount(0) - bit_depth_vars = naming_data.get("bit_depth_variants", {}) - for i, (map_type, pattern) in enumerate(bit_depth_vars.items()): - self.editor_table_bit_depth_variants.insertRow(i) - self.editor_table_bit_depth_variants.setItem(i, 0, QTableWidgetItem(map_type)) - self.editor_table_bit_depth_variants.setItem(i, 1, QTableWidgetItem(pattern)) - self.editor_list_extra_patterns.clear() - self.editor_list_extra_patterns.addItems(preset_data.get("move_to_extra_patterns", [])) - self.editor_table_map_type_mapping.setRowCount(0) - map_mappings = preset_data.get("map_type_mapping", []) - # --- UPDATED for new dictionary format --- - for i, mapping_dict in enumerate(map_mappings): - if isinstance(mapping_dict, dict) and "target_type" in mapping_dict and "keywords" in mapping_dict: - std_type = mapping_dict["target_type"] - keywords = mapping_dict["keywords"] - self.editor_table_map_type_mapping.insertRow(i) - self.editor_table_map_type_mapping.setItem(i, 0, QTableWidgetItem(std_type)) - # Ensure keywords are strings before joining - keywords_str = [str(k) for k in keywords if isinstance(k, str)] - self.editor_table_map_type_mapping.setItem(i, 1, QTableWidgetItem(", ".join(keywords_str))) - else: - log.warning(f"Skipping invalid map_type_mapping item during editor population: {mapping_dict}") - # --- END UPDATE --- - category_rules = preset_data.get("asset_category_rules", {}) - self.editor_list_model_patterns.clear() - self.editor_list_model_patterns.addItems(category_rules.get("model_patterns", [])) - self.editor_list_decal_keywords.clear() - self.editor_list_decal_keywords.addItems(category_rules.get("decal_keywords", [])) - self.editor_table_archetype_rules.setRowCount(0) - arch_rules = preset_data.get("archetype_rules", []) - for i, rule in enumerate(arch_rules): - if isinstance(rule, (list, tuple)) and len(rule) == 2: - arch_name, conditions = rule - match_any = ", ".join(conditions.get("match_any", [])) - match_all = ", ".join(conditions.get("match_all", [])) - self.editor_table_archetype_rules.insertRow(i) - self.editor_table_archetype_rules.setItem(i, 0, QTableWidgetItem(arch_name)) - self.editor_table_archetype_rules.setItem(i, 1, QTableWidgetItem(match_any)) - self.editor_table_archetype_rules.setItem(i, 2, QTableWidgetItem(match_all)) - finally: - self._is_loading_editor = False - - def _load_preset_for_editing(self, file_path: Path): - """Loads the content of the selected preset file into the editor widgets.""" - if not file_path or not file_path.is_file(): - self._clear_editor() - return - log.info(f"Loading preset into editor: {file_path.name}") - log.info(f"Loading preset into editor: {file_path.name}") - try: - with open(file_path, 'r', encoding='utf-8') as f: preset_data = json.load(f) - self._populate_editor_from_data(preset_data) - self._set_editor_enabled(True) - self.current_editing_preset_path = file_path - self.editor_unsaved_changes = False - self.editor_save_button.setEnabled(False) - self.setWindowTitle(f"Asset Processor Tool - {file_path.name}") - log.info(f"Preset '{file_path.name}' loaded into editor.") - log.debug("Preset loaded. Checking visibility states.") - log.debug(f"preview_placeholder_label visible: {self.preview_placeholder_label.isVisible()}") - log.debug(f"preview_table_view visible: {self.preview_table_view.isVisible()}") - except json.JSONDecodeError as json_err: - log.error(f"Invalid JSON in {file_path.name}: {json_err}") - QMessageBox.warning(self, "Load Error", f"Failed to load preset '{file_path.name}'.\nInvalid JSON structure:\n{json_err}") - self._clear_editor() - except Exception as e: - log.exception(f"Error loading preset file {file_path}: {e}") - QMessageBox.critical(self, "Error", f"Could not load preset file:\n{file_path}\n\nError: {e}") - self._clear_editor() - - def _load_selected_preset_for_editing(self, current_item: QListWidgetItem, previous_item: QListWidgetItem): - """Loads the preset currently selected in the editor list.""" - log.debug(f"currentItemChanged signal triggered. current_item: {current_item.text() if current_item else 'None'}, previous_item: {previous_item.text() if previous_item else 'None'}") - - # Check if the selected item is the placeholder - is_placeholder = current_item and current_item.data(Qt.ItemDataRole.UserRole) == "__PLACEHOLDER__" - - if self._check_editor_unsaved_changes(): - # If user cancels, revert selection - if previous_item: - log.debug("Unsaved changes check cancelled. Reverting selection.") - self.editor_preset_list.blockSignals(True) - self.editor_preset_list.setCurrentItem(previous_item) - self.editor_preset_list.blockSignals(False) - return - - if is_placeholder: - log.debug("Placeholder item selected. Clearing editor and preview.") - self._clear_editor() # This also hides the table and shows the placeholder label - self.preview_model.clear_data() # Ensure the model is empty - # Visibility is handled by _clear_editor, but explicitly set here for clarity - self.preview_placeholder_label.setVisible(True) - self.preview_table_view.setVisible(False) - self.start_button.setEnabled(False) # Disable start button - return # Stop processing as no real preset is selected - - # Existing logic for handling real preset items starts here - if current_item: - log.debug(f"Loading preset for editing: {current_item.text()}") - preset_path = current_item.data(Qt.ItemDataRole.UserRole) - self._load_preset_for_editing(preset_path) - self.start_button.setEnabled(True) # Enable start button - # --- Trigger preview update after loading editor --- - self.update_preview() - # --- End Trigger --- - - # Hide placeholder and show table view - log.debug("Real preset selected. Hiding placeholder, showing table view.") - self.preview_placeholder_label.setVisible(False) - self.preview_table_view.setVisible(True) - else: - # This case should ideally not be reached if the placeholder is always present - log.debug("No preset selected (unexpected state if placeholder is present). Clearing editor.") - self._clear_editor() # Clear editor if selection is cleared - # Ensure placeholder is visible if no preset is selected - log.debug("No preset selected. Showing placeholder, hiding table view.") - self.preview_placeholder_label.setVisible(True) - self.preview_table_view.setVisible(False) - - def _gather_editor_data(self) -> dict: - """Gathers data from all editor UI widgets and returns a dictionary.""" - preset_data = {} - preset_data["preset_name"] = self.editor_preset_name.text().strip() - preset_data["supplier_name"] = self.editor_supplier_name.text().strip() - preset_data["notes"] = self.editor_notes.toPlainText().strip() - naming_data = {} - naming_data["separator"] = self.editor_separator.text() - naming_data["part_indices"] = { "base_name": self.editor_spin_base_name_idx.value(), "map_type": self.editor_spin_map_type_idx.value() } - naming_data["glossiness_keywords"] = [self.editor_list_gloss_keywords.item(i).text() for i in range(self.editor_list_gloss_keywords.count())] - naming_data["bit_depth_variants"] = {self.editor_table_bit_depth_variants.item(r, 0).text(): self.editor_table_bit_depth_variants.item(r, 1).text() - for r in range(self.editor_table_bit_depth_variants.rowCount()) if self.editor_table_bit_depth_variants.item(r, 0) and self.editor_table_bit_depth_variants.item(r, 1)} - preset_data["source_naming"] = naming_data - preset_data["move_to_extra_patterns"] = [self.editor_list_extra_patterns.item(i).text() for i in range(self.editor_list_extra_patterns.count())] - # --- UPDATED for new dictionary format --- - map_mappings = [] - for r in range(self.editor_table_map_type_mapping.rowCount()): - type_item = self.editor_table_map_type_mapping.item(r, 0) - keywords_item = self.editor_table_map_type_mapping.item(r, 1) - # Ensure both items exist and have text before processing - if type_item and type_item.text() and keywords_item and keywords_item.text(): - target_type = type_item.text().strip() - keywords = [k.strip() for k in keywords_item.text().split(',') if k.strip()] - if target_type and keywords: # Only add if both parts are valid - map_mappings.append({"target_type": target_type, "keywords": keywords}) - else: - log.warning(f"Skipping row {r} in map type mapping table due to empty target type or keywords.") + self.preset_editor_widget.setEnabled(enabled) + if hasattr(self, 'main_panel_widget'): + self.main_panel_widget.set_controls_enabled(enabled) + if enabled: + model_has_items = self.unified_model.rowCount() > 0 + self.main_panel_widget.set_start_button_enabled(model_has_items) else: - log.warning(f"Skipping row {r} in map type mapping table due to missing items.") - preset_data["map_type_mapping"] = map_mappings - # --- END UPDATE --- - category_rules = {} - category_rules["model_patterns"] = [self.editor_list_model_patterns.item(i).text() for i in range(self.editor_list_model_patterns.count())] - category_rules["decal_keywords"] = [self.editor_list_decal_keywords.item(i).text() for i in range(self.editor_list_decal_keywords.count())] - preset_data["asset_category_rules"] = category_rules - arch_rules = [] - for r in range(self.editor_table_archetype_rules.rowCount()): - name_item = self.editor_table_archetype_rules.item(r, 0) - any_item = self.editor_table_archetype_rules.item(r, 1) - all_item = self.editor_table_archetype_rules.item(r, 2) - if name_item and any_item and all_item: - match_any = [k.strip() for k in any_item.text().split(',') if k.strip()] - match_all = [k.strip() for k in all_item.text().split(',') if k.strip()] - arch_rules.append([name_item.text().strip(), {"match_any": match_any, "match_all": match_all}]) - preset_data["archetype_rules"] = arch_rules - return preset_data + self.main_panel_widget.set_start_button_enabled(False) - def _save_current_preset(self) -> bool: - """Saves the current editor content to the currently loaded file path.""" - if not self.current_editing_preset_path: return self._save_preset_as() - log.info(f"Saving preset: {self.current_editing_preset_path.name}") - try: - preset_data = self._gather_editor_data() - if not preset_data.get("preset_name"): QMessageBox.warning(self, "Save Error", "Preset Name cannot be empty."); return False - if not preset_data.get("supplier_name"): QMessageBox.warning(self, "Save Error", "Supplier Name cannot be empty."); return False - content_to_save = json.dumps(preset_data, indent=4, ensure_ascii=False) - with open(self.current_editing_preset_path, 'w', encoding='utf-8') as f: f.write(content_to_save) - self.editor_unsaved_changes = False - self.editor_save_button.setEnabled(False) - self.setWindowTitle(f"Asset Processor Tool - {self.current_editing_preset_path.name}") - self.presets_changed_signal.emit() # Signal that presets changed - log.info("Preset saved successfully.") - # Refresh lists after save - self.populate_presets() - return True - except Exception as e: - log.exception(f"Error saving preset file {self.current_editing_preset_path}: {e}") - QMessageBox.critical(self, "Save Error", f"Could not save preset file:\n{self.current_editing_preset_path}\n\nError: {e}") - return False - def _save_preset_as(self) -> bool: - """Saves the current editor content to a new file chosen by the user.""" - log.debug("Save As action triggered.") - try: - preset_data = self._gather_editor_data() - new_preset_name = preset_data.get("preset_name") - if not new_preset_name: QMessageBox.warning(self, "Save As Error", "Preset Name cannot be empty."); return False - if not preset_data.get("supplier_name"): QMessageBox.warning(self, "Save As Error", "Supplier Name cannot be empty."); return False - content_to_save = json.dumps(preset_data, indent=4, ensure_ascii=False) - suggested_name = f"{new_preset_name}.json" - default_path = PRESETS_DIR / suggested_name - file_path_str, _ = QFileDialog.getSaveFileName(self, "Save Preset As", str(default_path), "JSON Files (*.json);;All Files (*)") - if not file_path_str: log.debug("Save As cancelled by user."); return False - save_path = Path(file_path_str) - if save_path.suffix.lower() != ".json": save_path = save_path.with_suffix(".json") - if save_path.exists() and save_path != self.current_editing_preset_path: - reply = QMessageBox.warning(self, "Confirm Overwrite", f"Preset '{save_path.name}' already exists. Overwrite?", QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, QMessageBox.StandardButton.No) - if reply == QMessageBox.StandardButton.No: log.debug("Save As overwrite cancelled."); return False - log.info(f"Saving preset as: {save_path.name}") - with open(save_path, 'w', encoding='utf-8') as f: f.write(content_to_save) - self.current_editing_preset_path = save_path # Update current path - self.editor_unsaved_changes = False - self.editor_save_button.setEnabled(False) - self.setWindowTitle(f"Asset Processor Tool - {save_path.name}") - self.presets_changed_signal.emit() # Signal change - log.info("Preset saved successfully (Save As).") - # Refresh lists and select the new item - self.populate_presets() - return True - except Exception as e: - log.exception(f"Error saving preset file (Save As): {e}") - QMessageBox.critical(self, "Save Error", f"Could not save preset file.\n\nError: {e}") - return False - - def _new_preset(self): - """Clears the editor and loads data from _template.json.""" - log.debug("New Preset action triggered.") - if self._check_editor_unsaved_changes(): return - self._clear_editor() - if TEMPLATE_PATH.is_file(): - log.info("Loading new preset from _template.json") - try: - with open(TEMPLATE_PATH, 'r', encoding='utf-8') as f: template_data = json.load(f) - self._populate_editor_from_data(template_data) - # Override specific fields for a new preset - self.editor_preset_name.setText("NewPreset") - self.setWindowTitle("Asset Processor Tool - New Preset*") - except Exception as e: - log.exception(f"Error loading template preset file {TEMPLATE_PATH}: {e}") - QMessageBox.critical(self, "Error", f"Could not load template preset file:\n{TEMPLATE_PATH}\n\nError: {e}") - self._clear_editor() - self.setWindowTitle("Asset Processor Tool - New Preset*") - else: - log.warning("Presets/_template.json not found. Creating empty preset.") - self.setWindowTitle("Asset Processor Tool - New Preset*") - self.editor_preset_name.setText("NewPreset") - self.editor_supplier_name.setText("MySupplier") - self._set_editor_enabled(True) - self.editor_unsaved_changes = True - self.editor_save_button.setEnabled(True) - - def _delete_selected_preset(self): - """Deletes the currently selected preset file from the editor list after confirmation.""" - current_item = self.editor_preset_list.currentItem() - if not current_item: QMessageBox.information(self, "Delete Preset", "Please select a preset from the list to delete."); return - preset_path = current_item.data(Qt.ItemDataRole.UserRole) - preset_name = preset_path.stem - reply = QMessageBox.warning(self, "Confirm Delete", f"Are you sure you want to permanently delete the preset '{preset_name}'?", QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, QMessageBox.StandardButton.No) - if reply == QMessageBox.StandardButton.Yes: - log.info(f"Deleting preset: {preset_path.name}") - try: - preset_path.unlink() - log.info("Preset deleted successfully.") - if self.current_editing_preset_path == preset_path: self._clear_editor() - self.presets_changed_signal.emit() # Signal change - # Refresh lists - self.populate_presets() - except Exception as e: - log.exception(f"Error deleting preset file {preset_path}: {e}") - QMessageBox.critical(self, "Delete Error", f"Could not delete preset file:\n{preset_path}\n\nError: {e}") - - # --- Menu Bar Setup --- def setup_menu_bar(self): - """Creates the main menu bar and View menu.""" + """Creates the main menu bar and adds menus/actions.""" self.menu_bar = self.menuBar() + + # --- File Menu (Optional, add if needed later) --- + # file_menu = self.menu_bar.addMenu("&File") + # Add actions like New, Open, Save, Exit + + edit_menu = self.menu_bar.addMenu("&Edit") + + self.preferences_action = QAction("&Preferences...", self) + self.preferences_action.triggered.connect(self._open_config_editor) + edit_menu.addAction(self.preferences_action) + edit_menu.addSeparator() + + self.definitions_editor_action = QAction("Edit Definitions...", self) + self.definitions_editor_action.triggered.connect(self._open_definitions_editor) + edit_menu.addAction(self.definitions_editor_action) + view_menu = self.menu_bar.addMenu("&View") - # Log Console Action self.toggle_log_action = QAction("Show Log Console", self, checkable=True) - self.toggle_log_action.setChecked(False) # Start hidden + self.toggle_log_action.setChecked(False) self.toggle_log_action.toggled.connect(self._toggle_log_console_visibility) view_menu.addAction(self.toggle_log_action) - # Detailed Preview Action - self.toggle_preview_action = QAction("Disable Detailed Preview", self, checkable=True) - self.toggle_preview_action.setChecked(False) # Start enabled (detailed view) - # Connect to update_preview, which now checks this action's state - self.toggle_preview_action.toggled.connect(self.update_preview) - view_menu.addAction(self.toggle_preview_action) - # Verbose Logging Action self.toggle_verbose_action = QAction("Verbose Logging (DEBUG)", self, checkable=True) - self.toggle_verbose_action.setChecked(False) # Start disabled (INFO level) + self.toggle_verbose_action.setChecked(False) self.toggle_verbose_action.toggled.connect(self._toggle_verbose_logging) view_menu.addAction(self.toggle_verbose_action) - # --- Logging Handler Setup --- def setup_logging_handler(self): """Creates and configures the custom QtLogHandler.""" self.log_handler = QtLogHandler(self) - # Set the formatter to match the basicConfig format - log_format = '%(levelname)s: %(message)s' # Simpler format for UI console + log_format = '%(levelname)s: %(message)s' formatter = logging.Formatter(log_format) self.log_handler.setFormatter(formatter) - # Set level (e.g., INFO to capture standard messages) self.log_handler.setLevel(logging.INFO) # Add handler to the root logger to capture logs from all modules logging.getLogger().addHandler(self.log_handler) - # Connect the signal to the slot - self.log_handler.log_record_received.connect(self._append_log_message) - log.info("UI Log Handler Initialized.") # Log that the handler is ready + self.log_handler.log_record_received.connect(self.log_console._append_log_message) + log.info("UI Log Handler Initialized.") + + @Slot() + def _open_config_editor(self): + """Opens the configuration editor dialog.""" + log.debug("Opening configuration editor dialog.") + try: + # Import locally to avoid circular dependency if needed + from .config_editor_dialog import ConfigEditorDialog + dialog = ConfigEditorDialog(self) + dialog.exec_() + log.debug("Configuration editor dialog closed.") + except ImportError: + log.error("Failed to import ConfigEditorDialog. Ensure gui/config_editor_dialog.py exists and is accessible.") + QMessageBox.critical(self, "Error", "Could not open configuration editor.\nRequired file not found or has errors.") + except Exception as e: + log.exception(f"Error opening configuration editor dialog: {e}") + QMessageBox.critical(self, "Error", f"An error occurred while opening the configuration editor:\n{e}") + + @Slot() # PySide6.QtCore.Slot + def _open_definitions_editor(self): + log.debug("Opening Definitions Editor dialog.") + try: + # DefinitionsEditorDialog is imported at the top of the file + dialog = DefinitionsEditorDialog(self) + dialog.exec_() # Use exec_() for modal dialog + log.debug("Definitions Editor dialog closed.") + except Exception as e: + log.exception(f"Error opening Definitions Editor dialog: {e}") + QMessageBox.critical(self, "Error", f"An error occurred while opening the Definitions Editor:\n{e}") - # --- Slots for Menu Actions and Logging --- @Slot(bool) def _toggle_log_console_visibility(self, checked): - """Shows or hides the log console widget based on menu action.""" - if hasattr(self, 'log_console_widget'): - self.log_console_widget.setVisible(checked) + """Shows or hides the log console widget.""" + if hasattr(self, 'log_console'): + self.log_console.setVisible(checked) log.debug(f"Log console visibility set to: {checked}") + else: + log.warning("Attempted to toggle log console visibility, but widget not found.") + @Slot(bool) def _toggle_verbose_logging(self, checked): @@ -1575,58 +940,406 @@ class MainWindow(QMainWindow): return new_level = logging.DEBUG if checked else logging.INFO - root_logger = logging.getLogger() # Get the root logger + root_logger = logging.getLogger() root_logger.setLevel(new_level) self.log_handler.setLevel(new_level) log.info(f"Root and GUI logging level set to: {logging.getLevelName(new_level)}") - # Update status bar or log console to indicate change self.statusBar().showMessage(f"Logging level set to {logging.getLevelName(new_level)}", 3000) - @Slot(str) - def _append_log_message(self, message): - """Appends a log message to the QTextEdit console.""" - if hasattr(self, 'log_console_output'): - # Optional: Add basic coloring (can be expanded) - # if message.startswith("ERROR"): - # message = f"{message}" - # elif message.startswith("WARNING"): - # message = f"{message}" - self.log_console_output.append(message) # Use append for plain text - # Optional: Limit history size - # MAX_LINES = 500 - # if self.log_console_output.document().blockCount() > MAX_LINES: - # cursor = self.log_console_output.textCursor() - # cursor.movePosition(QTextCursor.MoveOperation.Start) - # cursor.select(QTextCursor.SelectionType.BlockUnderCursor) - # cursor.removeSelectedText() - # cursor.deletePreviousChar() # Remove the newline potentially left behind - # Ensure the view scrolls to the bottom - self.log_console_output.verticalScrollBar().setValue(self.log_console_output.verticalScrollBar().maximum()) - - - # --- Overridden Close Event --- def closeEvent(self, event): - """Overrides close event to check for unsaved changes in the editor.""" - if self._check_editor_unsaved_changes(): - event.ignore() # Ignore close event if user cancels + """Overrides close event to check for unsaved changes in the editor widget.""" + if self.preset_editor_widget.check_unsaved_changes(): + event.ignore() else: - event.accept() # Accept close event + event.accept() + + + # Slot signature updated to match BasePredictionHandler.prediction_ready: Signal(str, list) + @Slot(str, list) + def _on_rule_hierarchy_ready(self, input_path: str, source_rules_list: list): + """ + Receives rule-based prediction results (a list containing one SourceRule) + for a single input path, updates the model preserving overrides, + and handles completion tracking. + """ + log.debug(f"--> Entered _on_rule_hierarchy_ready for '{input_path}'") + + if not input_path: + log.error("Received rule hierarchy ready signal with empty input_path. Cannot process.") + return + + if input_path not in self._pending_predictions: + log.warning(f"Received rule hierarchy for '{input_path}', but it was not in the pending set. Ignoring stale result? Pending: {self._pending_predictions}") + return + + if source_rules_list: + try: + log.info(f"Updating model with rule-based results for source: {input_path}") + log.debug(f"DEBUG: Type of self.unified_model: {type(self.unified_model)}") + log.debug(f"DEBUG: hasattr(self.unified_model, 'update_rules_for_sources'): {hasattr(self.unified_model, 'update_rules_for_sources')}") + self.unified_model.update_rules_for_sources(source_rules_list) + log.info("Model update call successful.") + if hasattr(self, 'main_panel_widget'): + self.main_panel_widget.unified_view.expandToDepth(1) + except Exception as e: + error_msg = f"Error updating model with rule-based results for {input_path}: {e}" + log.exception(error_msg) + self.statusBar().showMessage(error_msg, 8000) + # Fall through to completion handling even if model update fails + else: + log.warning(f"Received empty source_rules_list for '{input_path}'. Prediction likely failed. Model not updated.") + + self._handle_prediction_completion(input_path) + + # Replaced by _on_llm_prediction_ready_from_handler + + # Errors now connect to _on_prediction_error + + @Slot(str, list) + def _on_llm_prediction_ready_from_handler(self, input_path: str, source_rules: list): + """ + Handles the successful LLM prediction result received from LLMInteractionHandler. + Updates the model and handles completion tracking. + """ + log.info(f"Received LLM prediction result from handler for {input_path}. {len(source_rules)} source rule(s) found.") + + if source_rules: + try: + log.info(f"Updating model with LLM results for source: {input_path}") + self.unified_model.update_rules_for_sources(source_rules) + log.info("Model update call successful.") + if hasattr(self, 'main_panel_widget'): + self.main_panel_widget.unified_view.expandToDepth(1) + except Exception as e: + error_msg = f"Error updating model with LLM results for {input_path}: {e}" + log.exception(error_msg) + self.statusBar().showMessage(error_msg, 8000) + # Fall through to completion handling even if model update fails + else: + log.info(f"No source rules returned by LLM handler for {input_path}. Model not updated.") + + self._handle_prediction_completion(input_path) + + @Slot(str, str) + def _on_prediction_error(self, input_path: str, error_message: str): + """Handles errors reported by any prediction handler (RuleBased or LLM).""" + log.error(f"Prediction Error for '{input_path}': {error_message}") + self.statusBar().showMessage(f"Error analyzing {Path(input_path).name}: {error_message}", 8000) + + self._handle_prediction_completion(input_path) + + def _handle_prediction_completion(self, input_path: str): + """ + Centralized method to handle completion tracking for both successful + predictions and errors for a given input path. + """ + log.debug(f"--> Entered _handle_prediction_completion for '{input_path}'") + + if input_path in self._pending_predictions: + self._pending_predictions.discard(input_path) + self._completed_predictions.add(input_path) + log.debug(f"Marked '{input_path}' as completed. Pending: {len(self._pending_predictions)}, Completed: {len(self._completed_predictions)}") + + if not self._pending_predictions: + log.info("All pending predictions processed. Model should be up-to-date.") + self.statusBar().showMessage(f"Preview generation complete.", 5000) + if hasattr(self, 'main_panel_widget'): + self.main_panel_widget.set_start_button_enabled(self.unified_model.rowCount() > 0) + # Optional: Resize columns after all updates are done (Access view via panel) + if hasattr(self, 'main_panel_widget'): + view = self.main_panel_widget.unified_view + for col in range(self.unified_model.columnCount()): + view.resizeColumnToContents(col) + view.expandToDepth(1) + else: + completed_count = len(self._completed_predictions) + pending_count = len(self._pending_predictions) + # Estimate total based on initial request size (might be slightly off if items were added/removed) + total_requested = completed_count + pending_count + status_msg = f"Preview updated for {Path(input_path).name}. Waiting for {pending_count} more ({completed_count}/{total_requested} processed)..." + self.statusBar().showMessage(status_msg, 5000) + log.debug(status_msg) + else: + log.warning(f"Received completion signal for '{input_path}', but it was not in the pending set. Ignoring?") + + log.debug(f"<-- Exiting _handle_prediction_completion for '{input_path}'") + + + @Slot(str, str) + def _on_preset_selection_changed(self, mode: str, preset_name: str | None): + """ + Handles changes in the preset editor selection (preset, LLM, placeholder). + Switches between PresetEditorWidget and LLMEditorWidget. + """ + log.info(f"Preset selection changed: mode='{mode}', preset_name='{preset_name}'") + + if mode == "llm": + log.debug("Switching editor stack to LLM Editor Widget.") + # Force reset the LLM handler state in case it got stuck + if hasattr(self, 'llm_interaction_handler'): + self.llm_interaction_handler.force_reset_state() + self.editor_stack.setCurrentWidget(self.llm_editor_widget) + # Load settings *after* switching the stack + try: + self.llm_editor_widget.load_settings() + except Exception as e: + log.exception(f"Error loading LLM settings in _on_preset_selection_changed: {e}") + QMessageBox.critical(self, "LLM Settings Error", f"Failed to load LLM settings:\n{e}") + elif mode == "preset": + log.debug("Switching editor stack to Preset JSON Editor Widget.") + self.editor_stack.setCurrentWidget(self.preset_editor_widget.json_editor_container) + else: + log.debug("Switching editor stack to Preset JSON Editor Widget (placeholder selected).") + self.editor_stack.setCurrentWidget(self.preset_editor_widget.json_editor_container) + # The PresetEditorWidget's internal logic handles disabling/clearing the editor fields. + + if mode == "preset" and preset_name: + # This might be redundant if the editor handles its own title updates on save/load + # but good for consistency. + unsaved = self.preset_editor_widget.editor_unsaved_changes + self.setWindowTitle(f"Asset Processor Tool - {preset_name}{'*' if unsaved else ''}") + elif mode == "llm": + self.setWindowTitle("Asset Processor Tool - LLM Interpretation") + else: + self.setWindowTitle("Asset Processor Tool") + + if hasattr(self, 'main_panel_widget'): + model_has_items = self.unified_model.rowCount() > 0 + self.main_panel_widget.set_start_button_enabled(model_has_items) + self.main_panel_widget.set_llm_processing_status(self.llm_interaction_handler.is_processing()) + + # Display mode is always detailed, no need to set it here + # update_preview will now respect the mode set above + self.update_preview() + + @Slot() + def _on_llm_settings_saved(self): + """Slot called when LLM settings are saved successfully.""" + log.info("LLM settings saved signal received by MainWindow.") + self.statusBar().showMessage("LLM settings saved successfully.", 3000) + # Optionally, trigger a reload of configuration if needed elsewhere, + # or update the LLMInteractionHandler if it caches settings. + # For now, just show a status message. + # If the LLM handler uses the config directly, no action needed here. + # If it caches, we might need: self.llm_interaction_handler.reload_settings() + + @Slot(bool) + def _on_llm_processing_state_changed(self, is_processing: bool): + """Updates the UI based on the LLM handler's processing state.""" + log.debug(f"Received LLM processing state change from handler: {is_processing}") + self.preset_editor_widget.setEnabled(not is_processing) + if hasattr(self, 'main_panel_widget'): + self.main_panel_widget.set_llm_processing_status(is_processing) + + # Use self.llm_interaction_handler.is_processing() + + def get_llm_source_preset_name(self) -> str | None: + """ + Returns the name (stem) of the last valid preset that was loaded + before switching to LLM mode or triggering re-interpretation. + Used by delegates to populate dropdowns based on the original context. + Delegates this call to the PresetEditorWidget. + """ + if hasattr(self, 'preset_editor_widget'): + last_name = self.preset_editor_widget.get_last_valid_preset_name() + log.debug(f"get_llm_source_preset_name called, returning from widget: {last_name}") + return last_name + else: + log.warning("get_llm_source_preset_name called before preset_editor_widget was initialized.") + return None + + def keyPressEvent(self, event): + """Handles key press events for implementing keybinds.""" + log.debug(f"KeyPressEvent: key={event.key()}, modifiers={event.modifiers()}, text='{event.text()}'") + + if not self.main_panel_widget or not self.unified_model: + log.warning("Key press ignored: Main panel or unified model not available.") + super().keyPressEvent(event) + return + + selected_view_indexes = self.main_panel_widget.unified_view.selectionModel().selectedIndexes() + if not selected_view_indexes: + log.debug("Key press ignored: No items selected.") + super().keyPressEvent(event) + return + + # Assuming unified_view uses unified_model directly or proxy maps correctly + model_indexes_to_process = [] + unique_rows = set() + for view_idx in selected_view_indexes: + model_idx = view_idx + if model_idx.row() not in unique_rows: + # Ensure we are getting the index for column 0 if multiple columns are selected for the same row + model_indexes_to_process.append(self.unified_model.index(model_idx.row(), 0, model_idx.parent())) + unique_rows.add(model_idx.row()) + + if not model_indexes_to_process: + super().keyPressEvent(event) + return + + pressed_key = event.key() + modifiers = event.modifiers() + keybind_processed = False + + if pressed_key == Qt.Key_F2 and not modifiers: + log.debug("F2 pressed for asset name change.") + first_selected_item_index = model_indexes_to_process[0] + first_item_object = self.unified_model.getItem(first_selected_item_index) + current_name_suggestion = "" + + if isinstance(first_item_object, AssetRule): + # For AssetRule, its name is in COL_NAME (which is first_selected_item_index's column, typically 0) + # The index itself (first_selected_item_index) can be used as it's for COL_NAME. + current_name_suggestion = self.unified_model.data(first_selected_item_index, Qt.DisplayRole) or "" + elif isinstance(first_item_object, FileRule): + # For FileRule, its target asset name override is in COL_TARGET_ASSET + target_asset_col_idx = self.unified_model.COL_TARGET_ASSET + target_asset_index_for_suggestion = first_selected_item_index.siblingAtColumn(target_asset_col_idx) + current_name_suggestion = self.unified_model.data(target_asset_index_for_suggestion, Qt.DisplayRole) or "" + + new_name_input, ok = QInputDialog.getText(self, "Set Name", "Enter new name for selected items:", QLineEdit.EchoMode.Normal, current_name_suggestion) + if ok and new_name_input is not None: + stripped_name = new_name_input.strip() + if stripped_name: + log.info(f"User entered new name: '{stripped_name}' for selected items.") + + initial_selected_indices = self.main_panel_widget.unified_view.selectedIndexes() + objects_to_rename = [] + # To avoid processing same underlying item multiple times if multiple columns selected + processed_rows_for_object_collection = set() + + for view_idx in initial_selected_indices: + model_idx_for_item = self.unified_model.index(view_idx.row(), 0, view_idx.parent()) + if model_idx_for_item.row() not in processed_rows_for_object_collection: + item = self.unified_model.getItem(model_idx_for_item) + if isinstance(item, (AssetRule, FileRule)): + objects_to_rename.append(item) + processed_rows_for_object_collection.add(model_idx_for_item.row()) + else: + log.debug(f"F2 RENAME: Skipping item {item!r} (type: {type(item)}) during object collection as it's not AssetRule or FileRule.") + + log.debug(f"F2 RENAME: Collected {len(objects_to_rename)} AssetRule/FileRule objects to rename.") + + successful_renames = 0 + for item_object in objects_to_rename: + current_model_index = self.unified_model.findIndexForItem(item_object) + + if current_model_index is None or not current_model_index.isValid(): + item_repr = getattr(item_object, 'asset_name', getattr(item_object, 'file_path', repr(item_object))) + log.warning(f"F2 RENAME: Could not find current index for item {item_repr!r}. It might have been moved/deleted unexpectedly. Skipping.") + continue + + target_column = -1 + item_description_for_log = "" + + if isinstance(item_object, AssetRule): + target_column = self.unified_model.COL_NAME + item_description_for_log = f"AssetRule '{item_object.asset_name}'" + elif isinstance(item_object, FileRule): + target_column = self.unified_model.COL_TARGET_ASSET + item_description_for_log = f"FileRule '{Path(item_object.file_path).name}'" + + if target_column == -1: + log.warning(f"F2 RENAME: Unknown item type for {item_object!r}. Cannot determine target column. Skipping.") + continue + + index_to_update_in_column = current_model_index.siblingAtColumn(target_column) + + log.debug(f"F2 RENAME: Attempting to set new name '{stripped_name}' for {item_description_for_log} at index r={index_to_update_in_column.row()}, c={index_to_update_in_column.column()}") + success = self.unified_model.setData(index_to_update_in_column, stripped_name, Qt.EditRole) + + if success: + successful_renames += 1 + log.info(f"F2 RENAME: Successfully renamed {item_description_for_log} to '{stripped_name}'.") + else: + log.warning(f"F2 RENAME: Failed to rename {item_description_for_log} to '{stripped_name}'. setData returned False.") + + self.statusBar().showMessage(f"{successful_renames} item(s) renamed to '{stripped_name}'.", 3000) + keybind_processed = True + else: + log.debug("Asset name change aborted: name was empty after stripping.") + else: + log.debug("Asset name change cancelled or empty name entered.") + event.accept() + return + + if modifiers == Qt.ControlModifier: + log.debug(f"Ctrl modifier detected with key: {pressed_key}") + qt_key_sequence_str = QKeySequence(pressed_key).toString() + if pressed_key in self.qt_key_to_ftd_map: + target_ftd_keys = self.qt_key_to_ftd_map[pressed_key] + log.debug(f"Keybind match: Ctrl+{qt_key_sequence_str} maps to FTDs: {target_ftd_keys}") + if not target_ftd_keys: + log.warning(f"No FTDs configured for key Ctrl+{qt_key_sequence_str}") + super().keyPressEvent(event) + return + + for index in model_indexes_to_process: + item = self.unified_model.getItem(index) + log.debug(f"Processing item for keybind: row={index.row()}, column={index.column()}") + log.debug(f" Item object: {item!r}") + log.debug(f" Item type: {type(item)}") + log.debug(f" Is instance of FileRule: {isinstance(item, FileRule)}") + if hasattr(item, '__dict__'): + log.debug(f" Item attributes: {item.__dict__}") + + if not isinstance(item, FileRule): + log.debug(f"Skipping item at row {index.row()} because it's not a FileRule instance (actual type: {type(item)}).") + continue + + item_type_display_index = self.unified_model.index(index.row(), self.unified_model.COL_ITEM_TYPE, index.parent()) + current_map_type = self.unified_model.data(item_type_display_index, Qt.DisplayRole) + log.debug(f"Item at row {index.row()} ({Path(item.file_path).name}), current map_type (DisplayRole): '{current_map_type}'") + + new_map_type = "" + if len(target_ftd_keys) == 1: + new_map_type = target_ftd_keys[0] + log.debug(f" Single target FTD: '{new_map_type}'") + else: + log.debug(f" Toggle FTDs: {target_ftd_keys}. Current: '{current_map_type}'") + try: + current_ftd_index = target_ftd_keys.index(current_map_type) + next_ftd_index = (current_ftd_index + 1) % len(target_ftd_keys) + new_map_type = target_ftd_keys[next_ftd_index] + log.debug(f" Calculated next FTD: '{new_map_type}'") + except ValueError: + new_map_type = target_ftd_keys[0] + log.debug(f" Current not in toggle list, defaulting to first: '{new_map_type}'") + + if new_map_type and new_map_type != current_map_type: + log.debug(f" Updating item at row {index.row()} ({Path(item.file_path).name}) from '{current_map_type}' to '{new_map_type}'") + item_type_edit_index = self.unified_model.index(index.row(), self.unified_model.COL_ITEM_TYPE, index.parent()) + success = self.unified_model.setData(item_type_edit_index, new_map_type, Qt.EditRole) + log.debug(f" setData call successful: {success}") + elif not new_map_type: + log.debug(f" Skipping update for item at row {index.row()}, new_map_type is empty.") + else: + log.debug(f" Skipping update for item at row {index.row()}, new_map_type ('{new_map_type}') is same as current ('{current_map_type}').") + + # The model should emit dataChanged for each setData call. + self.statusBar().showMessage(f"File types updated for selected items.", 3000) + keybind_processed = True + event.accept() + return + + if not keybind_processed: + log.debug("Key press not handled by custom keybinds, passing to super.") + super().keyPressEvent(event) + -# --- Main Execution --- def run_gui(): """Initializes and runs the Qt application.""" - app = QApplication(sys.argv) - app.setStyle('Fusion') + print("--- Reached run_gui() ---") + from PySide6.QtGui import QKeySequence + + app = QApplication(sys.argv) - # Set a custom palette to override default Fusion colors palette = app.palette() grey_color = QColor("#3a3a3a") palette.setColor(QPalette.ColorRole.Base, grey_color) - palette.setColor(QPalette.ColorRole.AlternateBase, grey_color.lighter(110)) # Use a slightly lighter shade for alternate rows if needed + palette.setColor(QPalette.ColorRole.AlternateBase, grey_color.lighter(110)) # You might need to experiment with other roles depending on which widgets are affected - # palette.setColor(QPalette.ColorRole.Window, grey_color) - # palette.setColor(QPalette.ColorRole.WindowText, Qt.GlobalColor.white) # Example: set text color to white app.setPalette(palette) diff --git a/gui/prediction_handler.py b/gui/prediction_handler.py index f7965ee..9cdd641 100644 --- a/gui/prediction_handler.py +++ b/gui/prediction_handler.py @@ -1,15 +1,18 @@ -# gui/prediction_handler.py import logging from pathlib import Path -import time # For potential delays if needed -import os # For cpu_count -from concurrent.futures import ThreadPoolExecutor, as_completed # For parallel prediction +import time +import os +import re +import tempfile +import zipfile +from collections import defaultdict, Counter +from typing import List, Dict, Any # --- PySide6 Imports --- -from PySide6.QtCore import QObject, Signal, QThread # Import QThread +from PySide6.QtCore import QObject, Slot # Keep QObject for parent type hint, Slot for classify_files if kept as method +# Removed Signal, QThread as they are handled by BasePredictionHandler or caller # --- Backend Imports --- -# Adjust path to ensure modules can be found relative to this file's location import sys script_dir = Path(__file__).parent project_root = script_dir.parent @@ -18,215 +21,494 @@ if str(project_root) not in sys.path: try: from configuration import Configuration, ConfigurationError - from asset_processor import AssetProcessor, AssetProcessingError + from rule_structure import SourceRule, AssetRule, FileRule + from .base_prediction_handler import BasePredictionHandler BACKEND_AVAILABLE = True except ImportError as e: - print(f"ERROR (PredictionHandler): Failed to import backend modules: {e}") - # Define placeholders if imports fail + print(f"ERROR (RuleBasedPredictionHandler): Failed to import backend/config/base modules: {e}") Configuration = None - AssetProcessor = None + load_base_config = None ConfigurationError = Exception - AssetProcessingError = Exception + SourceRule, AssetRule, FileRule = (None,)*3 BACKEND_AVAILABLE = False log = logging.getLogger(__name__) -# Basic config if logger hasn't been set up elsewhere if not log.hasHandlers(): - logging.basicConfig(level=logging.INFO, format='%(levelname)s (PredictHandler): %(message)s') + logging.basicConfig(level=logging.INFO, format='%(levelname)s (RuleBasedPredictHandler): %(message)s') -class PredictionHandler(QObject): +def classify_files(file_list: List[str], config: Configuration) -> Dict[str, List[Dict[str, Any]]]: """ - Handles running predictions in a separate thread to avoid GUI freezes. + Analyzes a list of files based on configuration rules using a two-pass approach + to group them by asset and determine initial file properties. + Pass 1: Identifies and classifies prioritized bit depth variants. + Pass 2: Classifies extras, general maps (downgrading if primary exists), and ignores. + + Args: + file_list: List of absolute file paths. + config: The loaded Configuration object containing naming rules. + + Returns: + A dictionary grouping file information by predicted asset name. + Example: + { + 'AssetName1': [ + {'file_path': '/path/to/AssetName1_DISP16.png', 'item_type': 'DISP', 'asset_name': 'AssetName1'}, + {'file_path': '/path/to/AssetName1_DISP.png', 'item_type': 'EXTRA', 'asset_name': 'AssetName1'}, + {'file_path': '/path/to/AssetName1_Color.png', 'item_type': 'COL', 'asset_name': 'AssetName1'} + ], + # ... other assets + } + Returns an empty dict if classification fails or no files are provided. """ - # --- Signals --- - # Emits a list of dictionaries, each representing a file row for the table - # Dict format: {'original_path': str, 'predicted_asset_name': str | None, 'predicted_output_name': str | None, 'status': str, 'details': str | None, 'source_asset': str} - prediction_results_ready = Signal(list) - # Emitted when all predictions for a batch are done - prediction_finished = Signal() - # Emitted for status updates - status_message = Signal(str, int) + temp_grouped_files = defaultdict(list) + extra_files_to_associate = [] + primary_asset_names = set() + primary_assignments = set() + processed_in_pass1 = set() - def __init__(self, parent=None): - super().__init__(parent) - self._is_running = False - # No explicit cancel needed for prediction for now, it should be fast per-item + # --- Validation --- + if not file_list or not config: + log.warning("Classification skipped: Missing file list or config.") + return {} + if not hasattr(config, 'compiled_map_keyword_regex') or not config.compiled_map_keyword_regex: + log.warning("Classification skipped: Missing compiled map keyword regex in config.") + if not hasattr(config, 'compiled_extra_regex'): + log.warning("Configuration object missing 'compiled_extra_regex'. Cannot classify extra files.") + if not hasattr(config, 'compiled_bit_depth_regex_map'): + log.warning("Configuration object missing 'compiled_bit_depth_regex_map'. Cannot prioritize bit depth variants.") - @property - def is_running(self): - return self._is_running + compiled_map_regex = getattr(config, 'compiled_map_keyword_regex', {}) + compiled_extra_regex = getattr(config, 'compiled_extra_regex', []) + compiled_bit_depth_regex_map = getattr(config, 'compiled_bit_depth_regex_map', {}) - def _predict_single_asset(self, input_path_str: str, config: Configuration) -> list[dict]: - """ - Helper method to predict a single asset. Runs within the ThreadPoolExecutor. - Returns a list of prediction dictionaries for the asset, or a single error dict. - """ - input_path = Path(input_path_str) - source_asset_name = input_path.name # For reference in the results - asset_results = [] + num_map_rules = sum(len(patterns) for patterns in compiled_map_regex.values()) + num_extra_rules = len(compiled_extra_regex) + num_bit_depth_rules = len(compiled_bit_depth_regex_map) + + log.debug(f"Starting classification for {len(file_list)} files using {num_map_rules} map keyword patterns, {num_bit_depth_rules} bit depth patterns, and {num_extra_rules} extra patterns.") + + # --- Asset Name Extraction Helper --- + def get_asset_name(f_path: Path, cfg: Configuration) -> str: + filename = f_path.name + asset_name = None try: - # Create AssetProcessor instance (needs dummy output path) - # Ensure AssetProcessor is thread-safe or create a new instance per thread. - # Based on its structure (using temp dirs), creating new instances should be safe. - processor = AssetProcessor(input_path, config, Path(".")) # Dummy output path + separator = cfg.source_naming_separator + indices = cfg.source_naming_indices + base_name_index = indices.get('base_name') - # Get detailed file predictions - detailed_predictions = processor.get_detailed_file_predictions() - - if detailed_predictions is None: - log.error(f"Detailed prediction failed critically for {input_path_str}. Adding asset-level error.") - # Add a single error entry for the whole asset if the method returns None - asset_results.append({ - 'original_path': source_asset_name, # Use asset name as placeholder - 'predicted_asset_name': None, # New key - 'predicted_output_name': None, # New key - 'status': 'Error', - 'details': 'Critical prediction failure (check logs)', - 'source_asset': source_asset_name - }) + if separator is not None and base_name_index is not None: + stem = f_path.stem + parts = stem.split(separator) + if 0 <= base_name_index < len(parts): + asset_name = parts[base_name_index] + else: + log.warning(f"Preset base_name index {base_name_index} out of bounds for '{stem}' split by '{separator}'. Falling back.") else: - log.debug(f"Received {len(detailed_predictions)} detailed predictions for {input_path_str}.") - # Add source_asset key and ensure correct keys exist - for prediction_dict in detailed_predictions: - # Ensure all expected keys are present, even if None - result_entry = { - 'original_path': prediction_dict.get('original_path', '[Missing Path]'), - 'predicted_asset_name': prediction_dict.get('predicted_asset_name'), # New key - 'predicted_output_name': prediction_dict.get('predicted_output_name'), # New key - 'status': prediction_dict.get('status', 'Error'), - 'details': prediction_dict.get('details', '[Missing Details]'), - 'source_asset': source_asset_name # Add the source asset identifier - } - asset_results.append(result_entry) + log.debug(f"Preset rules for asset name extraction incomplete (separator: {separator}, index: {base_name_index}). Falling back for '{filename}'.") - except AssetProcessingError as e: # Catch errors during processor instantiation or prediction setup - log.error(f"Asset processing error during prediction setup for {input_path_str}: {e}") - asset_results.append({ - 'original_path': source_asset_name, - 'predicted_asset_name': None, - 'predicted_output_name': None, - 'status': 'Error', - 'details': f'Asset Error: {e}', - 'source_asset': source_asset_name - }) - except Exception as e: # Catch unexpected errors - log.exception(f"Unexpected error during prediction for {input_path_str}: {e}") - asset_results.append({ - 'original_path': source_asset_name, - 'predicted_asset_name': None, - 'predicted_output_name': None, - 'status': 'Error', - 'details': f'Unexpected Error: {e}', - 'source_asset': source_asset_name + if not asset_name: + asset_name = f_path.stem.split('_')[0] if '_' in f_path.stem else f_path.stem + log.debug(f"Used fallback asset name extraction: '{asset_name}' for '{filename}'.") + + except Exception as e: + log.exception(f"Error extracting asset name for '{filename}': {e}. Falling back to stem.") + asset_name = f_path.stem + + if not asset_name: + asset_name = f_path.stem + log.warning(f"Asset name extraction resulted in empty string for '{filename}'. Using stem: '{asset_name}'.") + return asset_name + + # --- Pass 1: Prioritized Bit Depth Variants --- + log.debug("--- Starting Classification Pass 1: Prioritized Variants ---") + for file_path_str in file_list: + file_path = Path(file_path_str) + filename = file_path.name + asset_name = get_asset_name(file_path, config) + processed = False + + for target_type, variant_regex in compiled_bit_depth_regex_map.items(): + match = variant_regex.search(filename) + if match: + log.debug(f"PASS 1: File '{filename}' matched PRIORITIZED bit depth variant for type '{target_type}'.") + matched_item_type = target_type + + if (asset_name, matched_item_type) in primary_assignments: + log.warning(f"PASS 1: Primary assignment ({asset_name}, {matched_item_type}) already exists. File '{filename}' will be handled in Pass 2.") + else: + primary_assignments.add((asset_name, matched_item_type)) + log.debug(f" PASS 1: Added primary assignment: ({asset_name}, {matched_item_type})") + primary_asset_names.add(asset_name) + + temp_grouped_files[asset_name].append({ + 'file_path': file_path_str, + 'item_type': matched_item_type, + 'asset_name': asset_name + }) + processed_in_pass1.add(file_path_str) + processed = True + break # Stop checking other variant patterns for this file + + log.debug(f"--- Finished Pass 1. Primary assignments made: {primary_assignments} ---") + + # --- Pass 2: Extras, General Maps, Ignores --- + log.debug("--- Starting Classification Pass 2: Extras, General Maps, Ignores ---") + for file_path_str in file_list: + if file_path_str in processed_in_pass1: + log.debug(f"PASS 2: Skipping '{Path(file_path_str).name}' (processed in Pass 1).") + continue + + file_path = Path(file_path_str) + filename = file_path.name + asset_name = get_asset_name(file_path, config) + is_extra = False + is_map = False + + # 1. Check for Extra Files FIRST in Pass 2 + for extra_pattern in compiled_extra_regex: + if extra_pattern.search(filename): + log.debug(f"PASS 2: File '{filename}' matched EXTRA pattern: {extra_pattern.pattern}") + extra_files_to_associate.append((file_path_str, filename)) + is_extra = True + break + + if is_extra: + continue + + # 2. Check for General Map Files in Pass 2 + for target_type, patterns_list in compiled_map_regex.items(): + for compiled_regex, original_keyword, rule_index in patterns_list: + match = compiled_regex.search(filename) + if match: + try: + # map_type_mapping_list = config.map_type_mapping # Old gloss logic source + # matched_rule_details = map_type_mapping_list[rule_index] # Old gloss logic source + # is_gloss_flag = matched_rule_details.get('is_gloss_source', False) # Old gloss logic + log.debug(f" PASS 2: Match found! Rule Index: {rule_index}, Keyword: '{original_keyword}', Target: '{target_type}'") # Removed Gloss from log + except Exception as e: + log.exception(f" PASS 2: Error accessing rule details for index {rule_index}: {e}") + + # *** Crucial Check: Has a prioritized variant claimed this type? *** + if (asset_name, target_type) in primary_assignments: + log.debug(f"PASS 2: File '{filename}' matched '{original_keyword}' for type '{target_type}', but primary already assigned via Pass 1. Classifying as EXTRA.") + matched_item_type = "EXTRA" + # is_gloss_flag = False # Old gloss logic + else: + log.debug(f"PASS 2: File '{filename}' matched '{original_keyword}' for item_type '{target_type}'.") + matched_item_type = target_type + + temp_grouped_files[asset_name].append({ + 'file_path': file_path_str, + 'item_type': matched_item_type, + 'asset_name': asset_name + }) + is_map = True + break + if is_map: + break + + # 3. Handle Unmatched Files in Pass 2 (Not Extra, Not Map) + if not is_extra and not is_map: + log.debug(f"PASS 2: File '{filename}' did not match any map/extra pattern. Grouping under asset '{asset_name}' as FILE_IGNORE.") + temp_grouped_files[asset_name].append({ + 'file_path': file_path_str, + 'item_type': "FILE_IGNORE", + 'asset_name': asset_name }) - finally: - # Cleanup for the single asset prediction if needed (AssetProcessor handles its own temp dir) - pass - return asset_results + + log.debug("--- Finished Pass 2 ---") + + # --- Determine Primary Asset Name for Extra Association (using Pass 1 results) --- + final_primary_asset_name = None + if primary_asset_names: + primary_map_asset_names_pass1 = [ + f_info['asset_name'] + for asset_files in temp_grouped_files.values() + for f_info in asset_files + if f_info['asset_name'] in primary_asset_names and (f_info['asset_name'], f_info['item_type']) in primary_assignments + ] + if primary_map_asset_names_pass1: + name_counts = Counter(primary_map_asset_names_pass1) + most_common_names = name_counts.most_common() + final_primary_asset_name = most_common_names[0][0] + if len(most_common_names) > 1 and most_common_names[0][1] == most_common_names[1][1]: + tied_names = sorted([name for name, count in most_common_names if count == most_common_names[0][1]]) + final_primary_asset_name = tied_names[0] + log.warning(f"Multiple primary asset names tied for most common based on Pass 1: {tied_names}. Using '{final_primary_asset_name}' for associating extra files.") + log.debug(f"Determined primary asset name for extras based on Pass 1 primary maps: '{final_primary_asset_name}'") + else: + log.warning("Primary asset names set (from Pass 1) was populated, but no corresponding groups found. Falling back.") + + if not final_primary_asset_name: + if temp_grouped_files and extra_files_to_associate: + fallback_name = sorted(temp_grouped_files.keys())[0] + final_primary_asset_name = fallback_name + log.warning(f"No primary map files found in Pass 1. Associating extras with first group found alphabetically: '{final_primary_asset_name}'.") + elif extra_files_to_associate: + log.warning(f"Could not determine any asset name to associate {len(extra_files_to_associate)} extra file(s) with. They will be ignored.") + else: + log.debug("No primary asset name determined (no maps or extras found).") - def run_prediction(self, input_paths: list[str], preset_name: str): + # --- Associate Extra Files (collected in Pass 2) --- + if final_primary_asset_name and extra_files_to_associate: + log.debug(f"Associating {len(extra_files_to_associate)} extra file(s) with primary asset '{final_primary_asset_name}'") + for file_path_str, filename in extra_files_to_associate: + if not any(f['file_path'] == file_path_str for f in temp_grouped_files[final_primary_asset_name]): + temp_grouped_files[final_primary_asset_name].append({ + 'file_path': file_path_str, + 'item_type': "EXTRA", + 'asset_name': final_primary_asset_name + }) + else: + log.debug(f"Skipping duplicate association of extra file: {filename}") + elif extra_files_to_associate: + pass + + + log.debug(f"Classification complete. Found {len(temp_grouped_files)} potential assets.") + return dict(temp_grouped_files) + + +class RuleBasedPredictionHandler(BasePredictionHandler): + """ + Handles running rule-based predictions in a separate thread using presets. + Generates the initial SourceRule hierarchy based on file lists and presets. + Inherits from BasePredictionHandler for common threading and signaling. + """ + + def __init__(self, input_source_identifier: str, original_input_paths: list[str], preset_name: str, parent: QObject = None): """ - Runs the prediction logic for the given paths and preset using a ThreadPoolExecutor. - This method is intended to be run in a separate QThread. + Initializes the rule-based handler. + + Args: + input_source_identifier: The unique identifier for the input source (e.g., file path). + original_input_paths: List of absolute file paths extracted from the source. + preset_name: The name of the preset configuration to use. + parent: The parent QObject. """ - if self._is_running: - log.warning("Prediction is already running.") - return - if not BACKEND_AVAILABLE: - log.error("Backend modules not available. Cannot run prediction.") - self.status_message.emit("Error: Backend components missing.", 5000) - self.prediction_finished.emit() - return - if not preset_name: - log.warning("No preset selected for prediction.") - self.status_message.emit("No preset selected.", 3000) - self.prediction_finished.emit() + super().__init__(input_source_identifier, parent) + self.original_input_paths = original_input_paths + self.preset_name = preset_name + self._current_input_path = None + self._current_file_list = None + self._current_preset_name = None + + # Re-introduce run_prediction as the main slot to receive requests + @Slot(str, list, str) + def run_prediction(self, input_source_identifier: str, original_input_paths: list[str], preset_name: str): + """ + Generates the initial SourceRule hierarchy for a given source identifier, + file list, and preset name. Populates only overridable fields based on + classification and preset defaults. + This method is intended to be run in the handler's QThread. + Uses the base class signals for reporting results/errors. + """ + # Check if already running a prediction for a *different* source + # Allow re-triggering for the *same* source if needed (e.g., preset changed) + if self._is_running and self._current_input_path != input_source_identifier: + log.warning(f"RuleBasedPredictionHandler is busy with '{self._current_input_path}'. Ignoring request for '{input_source_identifier}'.") return self._is_running = True - thread_id = QThread.currentThread() # Get current thread object - log.info(f"[{time.time():.4f}][T:{thread_id}] --> Entered PredictionHandler.run_prediction. Starting run for {len(input_paths)} items, Preset='{preset_name}'") - self.status_message.emit(f"Updating preview for {len(input_paths)} items...", 0) + self._is_cancelled = False + self._current_input_path = input_source_identifier + self._current_file_list = original_input_paths + self._current_preset_name = preset_name - config = None # Load config once if possible + log.info(f"Starting rule-based prediction for: {input_source_identifier} using preset: {preset_name}") + self.status_update.emit(f"Starting analysis for '{Path(input_source_identifier).name}'...") + + source_rules_list = [] try: + if not BACKEND_AVAILABLE: + raise RuntimeError("Backend/config modules not available. Cannot run prediction.") + + if not preset_name: + log.warning("No preset selected for prediction.") + self.status_update.emit("No preset selected.") + self.prediction_ready.emit(input_source_identifier, []) + self._is_running = False + return + + source_path = Path(input_source_identifier) + if not source_path.exists(): + log.warning(f"Input source path does not exist: '{input_source_identifier}'. Skipping prediction.") + raise FileNotFoundError(f"Input source path not found: {input_source_identifier}") + + # --- Load Configuration --- config = Configuration(preset_name) - except ConfigurationError as e: - log.error(f"Failed to load configuration for preset '{preset_name}': {e}") - self.status_message.emit(f"Error loading preset '{preset_name}': {e}", 5000) - # Emit error for all items? Or just finish? Finish for now. - self.prediction_finished.emit() + log.info(f"Successfully loaded configuration for preset '{preset_name}'.") + + if self._is_cancelled: raise RuntimeError("Prediction cancelled before classification.") + + # --- Perform Classification --- + self.status_update.emit(f"Classifying files for '{source_path.name}'...") + try: + classified_assets = classify_files(original_input_paths, config) + except Exception as e: + log.exception(f"Error during file classification for source '{input_source_identifier}': {e}") + raise RuntimeError(f"Error classifying files: {e}") from e + + if self._is_cancelled: raise RuntimeError("Prediction cancelled after classification.") + + if not classified_assets: + log.warning(f"Classification yielded no assets for source '{input_source_identifier}'.") + self.status_update.emit("No assets identified from files.") + self.prediction_ready.emit(input_source_identifier, []) + self._is_running = False + return + + # --- Build the Hierarchy --- + self.status_update.emit(f"Building rule hierarchy for '{source_path.name}'...") + try: + supplier_identifier = config.supplier_name + source_rule = SourceRule( + input_path=input_source_identifier, + supplier_identifier=supplier_identifier, + preset_name=preset_name + ) + asset_rules = [] + file_type_definitions = config._core_settings.get('FILE_TYPE_DEFINITIONS', {}) + + for asset_name, files_info in classified_assets.items(): + if self._is_cancelled: raise RuntimeError("Prediction cancelled during hierarchy building (assets).") + if not files_info: continue + + asset_category_rules = config.asset_category_rules + asset_type_definitions = config.get_asset_type_definitions() + asset_type_keys = list(asset_type_definitions.keys()) + + # Initialize predicted_asset_type using the validated default + predicted_asset_type = config.default_asset_category + log.debug(f"Asset '{asset_name}': Initial predicted_asset_type set to default: '{predicted_asset_type}'.") + + # 1. Check asset_category_rules from preset + determined_by_rule = False + + # Check for Model type based on file patterns + if "Model" in asset_type_keys: + model_patterns_regex = config.compiled_model_regex + for f_info in files_info: + if f_info['item_type'] in ["EXTRA", "FILE_IGNORE"]: + continue + file_path_obj = Path(f_info['file_path']) + for pattern_re in model_patterns_regex: + if pattern_re.search(file_path_obj.name): + predicted_asset_type = "Model" + determined_by_rule = True + log.debug(f"Asset '{asset_name}' classified as 'Model' due to file '{file_path_obj.name}' matching pattern '{pattern_re.pattern}'.") + break + if determined_by_rule: + break + + # Check for Decal type based on keywords in asset name (if not already Model) + if not determined_by_rule and "Decal" in asset_type_keys: + decal_keywords = asset_category_rules.get('decal_keywords', []) + for keyword in decal_keywords: + # Ensure keyword is a string before trying to escape it + if isinstance(keyword, str) and keyword: + try: + if re.search(r'\b' + re.escape(keyword) + r'\b', asset_name, re.IGNORECASE): + predicted_asset_type = "Decal" + determined_by_rule = True + log.debug(f"Asset '{asset_name}' classified as 'Decal' due to keyword '{keyword}'.") + break + except re.error as e_re: + log.warning(f"Regex error with decal_keyword '{keyword}': {e_re}") + if determined_by_rule: + pass + + # 2. If not determined by specific rules, check for Surface (if not Model/Decal by rule) + if not determined_by_rule and predicted_asset_type == config.default_asset_category and "Surface" in asset_type_keys: + item_types_in_asset = {f_info['item_type'] for f_info in files_info} + # Ensure we are checking against standard map types from FILE_TYPE_DEFINITIONS + # This check is primarily for PBR texture sets. + material_indicators = { + ft_key for ft_key, ft_def in config.get_file_type_definitions_with_examples().items() + if ft_def.get('standard_type') and ft_def.get('standard_type') not in ["", "EXTRA", "FILE_IGNORE", "MODEL"] + } + # Add common direct standard types as well for robustness + material_indicators.update({"COL", "NRM", "ROUGH", "METAL", "AO", "DISP"}) + + + has_material_map = False + for item_type in item_types_in_asset: + # Check if the item_type itself is a material indicator or its standard_type is + if item_type in material_indicators: + has_material_map = True + break + # Check standard type if item_type is a key in FILE_TYPE_DEFINITIONS + item_def = config.get_file_type_definitions_with_examples().get(item_type) + if item_def and item_def.get('standard_type') in material_indicators: + has_material_map = True + break + + if has_material_map: + predicted_asset_type = "Surface" + log.debug(f"Asset '{asset_name}' classified as 'Surface' due to material indicators.") + + # 3. Final validation: Ensure predicted_asset_type is a valid key. + if predicted_asset_type not in asset_type_keys: + log.warning(f"Derived AssetType '{predicted_asset_type}' for asset '{asset_name}' is not in ASSET_TYPE_DEFINITIONS. " + f"Falling back to default: '{config.default_asset_category}'.") + predicted_asset_type = config.default_asset_category + + asset_rule = AssetRule(asset_name=asset_name, asset_type=predicted_asset_type) + file_rules = [] + for file_info in files_info: + if self._is_cancelled: raise RuntimeError("Prediction cancelled during hierarchy building (files).") + + base_item_type = file_info['item_type'] + target_asset_name_override = file_info['asset_name'] + final_item_type = base_item_type + if not base_item_type.startswith("MAP_") and base_item_type not in ["FILE_IGNORE", "EXTRA", "MODEL"]: + final_item_type = f"MAP_{base_item_type}" + + if file_type_definitions and final_item_type not in file_type_definitions and base_item_type not in ["FILE_IGNORE", "EXTRA"]: + log.warning(f"Predicted ItemType '{base_item_type}' (checked as '{final_item_type}') for file '{file_info['file_path']}' is not in FILE_TYPE_DEFINITIONS. Setting to FILE_IGNORE.") + final_item_type = "FILE_IGNORE" + + + # is_gloss_source_value = file_info.get('is_gloss_source', False) # Removed + + file_rule = FileRule( + file_path=file_info['file_path'], + item_type=final_item_type, + item_type_override=final_item_type, + target_asset_name_override=target_asset_name_override, + output_format_override=None, + # is_gloss_source=is_gloss_source_value if isinstance(is_gloss_source_value, bool) else False, # Removed + resolution_override=None, + channel_merge_instructions={}, + ) + file_rules.append(file_rule) + asset_rule.files = file_rules + asset_rules.append(asset_rule) + source_rule.assets = asset_rules + source_rules_list.append(source_rule) + + except Exception as e: + log.exception(f"Error building rule hierarchy for source '{input_source_identifier}': {e}") + raise RuntimeError(f"Error building rule hierarchy: {e}") from e + + # --- Emit Success Signal --- + log.info(f"Rule-based prediction finished successfully for '{input_source_identifier}'.") + self.prediction_ready.emit(input_source_identifier, source_rules_list) + + except Exception as e: + # --- Emit Error Signal --- + log.exception(f"Error during rule-based prediction for '{input_source_identifier}': {e}") + error_msg = f"Error analyzing '{Path(input_source_identifier).name}': {e}" + self.prediction_error.emit(input_source_identifier, error_msg) + + finally: self._is_running = False - return - except Exception as e: - log.exception(f"Unexpected error loading configuration for preset '{preset_name}': {e}") - self.status_message.emit(f"Unexpected error loading preset '{preset_name}'.", 5000) - self.prediction_finished.emit() - return - - all_file_results = [] # Accumulate results here - futures = [] - # Determine number of workers - use half the cores, minimum 1, max 8? - max_workers = min(max(1, (os.cpu_count() or 1) // 2), 8) - log.info(f"Using ThreadPoolExecutor with max_workers={max_workers} for prediction.") - - try: - with ThreadPoolExecutor(max_workers=max_workers) as executor: - # Submit tasks for each input path - for input_path_str in input_paths: - future = executor.submit(self._predict_single_asset, input_path_str, config) - futures.append(future) - - # Process results as they complete - for future in as_completed(futures): - try: - # Result is a list of dicts for one asset - asset_result_list = future.result() - if asset_result_list: # Check if list is not empty - all_file_results.extend(asset_result_list) - except Exception as exc: - # This catches errors within the future execution itself if not handled by _predict_single_asset - log.error(f'Prediction task generated an exception: {exc}', exc_info=True) - # We might not know which input path failed here easily without more mapping - # Add a generic error? - all_file_results.append({ - 'original_path': '[Unknown Asset - Executor Error]', - 'predicted_asset_name': None, - 'predicted_output_name': None, - 'status': 'Error', - 'details': f'Executor Error: {exc}', - 'source_asset': '[Unknown]' - }) - - except Exception as pool_exc: - log.exception(f"An error occurred with the prediction ThreadPoolExecutor: {pool_exc}") - self.status_message.emit(f"Error during prediction setup: {pool_exc}", 5000) - # Add a generic error if the pool fails - all_file_results.append({ - 'original_path': '[Prediction Pool Error]', - 'predicted_asset_name': None, - 'predicted_output_name': None, - 'status': 'Error', - 'details': f'Pool Error: {pool_exc}', - 'source_asset': '[System]' - }) - - # Emit the combined list of detailed file results at the end - # Note: thread_id was already defined earlier in this function - log.info(f"[{time.time():.4f}][T:{thread_id}] Parallel prediction run finished. Preparing to emit {len(all_file_results)} file results.") - # <<< Add logging before emit >>> - log.debug(f"[{time.time():.4f}][T:{thread_id}] Type of all_file_results before emit: {type(all_file_results)}") - try: - log.debug(f"[{time.time():.4f}][T:{thread_id}] Content of all_file_results (first 5) before emit: {all_file_results[:5]}") - except Exception as e: - log.error(f"[{time.time():.4f}][T:{thread_id}] Error logging all_file_results content: {e}") - # <<< End added logging >>> - log.info(f"[{time.time():.4f}][T:{thread_id}] Emitting prediction_results_ready signal...") - self.prediction_results_ready.emit(all_file_results) - log.info(f"[{time.time():.4f}][T:{thread_id}] Emitted prediction_results_ready signal.") - self.status_message.emit("Preview update complete.", 3000) - self.prediction_finished.emit() - self._is_running = False - log.info(f"[{time.time():.4f}][T:{thread_id}] <-- Exiting PredictionHandler.run_prediction.") \ No newline at end of file + self._current_input_path = None + self._current_file_list = None + self._current_preset_name = None + log.info(f"Finished rule-based prediction run for: {input_source_identifier}") +def is_running(self) -> bool: + """Returns True if the handler is currently processing a prediction request.""" + return self._is_running diff --git a/gui/preset_editor_widget.py b/gui/preset_editor_widget.py new file mode 100644 index 0000000..25b9629 --- /dev/null +++ b/gui/preset_editor_widget.py @@ -0,0 +1,782 @@ +import sys +import os +import json +import logging +from pathlib import Path +from functools import partial + +from PySide6.QtWidgets import ( + QWidget, QVBoxLayout, QHBoxLayout, QListWidget, QPushButton, QLabel, QTabWidget, QComboBox, + QLineEdit, QTextEdit, QSpinBox, QTableWidget, QGroupBox, QFormLayout, + QHeaderView, QAbstractItemView, QListWidgetItem, QTableWidgetItem, QMessageBox, + QFileDialog, QInputDialog, QSizePolicy +) +from PySide6.QtCore import Qt, Signal, QObject, Slot +from PySide6.QtGui import QAction # Keep QAction if needed for context menus within editor later + +# --- Constants --- +# Assuming project root is parent of the directory containing this file +script_dir = Path(__file__).parent +project_root = script_dir.parent +PRESETS_DIR = project_root / "Presets" +TEMPLATE_PATH = PRESETS_DIR / "_template.json" +APP_SETTINGS_PATH_LOCAL = project_root / "config" / "app_settings.json" + + +log = logging.getLogger(__name__) + +# --- Preset Editor Widget --- + +class PresetEditorWidget(QWidget): + """ + Widget dedicated to managing and editing presets. + Contains the preset list, editor tabs, and save/load functionality. + """ + # Signal emitted when presets list changes (saved, deleted, new) + presets_changed_signal = Signal() + # Signal emitted when the selected preset (or LLM/Placeholder) changes + # Emits: mode ("preset", "llm", "placeholder"), preset_name (str or None) + preset_selection_changed_signal = Signal(str, str) + + def __init__(self, parent=None): + super().__init__(parent) + + # --- Internal State --- + self._last_valid_preset_name = None # Store the name of the last valid preset loaded + self.current_editing_preset_path = None + self.editor_unsaved_changes = False + self._is_loading_editor = False # Flag to prevent signals during load + + # --- UI Setup --- + self._init_ui() + + # --- Initial State --- + self._ftd_keys = self._get_file_type_definition_keys() + self._clear_editor() + self._set_editor_enabled(False) + self.populate_presets() + + # --- Connect Editor Signals --- + self._connect_editor_change_signals() + + def _get_file_type_definition_keys(self) -> list[str]: + """Loads FILE_TYPE_DEFINITIONS keys from app_settings.json.""" + keys = [] + try: + if APP_SETTINGS_PATH_LOCAL.is_file(): + with open(APP_SETTINGS_PATH_LOCAL, 'r', encoding='utf-8') as f: + settings = json.load(f) + ftd = settings.get("FILE_TYPE_DEFINITIONS", {}) + keys = list(ftd.keys()) + log.debug(f"Successfully loaded {len(keys)} FILE_TYPE_DEFINITIONS keys.") + else: + log.error(f"app_settings.json not found at {APP_SETTINGS_PATH_LOCAL} for PresetEditorWidget.") + except json.JSONDecodeError as e: + log.error(f"Failed to parse app_settings.json in PresetEditorWidget: {e}") + except Exception as e: + log.error(f"Error loading FILE_TYPE_DEFINITIONS keys in PresetEditorWidget: {e}") + return keys + + def _init_ui(self): + """Initializes the UI elements for the preset editor.""" + main_layout = QVBoxLayout(self) + main_layout.setContentsMargins(0, 0, 0, 0) # Let containers manage margins + main_layout.setSpacing(0) # No space between selector and editor containers + + # Preset List and Controls + self.selector_container = QWidget() + selector_layout = QVBoxLayout(self.selector_container) + selector_layout.setContentsMargins(5, 5, 5, 5) # Margins for selector area + + selector_layout.addWidget(QLabel("Presets:")) + self.editor_preset_list = QListWidget() + self.editor_preset_list.currentItemChanged.connect(self._load_selected_preset_for_editing) + selector_layout.addWidget(self.editor_preset_list) + + list_button_layout = QHBoxLayout() + self.editor_new_button = QPushButton("New") + self.editor_delete_button = QPushButton("Delete") + self.editor_new_button.clicked.connect(self._new_preset) + self.editor_delete_button.clicked.connect(self._delete_selected_preset) + list_button_layout.addWidget(self.editor_new_button) + list_button_layout.addWidget(self.editor_delete_button) + selector_layout.addLayout(list_button_layout) + main_layout.addWidget(self.selector_container) + + # Editor Tabs + self.json_editor_container = QWidget() + editor_layout = QVBoxLayout(self.json_editor_container) + editor_layout.setContentsMargins(5, 0, 5, 5) # Margins for editor area (no top margin) + + self.editor_tab_widget = QTabWidget() + self.editor_tab_general_naming = QWidget() + self.editor_tab_mapping_rules = QWidget() + self.editor_tab_widget.addTab(self.editor_tab_general_naming, "General & Naming") + self.editor_tab_widget.addTab(self.editor_tab_mapping_rules, "Mapping & Rules") + self._create_editor_general_tab() + self._create_editor_mapping_tab() + editor_layout.addWidget(self.editor_tab_widget, 1) # Allow tabs to stretch + + # Save Buttons + save_button_layout = QHBoxLayout() + self.editor_save_button = QPushButton("Save") + self.editor_save_as_button = QPushButton("Save As...") + self.editor_save_button.setEnabled(False) + self.editor_save_button.clicked.connect(self._save_current_preset) + self.editor_save_as_button.clicked.connect(self._save_preset_as) + save_button_layout.addStretch() + save_button_layout.addWidget(self.editor_save_button) + save_button_layout.addWidget(self.editor_save_as_button) + editor_layout.addLayout(save_button_layout) + + main_layout.addWidget(self.json_editor_container) + + def _create_editor_general_tab(self): + """Creates the widgets and layout for the 'General & Naming' editor tab.""" + layout = QVBoxLayout(self.editor_tab_general_naming) + form_layout = QFormLayout() + form_layout.setFieldGrowthPolicy(QFormLayout.FieldGrowthPolicy.ExpandingFieldsGrow) + + # Basic Info + self.editor_preset_name = QLineEdit() + self.editor_supplier_name = QLineEdit() + self.editor_notes = QTextEdit() + self.editor_notes.setAcceptRichText(False) + self.editor_notes.setFixedHeight(60) + form_layout.addRow("Preset Name:", self.editor_preset_name) + form_layout.addRow("Supplier Name:", self.editor_supplier_name) + form_layout.addRow("Notes:", self.editor_notes) + + layout.addLayout(form_layout) + + # Source Naming Group + naming_group = QGroupBox("Source File Naming Rules") + naming_layout_outer = QVBoxLayout(naming_group) + naming_layout_form = QFormLayout() + self.editor_separator = QLineEdit() + self.editor_separator.setMaxLength(1) + self.editor_spin_base_name_idx = QSpinBox() + self.editor_spin_base_name_idx.setMinimum(-1) + self.editor_spin_map_type_idx = QSpinBox() + self.editor_spin_map_type_idx.setMinimum(-1) + naming_layout_form.addRow("Separator:", self.editor_separator) + naming_layout_form.addRow("Base Name Index:", self.editor_spin_base_name_idx) + naming_layout_form.addRow("Map Type Index:", self.editor_spin_map_type_idx) + naming_layout_outer.addLayout(naming_layout_form) + # Gloss Keywords List + self._setup_list_widget_with_controls(naming_layout_outer, "Glossiness Keywords", "editor_list_gloss_keywords") + # Bit Depth Variants Table + self._setup_table_widget_with_controls(naming_layout_outer, "16-bit Variant Patterns", "editor_table_bit_depth_variants", ["Map Type", "Pattern"]) + self.editor_table_bit_depth_variants.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.ResizeToContents) + self.editor_table_bit_depth_variants.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch) + layout.addWidget(naming_group) + + # Extra Files Group + self._setup_list_widget_with_controls(layout, "Move to 'Extra' Folder Patterns", "editor_list_extra_patterns") + + layout.addStretch(1) + + def _create_editor_mapping_tab(self): + """Creates the widgets and layout for the 'Mapping & Rules' editor tab.""" + layout = QVBoxLayout(self.editor_tab_mapping_rules) + + # Map Type Mapping Group + self._setup_table_widget_with_controls(layout, "Map Type Mapping (Standard Type <- Input Keywords)", "editor_table_map_type_mapping", ["Standard Type", "Input Keywords (comma-sep)"]) + self.editor_table_map_type_mapping.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.ResizeToContents) + self.editor_table_map_type_mapping.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch) + + # Category Rules Group + category_group = QGroupBox("Asset Category Rules") + category_layout = QVBoxLayout(category_group) + self._setup_list_widget_with_controls(category_layout, "Model File Patterns", "editor_list_model_patterns") + self._setup_list_widget_with_controls(category_layout, "Decal Keywords", "editor_list_decal_keywords") + layout.addWidget(category_group) + + # Archetype Rules Group + self._setup_table_widget_with_controls(layout, "Archetype Rules", "editor_table_archetype_rules", ["Archetype Name", "Match Any (comma-sep)", "Match All (comma-sep)"]) + self.editor_table_archetype_rules.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.ResizeToContents) + self.editor_table_archetype_rules.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch) + self.editor_table_archetype_rules.horizontalHeader().setSectionResizeMode(2, QHeaderView.ResizeMode.Stretch) + + layout.addStretch(1) + + # --- Helper Functions for UI Setup (Moved into class) --- + def _setup_list_widget_with_controls(self, parent_layout, label_text, attribute_name): + """Adds a QListWidget with Add/Remove buttons to a layout.""" + list_widget = QListWidget() + list_widget.setAlternatingRowColors(True) + list_widget.setEditTriggers(QAbstractItemView.EditTrigger.DoubleClicked | QAbstractItemView.EditTrigger.SelectedClicked | QAbstractItemView.EditTrigger.EditKeyPressed) + setattr(self, attribute_name, list_widget) + + add_button = QPushButton("+") + remove_button = QPushButton("-") + add_button.setFixedWidth(30) + remove_button.setFixedWidth(30) + + button_layout = QVBoxLayout() + button_layout.addWidget(add_button) + button_layout.addWidget(remove_button) + button_layout.addStretch() + + list_layout = QHBoxLayout() + list_layout.addWidget(list_widget) + list_layout.addLayout(button_layout) + + group_box = QGroupBox(label_text) + group_box_layout = QVBoxLayout(group_box) + group_box_layout.addLayout(list_layout) + + parent_layout.addWidget(group_box) + + # Connections + add_button.clicked.connect(partial(self._editor_add_list_item, list_widget)) + remove_button.clicked.connect(partial(self._editor_remove_list_item, list_widget)) + list_widget.itemChanged.connect(self._mark_editor_unsaved) + + def _setup_table_widget_with_controls(self, parent_layout, label_text, attribute_name, columns): + """Adds a QTableWidget with Add/Remove buttons to a layout.""" + table_widget = QTableWidget() + table_widget.setColumnCount(len(columns)) + table_widget.setHorizontalHeaderLabels(columns) + table_widget.setAlternatingRowColors(True) + setattr(self, attribute_name, table_widget) + + add_button = QPushButton("+ Row") + remove_button = QPushButton("- Row") + + button_layout = QHBoxLayout() + button_layout.addStretch() + button_layout.addWidget(add_button) + button_layout.addWidget(remove_button) + + group_box = QGroupBox(label_text) + group_box_layout = QVBoxLayout(group_box) + group_box_layout.addWidget(table_widget) + group_box_layout.addLayout(button_layout) + + parent_layout.addWidget(group_box) + + # Connections + add_button.clicked.connect(partial(self._editor_add_table_row, table_widget)) + remove_button.clicked.connect(partial(self._editor_remove_table_row, table_widget)) + table_widget.itemChanged.connect(self._mark_editor_unsaved) + + # --- Preset Population and Handling --- + def populate_presets(self): + """Scans presets dir and populates the editor list.""" + log.debug("Populating preset list in PresetEditorWidget...") + current_list_item = self.editor_preset_list.currentItem() + current_list_selection_text = current_list_item.text() if current_list_item else None + + self.editor_preset_list.clear() + log.debug("Preset list cleared.") + + placeholder_item = QListWidgetItem("--- Select a Preset ---") + placeholder_item.setFlags(placeholder_item.flags() & ~Qt.ItemFlag.ItemIsSelectable & ~Qt.ItemFlag.ItemIsEditable) + placeholder_item.setData(Qt.ItemDataRole.UserRole, "__PLACEHOLDER__") + self.editor_preset_list.addItem(placeholder_item) + log.debug("Added '--- Select a Preset ---' placeholder item.") + + llm_item = QListWidgetItem("- LLM Interpretation -") + llm_item.setData(Qt.ItemDataRole.UserRole, "__LLM__") # Special identifier + self.editor_preset_list.addItem(llm_item) + log.debug("Added '- LLM Interpretation -' item.") + + if not PRESETS_DIR.is_dir(): + msg = f"Error: Presets directory not found at {PRESETS_DIR}" + log.error(msg) + return + + presets = sorted([f for f in PRESETS_DIR.glob("*.json") if f.is_file() and not f.name.startswith('_')]) + + if not presets: + msg = "Warning: No presets found in presets directory." + log.warning(msg) + else: + for preset_path in presets: + item = QListWidgetItem(preset_path.stem) + item.setData(Qt.ItemDataRole.UserRole, preset_path) + self.editor_preset_list.addItem(item) + log.info(f"Loaded {len(presets)} presets into editor list.") + + # Select the "Select a Preset" item by default + log.debug("Preset list populated. Selecting '--- Select a Preset ---' item.") + self.editor_preset_list.setCurrentItem(placeholder_item) + + # --- Preset Editor Methods --- + + def _editor_add_list_item(self, list_widget: QListWidget): + """Adds an editable item to the specified list widget in the editor.""" + text, ok = QInputDialog.getText(self, f"Add Item", "Enter value:") + if ok and text: + item = QListWidgetItem(text) + list_widget.addItem(item) + self._mark_editor_unsaved() + + def _editor_remove_list_item(self, list_widget: QListWidget): + """Removes the selected item from the specified list widget in the editor.""" + selected_items = list_widget.selectedItems() + if not selected_items: return + for item in selected_items: list_widget.takeItem(list_widget.row(item)) + self._mark_editor_unsaved() + + def _editor_add_table_row(self, table_widget: QTableWidget): + """Adds an empty row to the specified table widget in the editor.""" + row_count = table_widget.rowCount() + table_widget.insertRow(row_count) + + if table_widget == self.editor_table_map_type_mapping: + # Column 0: Standard Type (QComboBox) + combo_box = QComboBox() + if self._ftd_keys: + combo_box.addItems(self._ftd_keys) + else: + log.warning("FILE_TYPE_DEFINITIONS keys not available for ComboBox in map_type_mapping.") + combo_box.currentIndexChanged.connect(self._mark_editor_unsaved) + table_widget.setCellWidget(row_count, 0, combo_box) + # Column 1: Input Keywords (QTableWidgetItem) + table_widget.setItem(row_count, 1, QTableWidgetItem("")) + else: # For other tables + for col in range(table_widget.columnCount()): + table_widget.setItem(row_count, col, QTableWidgetItem("")) + self._mark_editor_unsaved() + + def _editor_remove_table_row(self, table_widget: QTableWidget): + """Removes the selected row(s) from the specified table widget in the editor.""" + selected_rows = sorted(list(set(index.row() for index in table_widget.selectedIndexes())), reverse=True) + if not selected_rows: + if table_widget.rowCount() > 0: selected_rows = [table_widget.rowCount() - 1] + else: return + for row in selected_rows: table_widget.removeRow(row) + self._mark_editor_unsaved() + + def _mark_editor_unsaved(self): + """Marks changes in the editor panel as unsaved.""" + if self._is_loading_editor: return + self.editor_unsaved_changes = True + self.editor_save_button.setEnabled(True) + + def _connect_editor_change_signals(self): + """Connect signals from all editor widgets to mark_editor_unsaved.""" + self.editor_preset_name.textChanged.connect(self._mark_editor_unsaved) + self.editor_supplier_name.textChanged.connect(self._mark_editor_unsaved) + self.editor_notes.textChanged.connect(self._mark_editor_unsaved) + self.editor_separator.textChanged.connect(self._mark_editor_unsaved) + self.editor_spin_base_name_idx.valueChanged.connect(self._mark_editor_unsaved) + self.editor_spin_map_type_idx.valueChanged.connect(self._mark_editor_unsaved) + # List/Table widgets are connected via helper functions + def check_unsaved_changes(self) -> bool: + """ + Checks for unsaved changes in the editor and prompts the user. + Returns True if the calling action should be cancelled. + (Called by MainWindow's closeEvent or before loading a new preset). + """ + if not self.editor_unsaved_changes: return False # No unsaved changes, proceed + reply = QMessageBox.question(self, "Unsaved Preset Changes", # Use self as parent + "You have unsaved changes in the preset editor. Discard them?", + QMessageBox.StandardButton.Save | QMessageBox.StandardButton.Discard | QMessageBox.StandardButton.Cancel, + QMessageBox.StandardButton.Cancel) + if reply == QMessageBox.StandardButton.Save: + save_successful = self._save_current_preset() + return not save_successful # Return True (cancel) if save fails + elif reply == QMessageBox.StandardButton.Discard: + return False # Discarded, proceed + else: # Cancelled + return True # Cancel the original action + + def _set_editor_enabled(self, enabled: bool): + """Enables or disables all editor widgets.""" + # Target the container holding the tabs and save buttons + self.json_editor_container.setEnabled(enabled) + # Save button state still depends on unsaved changes, but only if container is enabled + self.editor_save_button.setEnabled(enabled and self.editor_unsaved_changes) + + def _clear_editor(self): + """Clears the editor fields and resets state.""" + self._is_loading_editor = True + try: + self.editor_preset_name.clear() + self.editor_supplier_name.clear() + self.editor_notes.clear() + self.editor_separator.clear() + self.editor_spin_base_name_idx.setValue(0) + self.editor_spin_map_type_idx.setValue(1) + self.editor_list_gloss_keywords.clear() + self.editor_table_bit_depth_variants.setRowCount(0) + self.editor_list_extra_patterns.clear() + self.editor_table_map_type_mapping.setRowCount(0) + self.editor_list_model_patterns.clear() + self.editor_list_decal_keywords.clear() + self.editor_table_archetype_rules.setRowCount(0) + self.current_editing_preset_path = None + self.editor_unsaved_changes = False + self.editor_save_button.setEnabled(False) + self._set_editor_enabled(False) + finally: + self._is_loading_editor = False + + def _populate_editor_from_data(self, preset_data: dict): + """Helper method to populate editor UI widgets from a preset data dictionary.""" + self._is_loading_editor = True + try: + self.editor_preset_name.setText(preset_data.get("preset_name", "")) + self.editor_supplier_name.setText(preset_data.get("supplier_name", "")) + self.editor_notes.setText(preset_data.get("notes", "")) + naming_data = preset_data.get("source_naming", {}) + self.editor_separator.setText(naming_data.get("separator", "_")) + indices = naming_data.get("part_indices", {}) + self.editor_spin_base_name_idx.setValue(indices.get("base_name", 0)) + self.editor_spin_map_type_idx.setValue(indices.get("map_type", 1)) + self.editor_list_gloss_keywords.clear() + self.editor_list_gloss_keywords.addItems(naming_data.get("glossiness_keywords", [])) + self.editor_table_bit_depth_variants.setRowCount(0) + bit_depth_vars = naming_data.get("bit_depth_variants", {}) + for i, (map_type, pattern) in enumerate(bit_depth_vars.items()): + self.editor_table_bit_depth_variants.insertRow(i) + self.editor_table_bit_depth_variants.setItem(i, 0, QTableWidgetItem(map_type)) + self.editor_table_bit_depth_variants.setItem(i, 1, QTableWidgetItem(pattern)) + self.editor_list_extra_patterns.clear() + self.editor_list_extra_patterns.addItems(preset_data.get("move_to_extra_patterns", [])) + + self.editor_table_map_type_mapping.setRowCount(0) # Clear before populating + map_mappings = preset_data.get("map_type_mapping", []) + for i, mapping_dict in enumerate(map_mappings): + if isinstance(mapping_dict, dict) and "target_type" in mapping_dict and "keywords" in mapping_dict: + std_type = mapping_dict["target_type"] + keywords = mapping_dict["keywords"] + self.editor_table_map_type_mapping.insertRow(i) + + # Column 0: Standard Type (QComboBox) + combo_box = QComboBox() + if self._ftd_keys: + combo_box.addItems(self._ftd_keys) + if std_type in self._ftd_keys: + combo_box.setCurrentText(std_type) + else: + log.warning(f"Preset '{preset_data.get('preset_name', 'Unknown')}': target_type '{std_type}' not found in FILE_TYPE_DEFINITIONS. Selecting first available.") + if self._ftd_keys: combo_box.setCurrentIndex(0) + else: + log.warning("FILE_TYPE_DEFINITIONS keys not available for ComboBox in map_type_mapping during population.") + + combo_box.currentIndexChanged.connect(self._mark_editor_unsaved) + self.editor_table_map_type_mapping.setCellWidget(i, 0, combo_box) + + # Column 1: Input Keywords (QTableWidgetItem) + keywords_str = [str(k) for k in keywords if isinstance(k, str)] + self.editor_table_map_type_mapping.setItem(i, 1, QTableWidgetItem(", ".join(keywords_str))) + else: + log.warning(f"Skipping invalid map_type_mapping item during editor population: {mapping_dict}") + + category_rules = preset_data.get("asset_category_rules", {}) + self.editor_list_model_patterns.clear() + self.editor_list_model_patterns.addItems(category_rules.get("model_patterns", [])) + self.editor_list_decal_keywords.clear() + self.editor_list_decal_keywords.addItems(category_rules.get("decal_keywords", [])) + # Archetype rules population (assuming table exists) + self.editor_table_archetype_rules.setRowCount(0) + arch_rules_data = preset_data.get("archetype_rules", []) + for i, rule_entry in enumerate(arch_rules_data): + # Handle both list and dict format for backward compatibility? Assuming list for now. + if isinstance(rule_entry, (list, tuple)) and len(rule_entry) == 2: + name, conditions = rule_entry + if isinstance(conditions, dict): + match_any = conditions.get("match_any", []) + match_all = conditions.get("match_all", []) + self.editor_table_archetype_rules.insertRow(i) + self.editor_table_archetype_rules.setItem(i, 0, QTableWidgetItem(str(name))) + self.editor_table_archetype_rules.setItem(i, 1, QTableWidgetItem(", ".join(map(str, match_any)))) + self.editor_table_archetype_rules.setItem(i, 2, QTableWidgetItem(", ".join(map(str, match_all)))) + else: + log.warning(f"Skipping invalid archetype rule condition format: {conditions}") + else: + log.warning(f"Skipping invalid archetype rule format: {rule_entry}") + + finally: + self._is_loading_editor = False + + def _load_preset_for_editing(self, file_path: Path): + """Loads the content of the selected preset file into the editor widgets.""" + if not file_path or not file_path.is_file(): + self._clear_editor() + return + log.info(f"Loading preset into editor: {file_path.name}") + try: + with open(file_path, 'r', encoding='utf-8') as f: preset_data = json.load(f) + self._populate_editor_from_data(preset_data) + self._set_editor_enabled(True) + self.current_editing_preset_path = file_path + self.editor_unsaved_changes = False + self.editor_save_button.setEnabled(False) + log.info(f"Preset '{file_path.name}' loaded into editor.") + except json.JSONDecodeError as json_err: + log.error(f"Invalid JSON in {file_path.name}: {json_err}") + QMessageBox.warning(self, "Load Error", f"Failed to load preset '{file_path.name}'.\nInvalid JSON structure:\n{json_err}") + self._clear_editor() + except Exception as e: + log.exception(f"Error loading preset file {file_path}: {e}") + QMessageBox.critical(self, "Error", f"Could not load preset file:\n{file_path}\n\nError: {e}") + self._clear_editor() + + @Slot(QListWidgetItem, QListWidgetItem) + def _load_selected_preset_for_editing(self, current_item: QListWidgetItem, previous_item: QListWidgetItem): + """Loads the preset currently selected in the editor list and emits selection change signal.""" + log.debug(f"PresetEditor: currentItemChanged signal triggered. current: {current_item.text() if current_item else 'None'}") + + mode = "placeholder" + preset_name = None + + # Check for unsaved changes before proceeding + if self.check_unsaved_changes(): + # If user cancels, revert selection + if previous_item: + log.debug("Unsaved changes check cancelled. Reverting selection.") + self.editor_preset_list.blockSignals(True) + self.editor_preset_list.setCurrentItem(previous_item) + self.editor_preset_list.blockSignals(False) + return # Stop processing + + # Determine mode and preset name based on selection + if current_item: + item_data = current_item.data(Qt.ItemDataRole.UserRole) + if item_data == "__PLACEHOLDER__": + log.debug("Placeholder item selected.") + self._clear_editor() + self._set_editor_enabled(False) + mode = "placeholder" + self._last_valid_preset_name = None # Clear last valid name + elif item_data == "__LLM__": + log.debug("LLM Interpretation item selected.") + self._clear_editor() + self._set_editor_enabled(False) + mode = "llm" + # Keep _last_valid_preset_name as it was + elif isinstance(item_data, Path): + log.debug(f"Loading preset for editing: {current_item.text()}") + preset_path = item_data + self._load_preset_for_editing(preset_path) + self._last_valid_preset_name = preset_path.stem + mode = "preset" + preset_name = self._last_valid_preset_name + else: + log.error(f"Invalid data type for preset path: {type(item_data)}. Clearing editor.") + self._clear_editor() + self._set_editor_enabled(False) + mode = "placeholder" # Treat as placeholder on error + self._last_valid_preset_name = None + else: + log.debug("No preset selected. Clearing editor.") + self._clear_editor() + self._set_editor_enabled(False) + mode = "placeholder" + self._last_valid_preset_name = None + + # Emit the signal regardless of what was selected + log.debug(f"Emitting preset_selection_changed_signal: mode='{mode}', preset_name='{preset_name}'") + self.preset_selection_changed_signal.emit(mode, preset_name) + + def _gather_editor_data(self) -> dict: + """Gathers data from all editor UI widgets and returns a dictionary.""" + preset_data = {} + preset_data["preset_name"] = self.editor_preset_name.text().strip() + preset_data["supplier_name"] = self.editor_supplier_name.text().strip() + preset_data["notes"] = self.editor_notes.toPlainText().strip() + naming_data = {} + naming_data["separator"] = self.editor_separator.text() + naming_data["part_indices"] = { "base_name": self.editor_spin_base_name_idx.value(), "map_type": self.editor_spin_map_type_idx.value() } + naming_data["glossiness_keywords"] = [self.editor_list_gloss_keywords.item(i).text() for i in range(self.editor_list_gloss_keywords.count())] + naming_data["bit_depth_variants"] = {self.editor_table_bit_depth_variants.item(r, 0).text(): self.editor_table_bit_depth_variants.item(r, 1).text() + for r in range(self.editor_table_bit_depth_variants.rowCount()) if self.editor_table_bit_depth_variants.item(r, 0) and self.editor_table_bit_depth_variants.item(r, 1)} + preset_data["source_naming"] = naming_data + preset_data["move_to_extra_patterns"] = [self.editor_list_extra_patterns.item(i).text() for i in range(self.editor_list_extra_patterns.count())] + + map_mappings = [] + for r in range(self.editor_table_map_type_mapping.rowCount()): + target_type_widget = self.editor_table_map_type_mapping.cellWidget(r, 0) + keywords_item = self.editor_table_map_type_mapping.item(r, 1) + + target_type = "" + if isinstance(target_type_widget, QComboBox): + target_type = target_type_widget.currentText() + elif self.editor_table_map_type_mapping.item(r, 0): # Fallback if item is not a widget + target_type_item = self.editor_table_map_type_mapping.item(r, 0) + if target_type_item: + target_type = target_type_item.text().strip() + + if target_type and keywords_item and keywords_item.text(): + keywords = [k.strip() for k in keywords_item.text().split(',') if k.strip()] + if keywords: # Ensure keywords list is not empty after stripping + map_mappings.append({"target_type": target_type, "keywords": keywords}) + else: + log.warning(f"Skipping row {r} in map type mapping table due to empty keywords after processing for target_type '{target_type}'.") + else: + # Log if target_type is empty or keywords_item is problematic + if not target_type: + log.warning(f"Skipping row {r} in map type mapping table due to empty target_type.") + if not (keywords_item and keywords_item.text()): + log.warning(f"Skipping row {r} in map type mapping table for target_type '{target_type}' due to missing or empty keywords item.") + preset_data["map_type_mapping"] = map_mappings + + category_rules = {} + category_rules["model_patterns"] = [self.editor_list_model_patterns.item(i).text() for i in range(self.editor_list_model_patterns.count())] + category_rules["decal_keywords"] = [self.editor_list_decal_keywords.item(i).text() for i in range(self.editor_list_decal_keywords.count())] + preset_data["asset_category_rules"] = category_rules + arch_rules = [] + for r in range(self.editor_table_archetype_rules.rowCount()): + name_item = self.editor_table_archetype_rules.item(r, 0) + any_item = self.editor_table_archetype_rules.item(r, 1) + all_item = self.editor_table_archetype_rules.item(r, 2) + if name_item and name_item.text() and any_item and all_item: # Check name has text + match_any = [k.strip() for k in any_item.text().split(',') if k.strip()] + match_all = [k.strip() for k in all_item.text().split(',') if k.strip()] + # Only add if name is present and at least one condition list is non-empty? Or allow empty conditions? + # Let's allow empty conditions for now. + arch_rules.append([name_item.text().strip(), {"match_any": match_any, "match_all": match_all}]) + else: + log.warning(f"Skipping row {r} in archetype rules table due to missing items or empty name.") + preset_data["archetype_rules"] = arch_rules + return preset_data + + def _save_current_preset(self) -> bool: + """Saves the current editor content to the currently loaded file path.""" + if not self.current_editing_preset_path: return self._save_preset_as() + log.info(f"Saving preset: {self.current_editing_preset_path.name}") + try: + preset_data = self._gather_editor_data() + if not preset_data.get("preset_name"): QMessageBox.warning(self, "Save Error", "Preset Name cannot be empty."); return False + if not preset_data.get("supplier_name"): QMessageBox.warning(self, "Save Error", "Supplier Name cannot be empty."); return False + content_to_save = json.dumps(preset_data, indent=4, ensure_ascii=False) + with open(self.current_editing_preset_path, 'w', encoding='utf-8') as f: f.write(content_to_save) + self.editor_unsaved_changes = False + self.editor_save_button.setEnabled(False) + self.presets_changed_signal.emit() + log.info("Preset saved successfully.") + self.populate_presets() + # Reselect the saved item + items = self.editor_preset_list.findItems(self.current_editing_preset_path.stem, Qt.MatchFlag.MatchExactly) + if items: self.editor_preset_list.setCurrentItem(items[0]) + return True + except Exception as e: + log.exception(f"Error saving preset file {self.current_editing_preset_path}: {e}") + QMessageBox.critical(self, "Save Error", f"Could not save preset file:\n{self.current_editing_preset_path}\n\nError: {e}") + return False + + def _save_preset_as(self) -> bool: + """Saves the current editor content to a new file chosen by the user.""" + log.debug("Save As action triggered.") + try: + preset_data = self._gather_editor_data() + new_preset_name = preset_data.get("preset_name") + if not new_preset_name: QMessageBox.warning(self, "Save As Error", "Preset Name cannot be empty."); return False + if not preset_data.get("supplier_name"): QMessageBox.warning(self, "Save As Error", "Supplier Name cannot be empty."); return False + content_to_save = json.dumps(preset_data, indent=4, ensure_ascii=False) + suggested_name = f"{new_preset_name}.json" + default_path = PRESETS_DIR / suggested_name + file_path_str, _ = QFileDialog.getSaveFileName(self, "Save Preset As", str(default_path), "JSON Files (*.json);;All Files (*)") + if not file_path_str: log.debug("Save As cancelled by user."); return False + save_path = Path(file_path_str) + if save_path.suffix.lower() != ".json": save_path = save_path.with_suffix(".json") + if save_path.exists() and save_path != self.current_editing_preset_path: + reply = QMessageBox.warning(self, "Confirm Overwrite", f"Preset '{save_path.name}' already exists. Overwrite?", QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, QMessageBox.StandardButton.No) + if reply == QMessageBox.StandardButton.No: log.debug("Save As overwrite cancelled."); return False + log.info(f"Saving preset as: {save_path.name}") + with open(save_path, 'w', encoding='utf-8') as f: f.write(content_to_save) + self.current_editing_preset_path = save_path + self.editor_unsaved_changes = False + self.editor_save_button.setEnabled(False) + self.presets_changed_signal.emit() + log.info("Preset saved successfully (Save As).") + # Refresh list and select the new item + self.populate_presets() + items = self.editor_preset_list.findItems(save_path.stem, Qt.MatchFlag.MatchExactly) + if items: self.editor_preset_list.setCurrentItem(items[0]) + return True + except Exception as e: + log.exception(f"Error saving preset file (Save As): {e}") + QMessageBox.critical(self, "Save Error", f"Could not save preset file.\n\nError: {e}") + return False + + def _new_preset(self): + """Clears the editor and loads data from _template.json.""" + log.debug("New Preset action triggered.") + if self.check_unsaved_changes(): return # Check unsaved changes first + self._clear_editor() + if TEMPLATE_PATH.is_file(): + log.info("Loading new preset from _template.json") + try: + with open(TEMPLATE_PATH, 'r', encoding='utf-8') as f: template_data = json.load(f) + self._populate_editor_from_data(template_data) + # Override specific fields for a new preset + self.editor_preset_name.setText("NewPreset") + except Exception as e: + log.exception(f"Error loading template preset file {TEMPLATE_PATH}: {e}") + QMessageBox.critical(self, "Error", f"Could not load template preset file:\n{TEMPLATE_PATH}\n\nError: {e}") + self._clear_editor() + self.editor_supplier_name.setText("MySupplier") + else: + log.warning("Presets/_template.json not found. Creating empty preset.") + self.editor_preset_name.setText("NewPreset") + self.editor_supplier_name.setText("MySupplier") + self._set_editor_enabled(True) + self.editor_unsaved_changes = True + self.editor_save_button.setEnabled(True) + # Select the placeholder item to avoid auto-loading the "NewPreset" + placeholder_item = self.editor_preset_list.findItems("--- Select a Preset ---", Qt.MatchFlag.MatchExactly) + if placeholder_item: + self.editor_preset_list.setCurrentItem(placeholder_item[0]) + # Emit selection change for the new state (effectively placeholder) + self.preset_selection_changed_signal.emit("placeholder", None) + + + def _delete_selected_preset(self): + """Deletes the currently selected preset file from the editor list after confirmation.""" + current_item = self.editor_preset_list.currentItem() + if not current_item: QMessageBox.information(self, "Delete Preset", "Please select a preset from the list to delete."); return + + item_data = current_item.data(Qt.ItemDataRole.UserRole) + # Ensure it's a real preset path before attempting delete + if not isinstance(item_data, Path): + QMessageBox.information(self, "Delete Preset", "Cannot delete placeholder or LLM option.") + return + + preset_path = item_data + preset_name = preset_path.stem + reply = QMessageBox.warning(self, "Confirm Delete", f"Are you sure you want to permanently delete the preset '{preset_name}'?", QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, QMessageBox.StandardButton.No) + if reply == QMessageBox.StandardButton.Yes: + log.info(f"Deleting preset: {preset_path.name}") + try: + preset_path.unlink() + log.info("Preset deleted successfully.") + if self.current_editing_preset_path == preset_path: self._clear_editor() + self.presets_changed_signal.emit() + self.populate_presets() + except Exception as e: + log.exception(f"Error deleting preset file {preset_path}: {e}") + QMessageBox.critical(self, "Delete Error", f"Could not delete preset file:\n{preset_path}\n\nError: {e}") + + # --- Public Access Methods for MainWindow --- + + def get_selected_preset_mode(self) -> tuple[str, str | None]: + """ + Returns the current selection mode and preset name (if applicable). + Returns: tuple(mode_string, preset_name_string_or_None) + mode_string can be "preset", "llm", "placeholder" + """ + current_item = self.editor_preset_list.currentItem() + if current_item: + item_data = current_item.data(Qt.ItemDataRole.UserRole) + if item_data == "__PLACEHOLDER__": + return "placeholder", None + elif item_data == "__LLM__": + return "llm", None + elif isinstance(item_data, Path): + return "preset", item_data.stem + return "placeholder", None # Default or if no item selected + + def get_last_valid_preset_name(self) -> str | None: + """ + Returns the name (stem) of the last valid preset that was loaded. + Used by delegates to populate dropdowns based on the original context. + """ + return self._last_valid_preset_name + + # --- Slots for MainWindow Interaction --- diff --git a/gui/preview_table_model.py b/gui/preview_table_model.py index 53958ec..47ec316 100644 --- a/gui/preview_table_model.py +++ b/gui/preview_table_model.py @@ -1,15 +1,14 @@ -import logging # Import logging +import logging import time # For logging timestamps -from PySide6.QtCore import QAbstractTableModel, Qt, QModelIndex, QSortFilterProxyModel, QThread # Import QThread +from PySide6.QtCore import QAbstractTableModel, Qt, QModelIndex, QSortFilterProxyModel, QThread from PySide6.QtGui import QColor -log = logging.getLogger(__name__) # Get logger +log = logging.getLogger(__name__) # Define colors for alternating asset groups COLOR_ASSET_GROUP_1 = QColor("#292929") # Dark grey 1 COLOR_ASSET_GROUP_2 = QColor("#343434") # Dark grey 2 -# Define text colors for statuses class PreviewTableModel(QAbstractTableModel): """ Custom table model for the GUI preview table. @@ -54,11 +53,11 @@ class PreviewTableModel(QAbstractTableModel): self._headers_detailed = ["Status", "Predicted Asset", "Original Path", "Predicted Output", "Details", "Additional Files"] # Added new column header self._sorted_unique_assets = [] # Store sorted unique asset names for coloring self._headers_simple = ["Input Path"] - self.set_data(data or []) # Initialize data and simple_data + self.set_data(data or []) def set_simple_mode(self, enabled: bool): """Toggles the model between detailed and simple view modes.""" - thread_id = QThread.currentThread() # Get current thread object + thread_id = QThread.currentThread() log.info(f"[{time.time():.4f}][T:{thread_id}] --> Entered PreviewTableModel.set_simple_mode(enabled={enabled}). Current mode: {self._simple_mode}") if self._simple_mode != enabled: log.info(f"[{time.time():.4f}][T:{thread_id}] Calling beginResetModel()...") @@ -78,7 +77,6 @@ class PreviewTableModel(QAbstractTableModel): if parent.isValid(): return 0 row_count = len(self._simple_data) if self._simple_mode else len(self._table_rows) # Use _table_rows for detailed mode - # log.debug(f"PreviewTableModel.rowCount called. Mode: {self._simple_mode}, Row Count: {row_count}") return row_count def columnCount(self, parent=QModelIndex()): @@ -86,7 +84,6 @@ class PreviewTableModel(QAbstractTableModel): if parent.isValid(): return 0 col_count = len(self._headers_simple) if self._simple_mode else len(self._headers_detailed) # Use updated headers_detailed - # log.debug(f"PreviewTableModel.columnCount called. Mode: {self._simple_mode}, Column Count: {col_count}") return col_count def data(self, index: QModelIndex, role: int = Qt.ItemDataRole.DisplayRole): @@ -100,8 +97,7 @@ class PreviewTableModel(QAbstractTableModel): # --- Simple Mode --- if self._simple_mode: if row >= len(self._simple_data): - # log.warning(f"data called with out of bounds row in simple mode: {row}/{len(self._simple_data)}") - return None # Bounds check + return None source_asset_path = self._simple_data[row] if role == Qt.ItemDataRole.DisplayRole: if col == self.COL_SIMPLE_PATH: @@ -113,12 +109,10 @@ class PreviewTableModel(QAbstractTableModel): # --- Detailed Mode --- if row >= len(self._table_rows): # Use _table_rows - # log.warning(f"data called with out of bounds row in detailed mode: {row}/{len(self._table_rows)}") - return None # Bounds check + return None row_data = self._table_rows[row] # Get data from the structured row # --- Handle Custom Internal Roles --- - # These roles are now handled by the proxy model based on the structured data if role == self.ROLE_RAW_STATUS: # Return status of the main file if it exists, otherwise a placeholder for additional rows main_file = row_data.get('main_file') @@ -132,7 +126,7 @@ class PreviewTableModel(QAbstractTableModel): main_file = row_data.get('main_file') if main_file: raw_status = main_file.get('status', '[No Status]') - details = main_file.get('details', '') # Get details for parsing + details = main_file.get('details', '') # Implement status text simplification if raw_status == "Unmatched Extra": @@ -268,14 +262,6 @@ class PreviewTableModel(QAbstractTableModel): return None - # --- Handle Text Alignment Role --- - if role == Qt.ItemDataRole.TextAlignmentRole: - if col == self.COL_ORIGINAL_PATH: - return int(Qt.AlignmentFlag.AlignLeft | Qt.AlignmentFlag.AlignVCenter) - elif col == self.COL_ADDITIONAL_FILES: - return int(Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter) - # For other columns, return default alignment (or None) - return None return None @@ -291,7 +277,7 @@ class PreviewTableModel(QAbstractTableModel): def set_data(self, data: list): """Sets the model's data, extracts simple data, and emits signals.""" # Removed diagnostic import here - thread_id = QThread.currentThread() # Get current thread object + thread_id = QThread.currentThread() log.info(f"[{time.time():.4f}][T:{thread_id}] --> Entered PreviewTableModel.set_data. Received {len(data)} items.") log.info(f"[{time.time():.4f}][T:{thread_id}] Calling beginResetModel()...") self.beginResetModel() @@ -357,7 +343,7 @@ class PreviewTableModel(QAbstractTableModel): def clear_data(self): """Clears the model's data.""" - thread_id = QThread.currentThread() # Get current thread object + thread_id = QThread.currentThread() log.info(f"[{time.time():.4f}][T:{thread_id}] PreviewTableModel.clear_data called.") self.set_data([]) @@ -398,21 +384,18 @@ class PreviewSortFilterProxyModel(QSortFilterProxyModel): """ model = self.sourceModel() if not model: - # log.debug("ProxyModel.lessThan: No source model.") return super().lessThan(left, right) # Fallback if no source model # If in simple mode, sort by the simple path column if isinstance(model, PreviewTableModel) and model._simple_mode: left_path = model.data(left.siblingAtColumn(model.COL_SIMPLE_PATH), Qt.ItemDataRole.DisplayRole) right_path = model.data(right.siblingAtColumn(model.COL_SIMPLE_PATH), Qt.ItemDataRole.DisplayRole) - # log.debug(f"ProxyModel.lessThan (Simple Mode): Comparing '{left_path}' < '{right_path}'") if not left_path: return True if not right_path: return False return left_path < right_path # --- Detailed Mode Sorting --- - # log.debug("ProxyModel.lessThan (Detailed Mode).") # Get the full row data from the source model's _table_rows left_row_data = model._table_rows[left.row()] right_row_data = model._table_rows[right.row()] diff --git a/gui/processing_handler.py b/gui/processing_handler.py deleted file mode 100644 index 0d0d75c..0000000 --- a/gui/processing_handler.py +++ /dev/null @@ -1,345 +0,0 @@ -# gui/processing_handler.py -import logging -from pathlib import Path -from concurrent.futures import ProcessPoolExecutor, as_completed -import time # For potential delays if needed - -import subprocess # <<< ADDED IMPORT -import shutil # <<< ADDED IMPORT -from typing import Optional # <<< ADDED IMPORT - -# --- PySide6 Imports --- -# Inherit from QObject to support signals/slots for thread communication -from PySide6.QtCore import QObject, Signal - -# --- Backend Imports --- -# Need to import the worker function and potentially config/processor if needed directly -# Adjust path to ensure modules can be found relative to this file's location -import sys -script_dir = Path(__file__).parent -project_root = script_dir.parent -if str(project_root) not in sys.path: - sys.path.insert(0, str(project_root)) - -try: - # Import the worker function from main.py - from main import process_single_asset_wrapper - # Import exceptions if needed for type hinting or specific handling - from configuration import ConfigurationError - from asset_processor import AssetProcessingError - import config as core_config # <<< ADDED IMPORT - BACKEND_AVAILABLE = True -except ImportError as e: - print(f"ERROR (ProcessingHandler): Failed to import backend modules/worker: {e}") - # Define placeholders if imports fail, so the GUI doesn't crash immediately - process_single_asset_wrapper = None - ConfigurationError = Exception - AssetProcessingError = Exception - BACKEND_AVAILABLE = False - -log = logging.getLogger(__name__) -# Basic config if logger hasn't been set up elsewhere -if not log.hasHandlers(): - logging.basicConfig(level=logging.INFO, format='%(levelname)s (Handler): %(message)s') - - -class ProcessingHandler(QObject): - """ - Handles the execution of the asset processing pipeline in a way that - can be run in a separate thread and communicate progress via signals. - """ - # --- Signals --- - # Emitted for overall progress bar update - progress_updated = Signal(int, int) # current_count, total_count - # Emitted for updating status of individual files in the list - file_status_updated = Signal(str, str, str) # input_path_str, status ("processing", "processed", "skipped", "failed"), message - # Emitted when the entire batch processing is finished - processing_finished = Signal(int, int, int) # processed_count, skipped_count, failed_count - # Emitted for general status messages to the status bar - status_message = Signal(str, int) # message, timeout_ms - - def __init__(self, parent=None): - super().__init__(parent) - self._executor = None - self._futures = {} # Store future->input_path mapping - self._is_running = False - self._cancel_requested = False - - @property - def is_running(self): - return self._is_running - - def run_processing(self, input_paths: list[str], preset_name: str, output_dir_str: str, overwrite: bool, num_workers: int, - run_blender: bool, nodegroup_blend_path: str, materials_blend_path: str, verbose: bool): # <<< ADDED verbose PARAM - """ - Starts the asset processing task and optionally runs Blender scripts afterwards. - This method should be called when the handler is moved to a separate thread. - """ - if self._is_running: - log.warning("Processing is already running.") - self.status_message.emit("Processing already in progress.", 3000) - return - - if not BACKEND_AVAILABLE or not process_single_asset_wrapper: - log.error("Backend modules or worker function not available. Cannot start processing.") - self.status_message.emit("Error: Backend components missing. Cannot process.", 5000) - self.processing_finished.emit(0, 0, len(input_paths)) # Emit finished with all failed - return - - self._is_running = True - self._cancel_requested = False - self._futures = {} # Reset futures - total_files = len(input_paths) - processed_count = 0 - skipped_count = 0 - failed_count = 0 - completed_count = 0 - - log.info(f"Starting processing run: {total_files} assets, Preset='{preset_name}', Workers={num_workers}, Overwrite={overwrite}") - self.status_message.emit(f"Starting processing for {total_files} items...", 0) # Persistent message - - try: - # Use 'with' statement for ProcessPoolExecutor for cleanup - with ProcessPoolExecutor(max_workers=num_workers) as executor: - self._executor = executor # Store for potential cancellation - - # Submit tasks - for input_path in input_paths: - if self._cancel_requested: break # Check before submitting more - log.debug(f"Submitting task for: {input_path}") - future = executor.submit(process_single_asset_wrapper, input_path, preset_name, output_dir_str, overwrite, verbose=verbose) # Pass verbose flag from GUI - self._futures[future] = input_path # Map future back to input path - # Optionally emit "processing" status here - self.file_status_updated.emit(input_path, "processing", "") - - if self._cancel_requested: - log.info("Processing cancelled during task submission.") - # Count remaining unsubmitted tasks as failed/cancelled - failed_count = total_files - len(self._futures) - - # Process completed futures - for future in as_completed(self._futures): - completed_count += 1 - input_path = self._futures[future] # Get original path - asset_name = Path(input_path).name - status = "failed" # Default status - error_message = "Unknown error" - - if self._cancel_requested: - # If cancelled after submission, try to get result but count as failed - status = "failed" - error_message = "Cancelled" - failed_count += 1 - # Don't try future.result() if cancelled, it might raise CancelledError - else: - try: - # Get result tuple: (input_path_str, status_string, error_message_or_None) - result_tuple = future.result() - _, status, error_message = result_tuple - error_message = error_message or "" # Ensure it's a string - - # Increment counters based on status - if status == "processed": - processed_count += 1 - elif status == "skipped": - skipped_count += 1 - elif status == "failed": - failed_count += 1 - else: - log.warning(f"Unknown status '{status}' received for {asset_name}. Counting as failed.") - failed_count += 1 - error_message = f"Unknown status: {status}" - - except Exception as e: - # Catch errors if the future itself fails (e.g., worker process crashed hard) - log.exception(f"Critical worker failure for {asset_name}: {e}") - failed_count += 1 # Count crashes as failures - status = "failed" - error_message = f"Worker process crashed: {e}" - - # Emit progress signals - self.progress_updated.emit(completed_count, total_files) - self.file_status_updated.emit(input_path, status, error_message) - - # Check for cancellation again after processing each result - if self._cancel_requested: - log.info("Cancellation detected after processing a result.") - # Count remaining unprocessed futures as failed/cancelled - remaining_futures = total_files - completed_count - failed_count += remaining_futures - break # Exit the as_completed loop - - except Exception as pool_exc: - log.exception(f"An error occurred with the process pool: {pool_exc}") - self.status_message.emit(f"Error during processing: {pool_exc}", 5000) - # Mark all remaining as failed - failed_count = total_files - processed_count - skipped_count - - finally: - # --- Blender Script Execution (Optional) --- - if run_blender and not self._cancel_requested: - log.info("Asset processing complete. Checking for Blender script execution.") - self.status_message.emit("Asset processing complete. Starting Blender scripts...", 0) - blender_exe = self._find_blender_executable() - if blender_exe: - script_dir = Path(__file__).parent.parent / "blenderscripts" # Go up one level from gui/ - nodegroup_script_path = script_dir / "create_nodegroups.py" - materials_script_path = script_dir / "create_materials.py" - asset_output_root = output_dir_str # Use the same output dir - - # Run Nodegroup Script - if nodegroup_blend_path and Path(nodegroup_blend_path).is_file(): - if nodegroup_script_path.is_file(): - log.info("-" * 20 + " Running Nodegroup Script " + "-" * 20) - self.status_message.emit(f"Running Blender nodegroup script on {Path(nodegroup_blend_path).name}...", 0) - success_ng = self._run_blender_script_subprocess( - blender_exe_path=blender_exe, - blend_file_path=nodegroup_blend_path, - python_script_path=str(nodegroup_script_path), - asset_root_dir=asset_output_root - ) - if not success_ng: - log.error("Blender node group script execution failed.") - self.status_message.emit("Blender nodegroup script failed.", 5000) - else: - log.info("Blender nodegroup script finished successfully.") - self.status_message.emit("Blender nodegroup script finished.", 3000) - else: - log.error(f"Node group script not found: {nodegroup_script_path}") - self.status_message.emit(f"Error: Nodegroup script not found.", 5000) - elif run_blender and nodegroup_blend_path: # Log if path was provided but invalid - log.warning(f"Nodegroup blend path provided but invalid: {nodegroup_blend_path}") - self.status_message.emit(f"Warning: Invalid Nodegroup .blend path.", 5000) - - - # Run Materials Script (only if nodegroup script was attempted or not needed) - if materials_blend_path and Path(materials_blend_path).is_file(): - if materials_script_path.is_file(): - log.info("-" * 20 + " Running Materials Script " + "-" * 20) - self.status_message.emit(f"Running Blender materials script on {Path(materials_blend_path).name}...", 0) - # Pass the nodegroup blend path as the second argument to the script - success_mat = self._run_blender_script_subprocess( - blender_exe_path=blender_exe, - blend_file_path=materials_blend_path, - python_script_path=str(materials_script_path), - asset_root_dir=asset_output_root, - nodegroup_blend_file_path_arg=nodegroup_blend_path # Pass the nodegroup path - ) - if not success_mat: - log.error("Blender material script execution failed.") - self.status_message.emit("Blender material script failed.", 5000) - else: - log.info("Blender material script finished successfully.") - self.status_message.emit("Blender material script finished.", 3000) - else: - log.error(f"Material script not found: {materials_script_path}") - self.status_message.emit(f"Error: Material script not found.", 5000) - elif run_blender and materials_blend_path: # Log if path was provided but invalid - log.warning(f"Materials blend path provided but invalid: {materials_blend_path}") - self.status_message.emit(f"Warning: Invalid Materials .blend path.", 5000) - - else: - log.warning("Blender executable not found. Skipping Blender script execution.") - self.status_message.emit("Warning: Blender executable not found. Skipping scripts.", 5000) - elif self._cancel_requested: - log.info("Processing was cancelled. Skipping Blender script execution.") - # --- End Blender Script Execution --- - - final_message = f"Finished. Processed: {processed_count}, Skipped: {skipped_count}, Failed: {failed_count}" - log.info(final_message) - self.status_message.emit(final_message, 5000) # Show final summary - self.processing_finished.emit(processed_count, skipped_count, failed_count) - self._is_running = False - self._executor = None - self._futures = {} # Clear futures - - def request_cancel(self): - """Requests cancellation of the ongoing processing task.""" - if not self._is_running: - log.warning("Cancel requested but no processing is running.") - return - - if self._cancel_requested: - log.warning("Cancellation already requested.") - return - - log.info("Cancellation requested.") - self.status_message.emit("Cancellation requested...", 3000) - self._cancel_requested = True - - # Attempt to shutdown the executor - this might cancel pending tasks - # but won't forcefully stop running ones. `cancel_futures=True` is Python 3.9+ - if self._executor: - log.debug("Requesting executor shutdown...") - # For Python 3.9+: self._executor.shutdown(wait=False, cancel_futures=True) - # For older Python: - self._executor.shutdown(wait=False) - # Manually try cancelling futures that haven't started - for future in self._futures: - if not future.running() and not future.done(): - future.cancel() - log.debug("Executor shutdown requested.") - - # Note: True cancellation of running ProcessPoolExecutor tasks is complex. - # This implementation primarily prevents processing further results and - # attempts to cancel pending/unstarted tasks. - - def _find_blender_executable(self) -> Optional[str]: - """Finds the Blender executable path from config or system PATH.""" - try: - blender_exe_config = getattr(core_config, 'BLENDER_EXECUTABLE_PATH', None) - if blender_exe_config: - p = Path(blender_exe_config) - if p.is_file(): - log.info(f"Using Blender executable from config: {p}") - return str(p.resolve()) - else: - log.warning(f"Blender path in config not found: '{blender_exe_config}'. Trying PATH.") - else: - log.info("BLENDER_EXECUTABLE_PATH not set in config. Trying PATH.") - - blender_exe = shutil.which("blender") - if blender_exe: - log.info(f"Found Blender executable in PATH: {blender_exe}") - return blender_exe - else: - log.warning("Could not find 'blender' in system PATH.") - return None - except Exception as e: - log.error(f"Error checking Blender executable path: {e}") - return None - - def _run_blender_script_subprocess(self, blender_exe_path: str, blend_file_path: str, python_script_path: str, asset_root_dir: str, nodegroup_blend_file_path_arg: Optional[str] = None) -> bool: - """Internal helper to run a single Blender script via subprocess.""" - command_base = [ - blender_exe_path, - "--factory-startup", - "-b", - blend_file_path, - "--log", "*", # <<< ADDED BLENDER LOGGING FLAG - "--python", python_script_path, - "--", - asset_root_dir, - ] - # Add nodegroup blend file path if provided (for create_materials script) - if nodegroup_blend_file_path_arg: - command = command_base + [nodegroup_blend_file_path_arg] - else: - command = command_base - log.debug(f"Executing Blender command: {' '.join(map(str, command))}") # Ensure all parts are strings for join - try: - # Ensure all parts of the command are strings for subprocess - str_command = [str(part) for part in command] - result = subprocess.run(str_command, capture_output=True, text=True, check=False, encoding='utf-8') # Specify encoding - log.info(f"Blender script '{Path(python_script_path).name}' finished with exit code: {result.returncode}") - if result.stdout: log.debug(f"Blender stdout:\n{result.stdout.strip()}") - if result.stderr: - if result.returncode != 0: log.error(f"Blender stderr:\n{result.stderr.strip()}") - else: log.warning(f"Blender stderr (RC=0):\n{result.stderr.strip()}") - return result.returncode == 0 - except FileNotFoundError: - log.error(f"Blender executable not found at: {blender_exe_path}") - return False - except Exception as e: - log.exception(f"Error running Blender script '{Path(python_script_path).name}': {e}") - return False \ No newline at end of file diff --git a/gui/rule_editor_widget.py b/gui/rule_editor_widget.py new file mode 100644 index 0000000..6596f26 --- /dev/null +++ b/gui/rule_editor_widget.py @@ -0,0 +1,204 @@ +import sys +from PySide6.QtWidgets import (QApplication, QWidget, QVBoxLayout, QLabel, QLineEdit, + QFormLayout, QComboBox, QCheckBox, QSpinBox, QDoubleSpinBox) +from PySide6.QtCore import Signal, Slot, QObject + + +class RuleEditorWidget(QWidget): + """ + A widget to display and edit hierarchical processing rules (Source, Asset, File). + """ + rule_updated = Signal(object) + + def __init__(self, asset_types: list[str] | None = None, file_types: list[str] | None = None, parent=None): + """ + Initializes the RuleEditorWidget. + + Args: + asset_types (list[str] | None): A list of available asset type names. Defaults to None. + file_types (list[str] | None): A list of available file type names (keys from FILE_TYPE_DEFINITIONS). Defaults to None. + parent: The parent widget. + """ + super().__init__(parent) + self.asset_types = asset_types if asset_types else [] + self.file_types = file_types if file_types else [] + self.current_rule_type = None + self.current_rule_object = None + + self.layout = QVBoxLayout(self) + self.rule_type_label = QLabel("Select an item in the hierarchy to view/edit rules.") + self.layout.addWidget(self.rule_type_label) + + self.form_layout = QFormLayout() + self.layout.addLayout(self.form_layout) + + self.layout.addStretch() # Add stretch to push content to the top + + self.setLayout(self.layout) + self.clear_editor() + + @Slot(object, str) + def load_rule(self, rule_object, rule_type_name): + """ + Loads a rule object into the editor. + + Args: + rule_object: The SourceRule, AssetRule, or FileRule object. + rule_type_name: The name of the rule type ('SourceRule', 'AssetRule', 'FileRule'). + """ + self.clear_editor() + self.current_rule_object = rule_object + self.current_rule_type = rule_type_name + self.rule_type_label.setText(f"Editing: {rule_type_name}") + + if rule_object: + # Dynamically create form fields based on rule object attributes + for attr_name, attr_value in vars(rule_object).items(): + if attr_name.startswith('_'): # Skip private attributes + continue + + label = QLabel(attr_name.replace('_', ' ').title() + ":") + editor_widget = self._create_editor_widget(attr_name, attr_value) + if editor_widget: + self.form_layout.addRow(label, editor_widget) + self._connect_editor_signal(editor_widget, attr_name) + + def _create_editor_widget(self, attr_name, attr_value): + """ + Creates an appropriate editor widget based on the attribute type. + """ + # --- Special Handling for Asset Type Dropdown --- + if self.current_rule_type == 'AssetRule' and attr_name in ('asset_type', 'asset_type_override') and self.asset_types: + widget = QComboBox() + widget.addItems(self.asset_types) + # Handle None case for override: if None, don't select anything or select a placeholder + if attr_value is None and attr_name == 'asset_type_override': + # Optionally add a placeholder like "" or "" + widget.setCurrentIndex(-1) # No selection or placeholder + elif attr_value in self.asset_types: + widget.setCurrentText(attr_value) + elif self.asset_types: # Select first item if current value is invalid (and not None override) + widget.setCurrentIndex(0) + return widget + # --- Special Handling for FileRule item_type and item_type_override --- + elif self.current_rule_type == 'FileRule' and attr_name in ('item_type', 'item_type_override') and self.file_types: + widget = QComboBox() + widget.addItems(self.file_types) + if attr_value in self.file_types: + widget.setCurrentText(attr_value) + elif self.file_types: # Select first item if current value is invalid + widget.setCurrentIndex(0) + return widget + # --- Standard Type Handling --- + elif isinstance(attr_value, bool): + widget = QCheckBox() + widget.setChecked(attr_value) + return widget + elif isinstance(attr_value, int): + widget = QSpinBox() + widget.setRange(-2147483648, 2147483647) # Default integer range + widget.setValue(attr_value) + return widget + elif isinstance(attr_value, float): + widget = QDoubleSpinBox() + widget.setRange(-sys.float_info.max, sys.float_info.max) # Default float range + widget.setValue(attr_value) + return widget + elif isinstance(attr_value, (str, type(None))): # Handle None for strings + widget = QLineEdit() + widget.setText(str(attr_value) if attr_value is not None else "") + return widget + else: + # For unsupported types, just display the value + label = QLabel(str(attr_value)) + return label + + def _connect_editor_signal(self, editor_widget, attr_name): + """ + Connects the appropriate signal of the editor widget to the update logic. + """ + if isinstance(editor_widget, QLineEdit): + editor_widget.textChanged.connect(lambda text: self._update_rule_attribute(attr_name, text)) + elif isinstance(editor_widget, QCheckBox): + editor_widget.toggled.connect(lambda checked: self._update_rule_attribute(attr_name, checked)) + elif isinstance(editor_widget, QSpinBox): + editor_widget.valueChanged.connect(lambda value: self._update_rule_attribute(attr_name, value)) + elif isinstance(editor_widget, QDoubleSpinBox): + editor_widget.valueChanged.connect(lambda value: self._update_rule_attribute(attr_name, value)) + elif isinstance(editor_widget, QComboBox): + # Use currentTextChanged to get the string value directly + editor_widget.currentTextChanged.connect(lambda text: self._update_rule_attribute(attr_name, text)) + + def _update_rule_attribute(self, attr_name, value): + """ + Updates the attribute of the current rule object and emits the signal. + """ + if self.current_rule_object: + # Basic type conversion based on the original attribute type + original_value = getattr(self.current_rule_object, attr_name) + try: + if isinstance(original_value, bool): + converted_value = bool(value) + elif isinstance(original_value, int): + converted_value = int(value) + elif isinstance(original_value, float): + converted_value = float(value) + elif isinstance(original_value, (str, type(None))): + converted_value = str(value) if value != "" else None # Convert empty string to None for original None types + else: + converted_value = value # Fallback for other types + setattr(self.current_rule_object, attr_name, converted_value) + self.rule_updated.emit(self.current_rule_object) + except ValueError: + # Handle potential conversion errors (e.g., non-numeric input for int/float) + print(f"Error converting value '{value}' for attribute '{attr_name}'") + # Optionally, revert the editor widget to the original value or show an error indicator + + def clear_editor(self): + """ + Clears the form layout. + """ + self.current_rule_object = None + self.current_rule_type = None + self.rule_type_label.setText("Select an item in the hierarchy to view/edit rules.") + while self.form_layout.rowCount() > 0: + self.form_layout.removeRow(0) + +if __name__ == '__main__': + app = QApplication(sys.argv) + + # Placeholder Rule Classes for testing + from dataclasses import dataclass, field + + @dataclass + class SourceRule: + source_setting_1: str = "default_source_string" + source_setting_2: int = 123 + source_setting_3: bool = True + + @dataclass + class AssetRule: + asset_setting_a: float = 4.56 + asset_setting_b: str = None + asset_setting_c: bool = False + + @dataclass + class FileRule: + file_setting_x: int = 789 + file_setting_y: str = "default_file_string" + + # Example usage: Provide asset types during instantiation + asset_types_from_config = ["Surface", "Model", "Decal", "Atlas", "UtilityMap"] + file_types_from_config = ["MAP_COL", "MAP_NRM", "MAP_METAL", "MAP_ROUGH", "MAP_AO", "MAP_DISP", "MAP_REFL", "MAP_SSS", "MAP_FUZZ", "MAP_IDMAP", "MAP_MASK", "MAP_IMPERFECTION", "MODEL", "EXTRA", "FILE_IGNORE"] + editor = RuleEditorWidget(asset_types=asset_types_from_config, file_types=file_types_from_config) + + # Test loading different rule types + source_rule = SourceRule() + asset_rule = AssetRule() + file_rule = FileRule() + + editor.load_rule(source_rule, "SourceRule") + + + editor.show() + sys.exit(app.exec()) \ No newline at end of file diff --git a/gui/rule_hierarchy_model.py b/gui/rule_hierarchy_model.py new file mode 100644 index 0000000..21aec1a --- /dev/null +++ b/gui/rule_hierarchy_model.py @@ -0,0 +1,175 @@ +from PySide6.QtCore import QAbstractItemModel, QModelIndex, Qt, Signal, Slot +from PySide6.QtGui import QIcon # Assuming we might want icons later +from rule_structure import SourceRule, AssetRule, FileRule + +class RuleHierarchyModel(QAbstractItemModel): + """ + A custom model for displaying the hierarchical structure of SourceRule, + AssetRule, and FileRule objects in a QTreeView. + """ + def __init__(self, root_rule: SourceRule = None, parent=None): + super().__init__(parent) + self._root_rule = root_rule + + def set_root_rule(self, root_rule: SourceRule): + """Sets the root SourceRule for the model and resets the model.""" + self.beginResetModel() + self._root_rule = root_rule + self.endResetModel() + + def rowCount(self, parent: QModelIndex = QModelIndex()): + """Returns the number of rows (children) for the given parent index.""" + if not parent.isValid(): + # Root item (SourceRule) + return 1 if self._root_rule else 0 + else: + parent_item = parent.internalPointer() + if isinstance(parent_item, SourceRule): + # Children of SourceRule are AssetRules + return len(parent_item.assets) + elif isinstance(parent_item, AssetRule): + # Children of AssetRule are FileRules + return len(parent_item.files) + elif isinstance(parent_item, FileRule): + # FileRules have no children + return 0 + else: + return 0 + + def columnCount(self, parent: QModelIndex = QModelIndex()): + """Returns the number of columns.""" + return 1 # We only need one column for the hierarchy name + + def data(self, index: QModelIndex, role: int = Qt.ItemDataRole.DisplayRole): + """Returns the data for the given index and role.""" + if not index.isValid(): + return None + + item = index.internalPointer() + + if role == Qt.ItemDataRole.DisplayRole: + if isinstance(item, SourceRule): + return f"Source: {item.input_path}" # Or some other identifier + elif isinstance(item, AssetRule): + return f"Asset: {item.asset_name}" # Or some other identifier + elif isinstance(item, FileRule): + return f"File: {item.file_path}" # Or some other identifier + else: + return None + # Add other roles as needed (e.g., Qt.ItemDataRole.DecorationRole for icons) + + return None + + def index(self, row: int, column: int, parent: QModelIndex = QModelIndex()): + """Returns the model index for the given row, column, and parent index.""" + if not self.hasIndex(row, column, parent): + return QModelIndex() + + if not parent.isValid(): + # Requesting index for the root item (SourceRule) + if self._root_rule and row == 0: + return self.createIndex(row, column, self._root_rule) + else: + return QModelIndex() + else: + parent_item = parent.internalPointer() + if isinstance(parent_item, SourceRule): + # Children are AssetRules + if 0 <= row < len(parent_item.assets): + child_item = parent_item.assets[row] + return self.createIndex(row, column, child_item) + else: + return QModelIndex() + elif isinstance(parent_item, AssetRule): + # Children are FileRules + if 0 <= row < len(parent_item.files): + child_item = parent_item.files[row] + return self.createIndex(row, column, child_item) + else: + return QModelIndex() + else: + return QModelIndex() # Should not happen for FileRule parents + + def parent(self, index: QModelIndex): + """Returns the parent index for the given index.""" + if not index.isValid(): + return QModelIndex() + + child_item = index.internalPointer() + + if isinstance(child_item, SourceRule): + # SourceRule is the root, has no parent in the model hierarchy + return QModelIndex() + elif isinstance(child_item, AssetRule): + # Find the SourceRule that contains this AssetRule + if self._root_rule and child_item in self._root_rule.assets: + # The row of the SourceRule is always 0 in this model + return self.createIndex(0, 0, self._root_rule) + else: + return QModelIndex() # Should not happen if data is consistent + elif isinstance(child_item, FileRule): + # Find the AssetRule that contains this FileRule + if self._root_rule: + for asset_row, asset_rule in enumerate(self._root_rule.assets): + if child_item in asset_rule.files: + # The row of the parent AssetRule within the SourceRule's children + return self.createIndex(asset_row, 0, asset_rule) + return QModelIndex() # Should not happen if data is consistent + else: + return QModelIndex() # Unknown item type + + def headerData(self, section: int, orientation: Qt.Orientation, role: int = Qt.ItemDataRole.DisplayRole): + """Returns the data for the header.""" + if orientation == Qt.Orientation.Horizontal and role == Qt.ItemDataRole.DisplayRole: + if section == 0: + return "Hierarchy" + return None + + def get_item_from_index(self, index: QModelIndex): + """Helper to get the underlying rule object from a model index.""" + if index.isValid(): + return index.internalPointer() + return None + +if __name__ == '__main__': + # Example Usage (for testing the model) + from PySide6.QtWidgets import QApplication, QTreeView + from dataclasses import dataclass, field + + # Define placeholder rule structures if not imported + @dataclass + class FileRule: + name: str = "file" + setting_f1: str = "value1" + setting_f2: int = 10 + + @dataclass + class AssetRule: + name: str = "asset" + files: list[FileRule] = field(default_factory=list) + setting_a1: bool = True + setting_a2: float = 3.14 + + @dataclass + class SourceRule: + name: str = "source" + assets: list[AssetRule] = field(default_factory=list) + setting_s1: str = "hello" + + # Create a sample hierarchical structure + file1 = FileRule(name="texture_diffuse.png") + file2 = FileRule(name="texture_normal.png") + file3 = FileRule(name="model.obj") + + asset1 = AssetRule(name="Material_01", files=[file1, file2]) + asset2 = AssetRule(name="Model_01", files=[file3]) + + source_rule_instance = SourceRule(name="Input_Archive", assets=[asset1, asset2]) + + app = QApplication([]) + tree_view = QTreeView() + model = RuleHierarchyModel(source_rule_instance) + tree_view.setModel(model) + tree_view.setWindowTitle("Rule Hierarchy Example") + tree_view.show() + app.exec() \ No newline at end of file diff --git a/gui/unified_view_model.py b/gui/unified_view_model.py new file mode 100644 index 0000000..1ebd2bb --- /dev/null +++ b/gui/unified_view_model.py @@ -0,0 +1,1090 @@ +# gui/unified_view_model.py +import logging +log = logging.getLogger(__name__) +from PySide6.QtCore import QAbstractItemModel, QModelIndex, Qt, Signal, Slot, QMimeData, QByteArray, QDataStream, QIODevice +from PySide6.QtGui import QColor +from pathlib import Path +from rule_structure import SourceRule, AssetRule, FileRule +from configuration import load_base_config +from typing import List + +class CustomRoles: + MapTypeRole = Qt.UserRole + 1 + TargetAssetRole = Qt.UserRole + 2 + # Add other custom roles here as needed +class UnifiedViewModel(QAbstractItemModel): + # --- Color Constants for Row Backgrounds --- + # Old colors removed, using config now + fixed source color + SOURCE_RULE_COLOR = QColor("#306091") + # ----------------------------------------- + + """ + A QAbstractItemModel for displaying and editing the hierarchical structure + of SourceRule -> AssetRule -> FileRule. + """ + # Signal emitted when a FileRule's target asset override changes. + # Carries the FileRule object and the new target asset path (or None). + targetAssetOverrideChanged = Signal(FileRule, str, QModelIndex) + + # Signal emitted when an AssetRule's name changes. + # Carries the AssetRule object, the new name, and the index. + assetNameChanged = Signal(AssetRule, str, QModelIndex) + + Columns = [ + "Name", "Target Asset", "Supplier", + "Asset Type", "Item Type" + ] + + COL_NAME = 0 + COL_TARGET_ASSET = 1 + COL_SUPPLIER = 2 + COL_ASSET_TYPE = 3 + COL_ITEM_TYPE = 4 + # COL_STATUS = 5 # Removed + # COL_OUTPUT_PATH = 6 # Removed + + # --- Drag and Drop MIME Type --- + MIME_TYPE = "application/x-filerule-index-list" + + def __init__(self, parent=None): + super().__init__(parent) + self._source_rules = [] + # self._display_mode removed + self._asset_type_colors = {} + self._file_type_colors = {} + self._asset_type_keys = [] + self._file_type_keys = [] + self._load_definitions() + + def _load_definitions(self): + """Loads configuration and caches colors and type keys.""" + try: + base_config = load_base_config() + asset_type_defs = base_config.get('ASSET_TYPE_DEFINITIONS', {}) + file_type_defs = base_config.get('FILE_TYPE_DEFINITIONS', {}) + + # Cache Asset Type Definitions (Keys and Colors) + self._asset_type_keys = sorted(list(asset_type_defs.keys())) + for type_name, type_info in asset_type_defs.items(): + hex_color = type_info.get("color") + if hex_color: + try: + self._asset_type_colors[type_name] = QColor(hex_color) + except ValueError: + log.warning(f"Invalid hex color '{hex_color}' for asset type '{type_name}' in config.") + + # Cache File Type Definitions (Keys and Colors) + self._file_type_keys = sorted(list(file_type_defs.keys())) + for type_name, type_info in file_type_defs.items(): + hex_color = type_info.get("color") + if hex_color: + try: + self._file_type_colors[type_name] = QColor(hex_color) + except ValueError: + log.warning(f"Invalid hex color '{hex_color}' for file type '{type_name}' in config.") + + except Exception as e: + log.exception(f"Error loading or caching colors from configuration: {e}") + # Ensure caches/lists are empty if loading fails + self._asset_type_colors = {} + self._file_type_colors = {} + self._asset_type_keys = [] + self._file_type_keys = [] + + def load_data(self, source_rules_list: list): + """Loads or reloads the model with a list of SourceRule objects.""" + # Consider if color cache needs refreshing if config can change dynamically + # self._load_and_cache_colors() # Uncomment if config can change and needs refresh + self.beginResetModel() + self._source_rules = source_rules_list if source_rules_list else [] + # Ensure back-references for parent lookup are set on the NEW items + for source_rule in self._source_rules: + for asset_rule in source_rule.assets: + asset_rule.parent_source = source_rule + for file_rule in asset_rule.files: + file_rule.parent_asset = asset_rule + self.endResetModel() + + def clear_data(self): + """Clears the model data.""" + self.beginResetModel() + self._source_rules = [] + self.endResetModel() + + def get_all_source_rules(self) -> list: + """Returns the internal list of SourceRule objects.""" + return self._source_rules + + # set_display_mode removed + + def rowCount(self, parent: QModelIndex = QModelIndex()) -> int: + """Returns the number of rows under the given parent.""" + if not parent.isValid(): + # Parent is the invisible root. Children are the SourceRules. + return len(self._source_rules) + + parent_item = parent.internalPointer() + if isinstance(parent_item, SourceRule): + return len(parent_item.assets) + elif isinstance(parent_item, AssetRule): + return len(parent_item.files) + elif isinstance(parent_item, FileRule): + return 0 + + return 0 # Should not happen for valid items + + def columnCount(self, parent: QModelIndex = QModelIndex()) -> int: + """Returns the number of columns.""" + return len(self.Columns) + + def parent(self, index: QModelIndex) -> QModelIndex: + """Returns the parent of the model item with the given index.""" + if not index.isValid(): + return QModelIndex() + + child_item = index.internalPointer() + if child_item is None: + return QModelIndex() + + # Determine the parent based on the item type + if isinstance(child_item, SourceRule): + # Parent is the invisible root + return QModelIndex() + elif isinstance(child_item, AssetRule): + # Parent is a SourceRule. Find its row in the _source_rules list. + parent_item = getattr(child_item, 'parent_source', None) + if parent_item and parent_item in self._source_rules: + try: + parent_row = self._source_rules.index(parent_item) + return self.createIndex(parent_row, 0, parent_item) + except ValueError: + return QModelIndex() # Should not happen if parent_source is correct + else: + return QModelIndex() # Parent SourceRule not found or reference missing + + elif isinstance(child_item, FileRule): + # Parent is an AssetRule. Find its row within its parent SourceRule. + parent_item = getattr(child_item, 'parent_asset', None) + if parent_item: + grandparent_item = getattr(parent_item, 'parent_source', None) + if grandparent_item: + try: + parent_row = grandparent_item.assets.index(parent_item) + # We need the index of the grandparent (SourceRule) to create the parent index + grandparent_row = self._source_rules.index(grandparent_item) + return self.createIndex(parent_row, 0, parent_item) + except ValueError: + return QModelIndex() # Parent AssetRule or Grandparent SourceRule not found in respective lists + else: + return QModelIndex() # Grandparent (SourceRule) reference missing + else: + return QModelIndex() # Parent AssetRule reference missing + + return QModelIndex() # Should not be reached + + + def index(self, row: int, column: int, parent: QModelIndex = QModelIndex()) -> QModelIndex: + """Returns the index of the item in the model specified by the given row, column and parent index.""" + if not self.hasIndex(row, column, parent): + return QModelIndex() + + parent_item = None + if not parent.isValid(): + # Parent is invisible root. Children are SourceRules. + if row < len(self._source_rules): + child_item = self._source_rules[row] + return self.createIndex(row, column, child_item) + else: + return QModelIndex() # Row out of bounds for top-level items + else: + # Parent is a valid index, get its item + parent_item = parent.internalPointer() + + child_item = None + if isinstance(parent_item, SourceRule): + if row < len(parent_item.assets): + child_item = parent_item.assets[row] + if not hasattr(child_item, 'parent_source'): + child_item.parent_source = parent_item + elif isinstance(parent_item, AssetRule): + if row < len(parent_item.files): + child_item = parent_item.files[row] + if not hasattr(child_item, 'parent_asset'): + child_item.parent_asset = parent_item + + if child_item: + return self.createIndex(row, column, child_item) + else: + return QModelIndex() + + def data(self, index: QModelIndex, role: int = Qt.DisplayRole): + """Returns the data stored under the given role for the item referred to by the index.""" + if not index.isValid(): + return None + + item = index.internalPointer() + column = index.column() + + # --- Handle Background Role --- + if role == Qt.BackgroundRole: + if isinstance(item, SourceRule): + return self.SOURCE_RULE_COLOR + elif isinstance(item, AssetRule): + # Determine effective asset type + asset_type = item.asset_type_override if item.asset_type_override else item.asset_type + if asset_type: + # Use cached color + return self._asset_type_colors.get(asset_type) + else: + return None # Fallback if no asset_type determined + elif isinstance(item, FileRule): + # --- New Logic: Darkened Parent Background --- + parent_asset = getattr(item, 'parent_asset', None) + if parent_asset: + parent_asset_type = parent_asset.asset_type_override if parent_asset.asset_type_override else parent_asset.asset_type + parent_bg_color = self._asset_type_colors.get(parent_asset_type) if parent_asset_type else None + + if parent_bg_color: + # Darken the parent color by ~30% (factor 130) + return parent_bg_color.darker(130) + else: + # Parent has no specific color, use default background + return None + else: + # Should not happen if structure is correct, but fallback to default + return None + # --- End New Logic --- + else: + return None + # --- Handle Foreground Role (Text Color) --- + elif role == Qt.ForegroundRole: + if isinstance(item, FileRule): + # Determine effective item type + effective_item_type = item.item_type_override if item.item_type_override is not None else item.item_type + if effective_item_type: + # Use cached color for text + return self._file_type_colors.get(effective_item_type) + # For SourceRule and AssetRule, return None to use default text color (usually contrasts well) + return None + + # --- Handle other roles (Display, Edit, etc.) --- + if isinstance(item, SourceRule): + if role == Qt.DisplayRole or role == Qt.EditRole: + if column == self.COL_NAME: + return Path(item.input_path).name + elif column == self.COL_SUPPLIER: + display_value = item.supplier_override if item.supplier_override is not None else item.supplier_identifier + return display_value if display_value is not None else "" + return None # Other columns/roles are blank for SourceRule + + # --- Logic for AssetRule and FileRule (previously detailed mode only) --- + elif isinstance(item, AssetRule): + if role == Qt.DisplayRole: + if column == self.COL_NAME: return item.asset_name + elif column == self.COL_ASSET_TYPE: + display_value = item.asset_type_override if item.asset_type_override is not None else item.asset_type + return display_value if display_value else "" + elif role == Qt.EditRole: + if column == self.COL_NAME: + return item.asset_name + elif column == self.COL_ASSET_TYPE: + return item.asset_type_override + return None + + elif isinstance(item, FileRule): + if role == Qt.DisplayRole: + if column == self.COL_NAME: return Path(item.file_path).name + elif column == self.COL_TARGET_ASSET: + return item.target_asset_name_override if item.target_asset_name_override is not None else "" + elif column == self.COL_ITEM_TYPE: + override = item.item_type_override + initial_type = item.item_type + if override is not None: return override + else: return initial_type if initial_type else "" + elif role == Qt.EditRole: + if column == self.COL_TARGET_ASSET: return item.target_asset_name_override if item.target_asset_name_override is not None else "" + elif column == self.COL_ITEM_TYPE: return item.item_type_override + return None + + return None + + def setData(self, index: QModelIndex, value, role: int = Qt.EditRole) -> bool: + """Sets the role data for the item at index to value.""" + if not index.isValid() or role != Qt.EditRole: + return False + + item = index.internalPointer() + if item is None: + return False + column = index.column() + changed = False + + # --- Handle different item types --- + if isinstance(item, SourceRule): + if column == self.COL_SUPPLIER: + # Get the new value, strip whitespace, treat empty as None + log.debug(f"setData COL_SUPPLIER: Index=({index.row()},{column}), Value='{value}', Type={type(value)}") + new_value = str(value).strip() if value is not None and str(value).strip() else None + + # Get the original identifier (assuming it exists on SourceRule) + original_identifier = getattr(item, 'supplier_identifier', None) + + # If the new value is the same as the original, clear the override + if new_value == original_identifier: + new_value = None + + # Update supplier_override only if it's different + if item.supplier_override != new_value: + item.supplier_override = new_value + changed = True + + elif isinstance(item, AssetRule): + if column == self.COL_NAME: + new_asset_name = str(value).strip() if value else None + if not new_asset_name: + log.warning("setData: Asset name cannot be empty.") + return False + + if item.asset_name == new_asset_name: + return False + + # --- Validation: Check for duplicates within the same SourceRule --- + parent_source = getattr(item, 'parent_source', None) + if parent_source: + for existing_asset in parent_source.assets: + if existing_asset.asset_name == new_asset_name and existing_asset is not item: + log.warning(f"setData: Duplicate asset name '{new_asset_name}' detected within the same source. Aborting rename.") + # Optionally, provide user feedback here via a signal or message box + return False + else: + log.error("setData: Cannot validate asset name, parent SourceRule not found.") + # Decide how to handle this - proceed cautiously or abort? Aborting is safer. + return False + # --- End Validation --- + + log.info(f"setData: Renaming AssetRule from '{item.asset_name}' to '{new_asset_name}'") + old_asset_name = item.asset_name + item.asset_name = new_asset_name + changed = True + # Emit signal for asset name change, including the index + self.assetNameChanged.emit(item, new_asset_name, index) + + # --- Update Child FileRule Target Asset Overrides --- + log.debug(f"setData: Updating FileRule target overrides from '{old_asset_name}' to '{new_asset_name}'") + updated_file_indices = [] + for src_idx, source_rule in enumerate(self._source_rules): + source_rule_index = self.createIndex(src_idx, 0, source_rule) + for asset_idx, asset_rule in enumerate(source_rule.assets): + asset_rule_index = self.createIndex(asset_idx, 0, asset_rule) + for file_idx, file_rule in enumerate(asset_rule.files): + if file_rule.target_asset_name_override == old_asset_name: + log.debug(f" Updating target for file: {Path(file_rule.file_path).name}") + file_rule.target_asset_name_override = new_asset_name + # Get the correct index for the file rule to emit dataChanged + file_rule_parent_index = self.parent(self.createIndex(file_idx, 0, file_rule)) + file_rule_index = self.index(file_idx, self.COL_TARGET_ASSET, file_rule_parent_index) + if file_rule_index.isValid(): + updated_file_indices.append(file_rule_index) + else: + log.warning(f" Could not get valid index for updated file rule target: {Path(file_rule.file_path).name}") + + + # Emit dataChanged for all updated file rules *after* the loop + for file_index in updated_file_indices: + self.dataChanged.emit(file_index, file_index, [Qt.DisplayRole, Qt.EditRole]) + # --- End Child Update --- + + elif column == self.COL_ASSET_TYPE: + # Delegate provides string value (e.g., "Surface", "Model") or None + new_value = str(value) if value is not None else None + if new_value == "": new_value = None + # Update asset_type_override + if item.asset_type_override != new_value: + item.asset_type_override = new_value + changed = True + + elif isinstance(item, FileRule): + if column == self.COL_TARGET_ASSET: + # Ensure value is string or None + new_value = str(value).strip() if value is not None else None + if new_value == "": new_value = None + # Update target_asset_name_override + if item.target_asset_name_override != new_value: + old_value = item.target_asset_name_override + item.target_asset_name_override = new_value + changed = True + # Emit signal that the override changed, let handler deal with restructuring + # Pass the FileRule item itself, the new value, and the index + self.targetAssetOverrideChanged.emit(item, new_value, index) + elif column == self.COL_ITEM_TYPE: + # Delegate provides string value (e.g., "MAP_COL") or None + new_value = str(value) if value is not None else None + if new_value == "": new_value = None + # Update item_type_override + if item.item_type_override != new_value: + log.debug(f"setData COL_ITEM_TYPE: File='{Path(item.file_path).name}', Original Override='{item.item_type_override}', New Value='{new_value}'") + old_override = item.item_type_override + item.item_type_override = new_value + changed = True + + # standard_map_type is no longer stored on FileRule. + # Remove the logic that updated it here. + pass + + log.debug(f"setData COL_ITEM_TYPE: File='{Path(item.file_path).name}', Final Override='{item.item_type_override}'") + + + if changed: + # Emit dataChanged for the specific index and affected roles + self.dataChanged.emit(index, index, [Qt.DisplayRole, Qt.EditRole]) + return True + + return False + + def flags(self, index: QModelIndex) -> Qt.ItemFlags: + """Returns the item flags for the given index.""" + if not index.isValid(): + return Qt.NoItemFlags + + # Start with default flags for a valid item + default_flags = Qt.ItemIsEnabled | Qt.ItemIsSelectable + + item = index.internalPointer() + if not item: + return Qt.NoItemFlags + column = index.column() + + can_edit = False + if isinstance(item, SourceRule): + if column == self.COL_SUPPLIER: can_edit = True + elif isinstance(item, AssetRule): + if column == self.COL_NAME: can_edit = True + if column == self.COL_ASSET_TYPE: can_edit = True + # AssetRule items can accept drops + default_flags |= Qt.ItemIsDropEnabled + elif isinstance(item, FileRule): + if column == self.COL_TARGET_ASSET: can_edit = True + if column == self.COL_ITEM_TYPE: can_edit = True + # FileRule items can be dragged + default_flags |= Qt.ItemIsDragEnabled + + if can_edit: + default_flags |= Qt.ItemIsEditable + + return default_flags + # Removed erroneous else block + + def headerData(self, section: int, orientation: Qt.Orientation, role: int = Qt.DisplayRole): + """Returns the data for the given role and section in the header.""" + if orientation == Qt.Horizontal and role == Qt.DisplayRole: + if 0 <= section < len(self.Columns): + return self.Columns[section] + # Optionally handle Vertical header (row numbers) + # if orientation == Qt.Vertical and role == Qt.DisplayRole: + # return str(section + 1) + return None + + # Helper to get item from index + def getItem(self, index: QModelIndex): + """Safely returns the item associated with the index.""" + if index.isValid(): + item = index.internalPointer() + if item: + return item + return None + # --- Method to update model based on prediction results, preserving overrides --- + def update_rules_for_sources(self, new_source_rules: List[SourceRule]): + """ + Updates the model's internal data based on a list of new SourceRule objects + (typically from prediction results), merging them with existing data while + preserving user overrides. + + Args: + new_source_rules: A list of SourceRule objects containing the new structure. + """ + if not new_source_rules: + log.warning("UnifiedViewModel: update_rules_for_sources called with empty list.") + return + + log.info(f"UnifiedViewModel: Updating rules for {len(new_source_rules)} source(s).") + + for new_source_rule in new_source_rules: + source_path = new_source_rule.input_path + existing_source_rule = None + existing_source_row = -1 + + # 1. Find existing SourceRule in the model + for i, rule in enumerate(self._source_rules): + if rule.input_path == source_path: + existing_source_rule = rule + existing_source_row = i + break + + if existing_source_rule is None: + # 2. Add New SourceRule if not found + log.debug(f"Adding new SourceRule for '{source_path}'") + # Ensure parent references are set within the new rule hierarchy + for asset_rule in new_source_rule.assets: + asset_rule.parent_source = new_source_rule + for file_rule in asset_rule.files: + file_rule.parent_asset = asset_rule + + # Add to model's internal list and emit signal + insert_row = len(self._source_rules) + self.beginInsertRows(QModelIndex(), insert_row, insert_row) + self._source_rules.append(new_source_rule) + self.endInsertRows() + continue + + # 3. Merge Existing SourceRule + log.debug(f"Merging SourceRule for '{source_path}'") + existing_source_index = self.createIndex(existing_source_row, 0, existing_source_rule) + if not existing_source_index.isValid(): + log.error(f"Could not create valid index for existing SourceRule: {source_path}. Skipping.") + continue + + # Update non-override SourceRule fields (e.g., supplier identifier if needed) + if existing_source_rule.supplier_identifier != new_source_rule.supplier_identifier: + # Only update if override is not set, or if you want prediction to always update base identifier + if existing_source_rule.supplier_override is None: + existing_source_rule.supplier_identifier = new_source_rule.supplier_identifier + # Emit dataChanged for the supplier column if it's displayed/editable at source level + supplier_col_index = self.createIndex(existing_source_row, self.COL_SUPPLIER, existing_source_rule) + self.dataChanged.emit(supplier_col_index, supplier_col_index, [Qt.DisplayRole, Qt.EditRole]) + + + # --- Merge AssetRules --- + existing_assets_dict = {asset.asset_name: asset for asset in existing_source_rule.assets} + new_assets_dict = {asset.asset_name: asset for asset in new_source_rule.assets} + processed_asset_names = set() + + # Iterate through new assets to update existing or add new ones + for asset_name, new_asset in new_assets_dict.items(): + processed_asset_names.add(asset_name) + existing_asset = existing_assets_dict.get(asset_name) + + if existing_asset: + # --- Update Existing AssetRule --- + log.debug(f" Merging AssetRule: {asset_name}") + existing_asset_row = existing_source_rule.assets.index(existing_asset) + existing_asset_index = self.createIndex(existing_asset_row, 0, existing_asset) + + # Update non-override fields (e.g., asset_type) + if existing_asset.asset_type != new_asset.asset_type and existing_asset.asset_type_override is None: + existing_asset.asset_type = new_asset.asset_type + asset_type_col_index = self.createIndex(existing_asset_row, self.COL_ASSET_TYPE, existing_asset) + self.dataChanged.emit(asset_type_col_index, asset_type_col_index, [Qt.DisplayRole, Qt.EditRole, Qt.BackgroundRole]) + + # --- Merge FileRules within the AssetRule --- + self._merge_file_rules(existing_asset, new_asset, existing_asset_index) + + else: + # --- Add New AssetRule --- + log.debug(f" Adding new AssetRule: {asset_name}") + new_asset.parent_source = existing_source_rule + # Ensure file parents are set + for file_rule in new_asset.files: + file_rule.parent_asset = new_asset + + insert_row = len(existing_source_rule.assets) + self.beginInsertRows(existing_source_index, insert_row, insert_row) + existing_source_rule.assets.append(new_asset) + self.endInsertRows() + + # --- Remove Old AssetRules --- + # Find assets in existing but not in new, and remove them in reverse order + assets_to_remove = [] + for i, existing_asset in reversed(list(enumerate(existing_source_rule.assets))): + if existing_asset.asset_name not in processed_asset_names: + assets_to_remove.append((i, existing_asset.asset_name)) + + for row_index, asset_name_to_remove in assets_to_remove: + log.debug(f" Removing old AssetRule: {asset_name_to_remove}") + self.beginRemoveRows(existing_source_index, row_index, row_index) + existing_source_rule.assets.pop(row_index) + self.endRemoveRows() + + + def _merge_file_rules(self, existing_asset: AssetRule, new_asset: AssetRule, parent_asset_index: QModelIndex): + """Helper method to merge FileRules for a given AssetRule.""" + existing_files_dict = {file.file_path: file for file in existing_asset.files} + new_files_dict = {file.file_path: file for file in new_asset.files} + processed_file_paths = set() + + # Iterate through new files to update existing or add new ones + for file_path, new_file in new_files_dict.items(): + processed_file_paths.add(file_path) + existing_file = existing_files_dict.get(file_path) + + if existing_file: + # --- Update Existing FileRule --- + log.debug(f" Merging FileRule: {Path(file_path).name}") + existing_file_row = existing_asset.files.index(existing_file) + existing_file_index = self.createIndex(existing_file_row, 0, existing_file) + + # Update non-override fields (item_type, standard_map_type) + changed_roles = [] + if existing_file.item_type != new_file.item_type and existing_file.item_type_override is None: + existing_file.item_type = new_file.item_type + changed_roles.extend([Qt.DisplayRole, Qt.EditRole, Qt.BackgroundRole]) + + # standard_map_type is no longer stored on FileRule. + # Remove the logic that updated it during merge. + pass + + + # Emit dataChanged only if something actually changed + if changed_roles: + # Emit for all relevant columns potentially affected by type changes + for col in [self.COL_ITEM_TYPE]: + col_index = self.createIndex(existing_file_row, col, existing_file) + self.dataChanged.emit(col_index, col_index, changed_roles) + + else: + # --- Add New FileRule --- + log.debug(f" Adding new FileRule: {Path(file_path).name}") + new_file.parent_asset = existing_asset + insert_row = len(existing_asset.files) + self.beginInsertRows(parent_asset_index, insert_row, insert_row) + existing_asset.files.append(new_file) + self.endInsertRows() + + # --- Remove Old FileRules --- + files_to_remove = [] + for i, existing_file in reversed(list(enumerate(existing_asset.files))): + if existing_file.file_path not in processed_file_paths: + files_to_remove.append((i, Path(existing_file.file_path).name)) + + for row_index, file_name_to_remove in files_to_remove: + log.debug(f" Removing old FileRule: {file_name_to_remove}") + self.beginRemoveRows(parent_asset_index, row_index, row_index) + existing_asset.files.pop(row_index) + self.endRemoveRows() + + + # --- Dedicated Model Restructuring Methods --- + + def moveFileRule(self, source_file_index: QModelIndex, target_parent_asset_index: QModelIndex): + """Moves a FileRule (source_file_index) to a different AssetRule parent (target_parent_asset_index).""" + if not source_file_index.isValid() or not target_parent_asset_index.isValid(): + log.error("moveFileRule: Invalid source or target index provided.") + return False + + file_item = source_file_index.internalPointer() + target_parent_asset = target_parent_asset_index.internalPointer() + + if not isinstance(file_item, FileRule) or not isinstance(target_parent_asset, AssetRule): + log.error("moveFileRule: Invalid item types for source or target.") + return False + + old_parent_asset = getattr(file_item, 'parent_asset', None) + if not old_parent_asset: + log.error(f"moveFileRule: Source file '{Path(file_item.file_path).name}' has no parent asset.") + return False + + if old_parent_asset == target_parent_asset: + log.debug("moveFileRule: Source and target parent are the same. No move needed.") + return True + + # Get old parent index + source_rule = getattr(old_parent_asset, 'parent_source', None) + if not source_rule: + log.error(f"moveFileRule: Could not find SourceRule parent for old asset '{old_parent_asset.asset_name}'.") + return False + + try: + old_parent_row = source_rule.assets.index(old_parent_asset) + old_parent_index = self.createIndex(old_parent_row, 0, old_parent_asset) + source_row = old_parent_asset.files.index(file_item) + except ValueError: + log.error("moveFileRule: Could not find old parent or source file within their respective lists.") + return False + + target_row = len(target_parent_asset.files) + + log.debug(f"Moving file '{Path(file_item.file_path).name}' from '{old_parent_asset.asset_name}' (row {source_row}) to '{target_parent_asset.asset_name}' (row {target_row})") + self.beginMoveRows(old_parent_index, source_row, source_row, target_parent_asset_index, target_row) + # Restructure internal data + old_parent_asset.files.pop(source_row) + target_parent_asset.files.append(file_item) + file_item.parent_asset = target_parent_asset + self.endMoveRows() + return True + + def createAssetRule(self, source_rule: SourceRule, new_asset_name: str, copy_from_asset: AssetRule = None) -> QModelIndex: + """Creates a new AssetRule under the given SourceRule and returns its index.""" + if not isinstance(source_rule, SourceRule) or not new_asset_name: + log.error("createAssetRule: Invalid SourceRule or empty asset name provided.") + return QModelIndex() + + # Check if asset already exists under this source + for asset in source_rule.assets: + if asset.asset_name == new_asset_name: + log.warning(f"createAssetRule: Asset '{new_asset_name}' already exists under '{Path(source_rule.input_path).name}'.") + # Return existing index? Or fail? Let's return existing for now. + try: + existing_row = source_rule.assets.index(asset) + return self.createIndex(existing_row, 0, asset) + except ValueError: + log.error("createAssetRule: Found existing asset but failed to get its index.") + return QModelIndex() + + log.debug(f"Creating new AssetRule '{new_asset_name}' under '{Path(source_rule.input_path).name}'") + new_asset_rule = AssetRule(asset_name=new_asset_name) + new_asset_rule.parent_source = source_rule + + # Optionally copy type info from another asset + if isinstance(copy_from_asset, AssetRule): + new_asset_rule.asset_type = copy_from_asset.asset_type + new_asset_rule.asset_type_override = copy_from_asset.asset_type_override + + # Find parent SourceRule index + try: + grandparent_row = self._source_rules.index(source_rule) + grandparent_index = self.createIndex(grandparent_row, 0, source_rule) + except ValueError: + log.error(f"createAssetRule: Could not find SourceRule '{Path(source_rule.input_path).name}' in the model's root list.") + return QModelIndex() + + # Determine insertion row for the new parent (e.g., append) + new_parent_row = len(source_rule.assets) + + # Emit signals for inserting the new parent row + self.beginInsertRows(grandparent_index, new_parent_row, new_parent_row) + source_rule.assets.insert(new_parent_row, new_asset_rule) + self.endInsertRows() + + # Return index for the newly created asset + return self.createIndex(new_parent_row, 0, new_asset_rule) + + + def removeAssetRule(self, asset_rule_to_remove: AssetRule): + """Removes an AssetRule if it's empty.""" + if not isinstance(asset_rule_to_remove, AssetRule): + log.error("removeAssetRule: Invalid AssetRule provided.") + return False + + if asset_rule_to_remove.files: + log.warning(f"removeAssetRule: Asset '{asset_rule_to_remove.asset_name}' is not empty. Removal aborted.") + return False + + source_rule = getattr(asset_rule_to_remove, 'parent_source', None) + if not source_rule: + log.error(f"removeAssetRule: Could not find parent SourceRule for asset '{asset_rule_to_remove.asset_name}'.") + return False + + # Find parent SourceRule index and the row of the asset to remove + try: + grandparent_row = self._source_rules.index(source_rule) + grandparent_index = self.createIndex(grandparent_row, 0, source_rule) + asset_row_for_removal = source_rule.assets.index(asset_rule_to_remove) + except ValueError: + log.error(f"removeAssetRule: Could not find parent SourceRule or the AssetRule within its parent's list.") + return False + + def get_asset_type_keys(self) -> List[str]: + """Returns the cached list of asset type keys.""" + return self._asset_type_keys + + def get_file_type_keys(self) -> List[str]: + """Returns the cached list of file type keys.""" + return self._file_type_keys + + def findIndexForItem(self, target_item_object) -> QModelIndex | None: + """ + Finds the QModelIndex for a given item object (SourceRule, AssetRule, or FileRule) + by traversing the model's internal tree structure. + + Args: + target_item_object: The specific SourceRule, AssetRule, or FileRule object to find. + + Returns: + QModelIndex for the item if found, otherwise None. + """ + if target_item_object is None: + return None + + for sr_row, source_rule in enumerate(self._source_rules): + if source_rule is target_item_object: + return self.createIndex(sr_row, 0, source_rule) + + parent_source_rule_index = self.createIndex(sr_row, 0, source_rule) + if not parent_source_rule_index.isValid(): + log.error(f"findIndexForItem: Could not create valid index for SourceRule: {source_rule.input_path}") + continue + + + for ar_row, asset_rule in enumerate(source_rule.assets): + if asset_rule is target_item_object: + return self.index(ar_row, 0, parent_source_rule_index) + + parent_asset_rule_index = self.index(ar_row, 0, parent_source_rule_index) + if not parent_asset_rule_index.isValid(): + log.error(f"findIndexForItem: Could not create valid index for AssetRule: {asset_rule.asset_name}") + continue + + for fr_row, file_rule in enumerate(asset_rule.files): + if file_rule is target_item_object: + return self.index(fr_row, 0, parent_asset_rule_index) + + log.debug(f"findIndexForItem: Item {target_item_object!r} not found in the model.") + return None + + # --- removeAssetRule continued (log.debug was separated by the insert) --- + # This log line belongs to the removeAssetRule method defined earlier. + # It's being re-indented here to its correct place if it was part of that method's flow. + # However, looking at the original structure, the `return True` for removeAssetRule + # was at line 802, and the log.debug was at 798. This indicates the log.debug + # was likely the *start* of the problematic section in the previous attempt, + # and the `return True` was the end of `removeAssetRule`. + # The `log.debug` at original line 798 should be part of `removeAssetRule`'s positive path. + # The `return True` at original line 802 should be the final return of `removeAssetRule`. + + # Correcting the end of removeAssetRule: + log.debug(f"Removing empty AssetRule '{asset_rule_to_remove.asset_name}' at row {asset_row_for_removal} under '{Path(source_rule.input_path).name}'") + self.beginRemoveRows(grandparent_index, asset_row_for_removal, asset_row_for_removal) + source_rule.assets.pop(asset_row_for_removal) + self.endRemoveRows() + return True + + def update_status(self, source_path: str, status_text: str): + """ + Finds the SourceRule node for the given source_path and updates its status. + Emits dataChanged for the corresponding row. + """ + log.debug(f"Attempting to update status for source '{source_path}' to '{status_text}'") + found_row = -1 + found_rule = None + for i, rule in enumerate(self._source_rules): + if rule.input_path == source_path: + found_row = i + found_rule = rule + break + + if found_rule is not None and found_row != -1: + try: + # Attempt to set a status attribute (e.g., _status_message) + # Note: This attribute isn't formally defined in SourceRule structure yet. + setattr(found_rule, '_status_message', status_text) + log.info(f"Updated status for SourceRule '{source_path}' (row {found_row}) to '{status_text}'") + + # Emit dataChanged for the entire row to potentially trigger updates + # (e.g., delegates, background color based on status if implemented later) + start_index = self.createIndex(found_row, 0, found_rule) + end_index = self.createIndex(found_row, self.columnCount() - 1, found_rule) + self.dataChanged.emit(start_index, end_index, [Qt.DisplayRole]) + + except Exception as e: + log.exception(f"Error setting status attribute or emitting dataChanged for {source_path}: {e}") + else: + log.warning(f"Could not find SourceRule with path '{source_path}' to update status.") + + # --- Placeholder for node finding method (Original Request - Replaced by direct list search above) --- + # Kept for reference, but the logic above directly searches self._source_rules + + # --- Drag and Drop Methods --- + + def supportedDropActions(self) -> Qt.DropActions: + """Specifies that only Move actions are supported.""" + return Qt.MoveAction + + def mimeTypes(self) -> list[str]: + """Returns the list of supported MIME types for dragging.""" + return [self.MIME_TYPE] + + def mimeData(self, indexes: list[QModelIndex]) -> QMimeData: + """Encodes information about the dragged FileRule items.""" + mime_data = QMimeData() + encoded_data = QByteArray() + stream = QDataStream(encoded_data, QIODevice.OpenModeFlag.WriteOnly) + + dragged_file_info = [] + for index in indexes: + if not index.isValid() or index.column() != 0: + continue + item = index.internalPointer() + if isinstance(item, FileRule): + parent_index = self.parent(index) + if parent_index.isValid(): + # Store: source_row, source_parent_row, source_grandparent_row + # This allows reconstructing the index later + grandparent_index = self.parent(parent_index) + # Ensure grandparent_index is valid before accessing its row + if grandparent_index.isValid(): + dragged_file_info.append((index.row(), parent_index.row(), grandparent_index.row())) + else: + # Handle case where grandparent is the root (shouldn't happen for FileRule, but safety) + # Or if parent() failed unexpectedly + log.warning(f"mimeData: Could not get valid grandparent index for FileRule at row {index.row()}, parent row {parent_index.row()}") + + else: + log.warning(f"mimeData: Could not get parent index for FileRule at row {index.row()}") + + # Write the number of items first, then each tuple + stream.writeInt8(len(dragged_file_info)) + for info in dragged_file_info: + stream.writeInt8(info[0]) + stream.writeInt8(info[1]) + stream.writeInt8(info[2]) + + mime_data.setData(self.MIME_TYPE, encoded_data) + log.debug(f"mimeData: Encoded {len(dragged_file_info)} FileRule indices.") + return mime_data + + def canDropMimeData(self, data: QMimeData, action: Qt.DropAction, row: int, column: int, parent: QModelIndex) -> bool: + """Checks if the data can be dropped at the specified location.""" + if action != Qt.MoveAction or not data.hasFormat(self.MIME_TYPE): + return False + + # Check if the drop target is a valid AssetRule + if not parent.isValid(): + return False + + target_item = parent.internalPointer() + if not isinstance(target_item, AssetRule): + return False + + # Optional: Prevent dropping onto the original parent? (Might be confusing) + # For now, allow dropping onto the same parent (moveFileRule handles this) + + return True + + def dropMimeData(self, data: QMimeData, action: Qt.DropAction, row: int, column: int, parent: QModelIndex) -> bool: + """Handles the dropping of FileRule items onto an AssetRule.""" + if not self.canDropMimeData(data, action, row, column, parent): + log.warning("dropMimeData: canDropMimeData check failed.") + return False + + target_asset_item = parent.internalPointer() + if not isinstance(target_asset_item, AssetRule): + log.error("dropMimeData: Target item is not an AssetRule.") + return False + + encoded_data = data.data(self.MIME_TYPE) + stream = QDataStream(encoded_data, QIODevice.OpenModeFlag.ReadOnly) + + num_items = stream.readInt8() + source_indices_info = [] + for _ in range(num_items): + source_row = stream.readInt8() + source_parent_row = stream.readInt8() + source_grandparent_row = stream.readInt8() + source_indices_info.append((source_row, source_parent_row, source_grandparent_row)) + + log.debug(f"dropMimeData: Decoded {len(source_indices_info)} source indices. Target Asset: '{target_asset_item.asset_name}'") + + if not source_indices_info: + log.warning("dropMimeData: No valid source index information decoded.") + return False + + # Keep track of original parents that might become empty + original_parents = set() + moved_files_new_indices = {} + + # --- BEGIN FIX: Reconstruct all source indices BEFORE the move loop --- + source_indices_to_process = [] + log.debug("Reconstructing initial source indices...") + for src_row, src_parent_row, src_grandparent_row in source_indices_info: + grandparent_index = self.index(src_grandparent_row, 0, QModelIndex()) + if not grandparent_index.isValid(): + log.error(f"dropMimeData: Failed initial reconstruction of grandparent index (row {src_grandparent_row}). Skipping item.") + continue + old_parent_index = self.index(src_parent_row, 0, grandparent_index) + if not old_parent_index.isValid(): + log.error(f"dropMimeData: Failed initial reconstruction of old parent index (row {src_parent_row}). Skipping item.") + continue + source_file_index = self.index(src_row, 0, old_parent_index) + if not source_file_index.isValid(): + # Log the specific parent it failed under for better debugging + parent_name = getattr(old_parent_index.internalPointer(), 'asset_name', 'Unknown Parent') + log.error(f"dropMimeData: Failed initial reconstruction of source file index (original row {src_row}) under parent '{parent_name}'. Skipping item.") + continue + + # Check if the reconstructed index actually points to a FileRule + item_check = source_file_index.internalPointer() + if isinstance(item_check, FileRule): + source_indices_to_process.append(source_file_index) + log.debug(f" Successfully reconstructed index for file: {Path(item_check.file_path).name}") + else: + log.warning(f"dropMimeData: Initial reconstructed index (row {src_row}) does not point to a FileRule. Skipping.") + + log.debug(f"Successfully reconstructed {len(source_indices_to_process)} valid source indices.") + # --- END FIX --- + + + # Process moves using the pre-calculated valid indices + for source_file_index in source_indices_to_process: + # Get the file item (already validated during reconstruction) + file_item = source_file_index.internalPointer() + + # Track original parent for cleanup (using the valid index) + old_parent_index = self.parent(source_file_index) + if old_parent_index.isValid(): + old_parent_asset = old_parent_index.internalPointer() + if isinstance(old_parent_asset, AssetRule): + # Need grandparent row for the tuple key + grandparent_index = self.parent(old_parent_index) + if grandparent_index.isValid(): + original_parents.add((grandparent_index.row(), old_parent_asset.asset_name)) + else: + log.warning(f"Could not get grandparent index for original parent '{old_parent_asset.asset_name}' during cleanup tracking.") + else: + log.warning(f"Parent of file '{Path(file_item.file_path).name}' is not an AssetRule.") + else: + log.warning(f"Could not get valid parent index for file '{Path(file_item.file_path).name}' during cleanup tracking.") + + + # Perform the move using the model's method and the valid source_file_index + if self.moveFileRule(source_file_index, parent): + # --- Update Target Asset Override After Successful Move --- + # The file_item's parent_asset reference should now be updated by moveFileRule + new_parent_asset = getattr(file_item, 'parent_asset', None) + if new_parent_asset == target_asset_item: + if file_item.target_asset_name_override != target_asset_item.asset_name: + log.debug(f" Updating target override for '{Path(file_item.file_path).name}' to '{target_asset_item.asset_name}'") + file_item.target_asset_name_override = target_asset_item.asset_name + # Need the *new* index of the moved file to emit dataChanged + try: + new_row = target_asset_item.files.index(file_item) + new_file_index_col0 = self.index(new_row, 0, parent) + new_file_index_target_col = self.index(new_row, self.COL_TARGET_ASSET, parent) + if new_file_index_target_col.isValid(): + moved_files_new_indices[file_item.file_path] = new_file_index_target_col + else: + log.warning(f" Could not get valid *new* index for target column of moved file: {Path(file_item.file_path).name}") + except ValueError: + log.error(f" Could not find moved file '{Path(file_item.file_path).name}' in target parent's list after move.") + + else: + log.error(f" Move reported success, but file's parent reference ('{getattr(new_parent_asset, 'asset_name', 'None')}') doesn't match target ('{target_asset_item.asset_name}'). Override not updated.") + else: + log.error(f"dropMimeData: moveFileRule failed for file '{Path(file_item.file_path).name}'.") + # If one move fails, should we stop? For now, continue processing others. + continue + + # --- Emit dataChanged for Target Asset column AFTER all moves --- + for source_path, new_index in moved_files_new_indices.items(): + self.dataChanged.emit(new_index, new_index, [Qt.DisplayRole, Qt.EditRole]) + + # --- Cleanup: Remove any original parent AssetRules that are now empty --- + log.debug(f"dropMimeData: Checking original parents for cleanup: {list(original_parents)}") + for gp_row, asset_name in list(original_parents): + try: + if 0 <= gp_row < len(self._source_rules): + source_rule = self._source_rules[gp_row] + # Find the asset rule within the correct source rule + asset_rule_to_check = next((asset for asset in source_rule.assets if asset.asset_name == asset_name), None) + + if asset_rule_to_check and not asset_rule_to_check.files and asset_rule_to_check != target_asset_item: + log.info(f"dropMimeData: Attempting cleanup of now empty original parent: '{asset_rule_to_check.asset_name}'") + if not self.removeAssetRule(asset_rule_to_check): + log.warning(f"dropMimeData: Failed to remove empty original parent '{asset_rule_to_check.asset_name}'.") + elif not asset_rule_to_check: + log.warning(f"dropMimeData: Cleanup check failed. Could not find original parent asset '{asset_name}' in source rule at row {gp_row}.") + else: + log.warning(f"dropMimeData: Cleanup check failed. Invalid grandparent row index {gp_row} found in original_parents set.") + except Exception as e: + log.exception(f"dropMimeData: Error during cleanup check for parent '{asset_name}' (gp_row {gp_row}): {e}") + + + return True \ No newline at end of file diff --git a/llm_prototype/PLAN.md b/llm_prototype/PLAN.md deleted file mode 100644 index 88c04da..0000000 --- a/llm_prototype/PLAN.md +++ /dev/null @@ -1,80 +0,0 @@ -# LLM Asset Classifier Prototype Plan - -## 1. Goal - -Develop a standalone Python prototype module capable of classifying 3D asset files and determining relevant metadata by leveraging a Large Language Model (LLM). This prototype aims to: - -* Handle irregularly named input files where traditional regex/keyword-based presets fail. -* Support inputs containing multiple distinct assets within a single source directory or archive. -* Provide a flexible foundation for potential future integration into the main Asset Processor Tool. - -## 2. Approach - -We will adopt the following strategies for the prototype development: - -* **Standalone Prototype:** The core LLM interaction and classification logic will be built within a dedicated `llm_prototype/` directory. This isolates development, allows for focused testing, and de-risks the feature before considering integration into the main application. -* **Configurable LLM Endpoint:** The prototype will allow users to specify the LLM API endpoint via a configuration file (`config_llm.py`). This enables flexibility in using different LLM providers, including locally hosted models (e.g., via LM Studio) or commercial APIs. API keys will be managed securely via environment variables, not stored in configuration files. -* **Multi-Asset Handling via List Output:** To address inputs containing multiple assets (e.g., `Dinesen.zip`, `DG_Imperfections.zip`), the LLM will be prompted to return its findings as a JSON **list**. Each element in the list will represent a single, distinct asset identified within the input file set. -* **Chain of Thought (CoT) Prompting:** We will employ a Chain of Thought prompting technique. The LLM will be instructed first to outline its reasoning process (identifying asset groups, assigning files, determining metadata) within `` tags before generating the final, structured JSON output. This aims to improve the reliability and accuracy of handling the complex multi-asset identification task. -* **Unified Asset Category:** A single `asset_category` field will be used to classify the type of asset. The defined valid categories are: `Model`, `Surface`, `Decal`, `ATLAS`, `Imperfection`. -* **JSON Extraction & Validation:** The Python script (`llm_classifier.py`) will parse the full LLM response, extract the JSON list portion following the `` block, and perform strict validation against the expected schema and defined category/map type values. -* **Minimal Dependencies:** The prototype will aim for minimal external dependencies, primarily using the `requests` library for API communication. - -## 3. Prototype Structure (`llm_prototype/`) - -* `llm_classifier.py`: Main Python script containing the core logic for loading input, formatting prompts, calling the LLM, parsing the response, and validating the output. -* `config_llm.py`: Configuration file defining the LLM API endpoint, expected map types, expected asset categories, and the path to the prompt template. -* `requirements_llm.txt`: Lists Python dependencies (e.g., `requests`). -* `prompt_template.txt`: Text file containing the Chain of Thought prompt template with placeholders for input files and configuration values. -* `test_inputs/`: Directory containing example input file lists in JSON format (e.g., `test_inputs/dinesen_example.json`, `test_inputs/imperfections_example.json`). -* `README.md`: Instructions detailing setup, configuration (API endpoint, environment variables), and how to run the prototype. - -## 4. Key Schemas - -* **Input to Prototype (Example - `test_inputs/dinesen_example.json`):** - ```json - { - "files": [ - "3-HeartOak-RL-2-5m-300mm_COL-1_.jpg", - "3-HeartOak-RL-2-5m-300mm_DISP_.jpg", - "3-HeartOak-RL-2-5m-300mm_GLOSS_.jpg", - "3-HeartOak-RL-2-5m-300mm_NRM_.jpg", - "3-Oak-Classic-RL-2-5m-300mm_COL-1_.jpg", - "3-Oak-Classic-RL-2-5m-300mm_DISP_.jpg", - "3-Oak-Classic-RL-2-5m-300mm_GLOSS_.jpg", - "3-Oak-Classic-RL-2-5m-300mm_NRM_.jpg" - ] - } - ``` -* **Expected LLM Output (Conceptual Structure):** - ```json - [ - { - "asset_name": "3-HeartOak-RL-2-5m-300mm", - "asset_category": "Surface", // Unified: Model, Surface, Decal, ATLAS, Imperfection - "asset_archetype": "Wood", - "file_classifications": [ - {"input_path": "3-HeartOak-RL-2-5m-300mm_COL-1_.jpg", "classification": "Map", "map_type": "COL"}, - // ... other HeartOak files - ] - }, - { - "asset_name": "3-Oak-Classic-RL-2-5m-300mm", - "asset_category": "Surface", - "asset_archetype": "Wood", - "file_classifications": [ - {"input_path": "3-Oak-Classic-RL-2-5m-300mm_COL-1_.jpg", "classification": "Map", "map_type": "COL"}, - // ... other OakClassic files - ] - } - ] - ``` - -## 5. Next Steps - -1. Create the `llm_prototype/` directory structure. (Already exists, confirmed empty) -2. Create the initial placeholder files (`llm_classifier.py`, `config_llm.py`, `requirements_llm.txt`, `prompt_template.txt`, `README.md`). -3. Populate `config_llm.py` with initial configuration variables. -4. Draft the initial `prompt_template.txt`. -5. Create example input files in `test_inputs/`. -6. Begin implementing the core logic in `llm_classifier.py`. \ No newline at end of file diff --git a/llm_prototype/README.md b/llm_prototype/README.md deleted file mode 100644 index ec7aa4a..0000000 --- a/llm_prototype/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# LLM Asset Classifier Prototype - -This prototype demonstrates using a Large Language Model (LLM) to classify asset files and determine metadata from a list of filenames, particularly for irregularly named inputs and sources containing multiple assets. - -## Setup - -1. **Clone the repository:** If you haven't already, clone the main Asset Processor Tool repository. -2. **Navigate to the prototype directory:** `cd llm_prototype/` -3. **Create a Python Virtual Environment (Recommended):** - ```bash - python -m venv .venv - ``` -4. **Activate the Virtual Environment:** - * On Windows: `.venv\Scripts\activate` - * On macOS/Linux: `source .venv/bin/activate` -5. **Install Dependencies:** - ```bash - pip install -r requirements_llm.txt - ``` - -## Configuration - -Edit the `config_llm.py` file to configure the LLM API endpoint and other settings. - -* `LLM_API_ENDPOINT`: Set this to the URL of your LLM API. This could be a commercial API (like OpenAI) or a local server (like LM Studio). -* `LLM_MODEL_NAME`: If your API requires a specific model name, set it here. Leave empty if not needed. -* `LLM_API_KEY_ENV_VAR`: If your API requires an API key, set this to the name of the environment variable where your key is stored. - -**Important:** Do NOT put your API key directly in `config_llm.py`. Store it in an environment variable. - -**Example (Windows PowerShell):** -```powershell -$env:OPENAI_API_KEY="your_api_key_here" -``` - -**Example (Linux/macOS Bash):** -```bash -export OPENAI_API_KEY="your_api_key_here" -``` -*(Replace `OPENAI_API_KEY` and `"your_api_key_here"` as needed based on your `LLM_API_KEY_ENV_VAR` setting and actual key.)* - -## Running the Prototype - -The prototype takes a JSON file containing a list of filenames as input. - -1. **Prepare Input:** Create a JSON file (e.g., in the `test_inputs/` directory) with a structure like this: - ```json - { - "files": [ - "path/to/file1.png", - "path/to/model.fbx", - "another_file.tif" - ] - } - ``` - The paths should be relative paths as they would appear within an extracted asset source directory. - -2. **Execute the script:** - ```bash - python llm_classifier.py - ``` - Replace `` with the actual path to your input JSON file (e.g., `test_inputs/dinesen_example.json`). - -The script will load the configuration, format the prompt, call the LLM API, extract and validate the JSON response, and print the validated output. - -## Development Notes - -* The LLM prompt template is in `prompt_template.txt`. Modify this file to adjust the instructions given to the LLM. -* The core logic is in `llm_classifier.py`. This includes functions for loading config/input/prompt, calling the API, extracting JSON, and validating the output. -* The validation logic in `llm_classifier.py` is crucial for ensuring the LLM's output conforms to the expected structure and values. \ No newline at end of file diff --git a/llm_prototype/__pycache__/config_llm.cpython-313.pyc b/llm_prototype/__pycache__/config_llm.cpython-313.pyc deleted file mode 100644 index 6a94fe6..0000000 Binary files a/llm_prototype/__pycache__/config_llm.cpython-313.pyc and /dev/null differ diff --git a/llm_prototype/config_llm.py b/llm_prototype/config_llm.py deleted file mode 100644 index 8f95ffd..0000000 --- a/llm_prototype/config_llm.py +++ /dev/null @@ -1,94 +0,0 @@ -# llm_prototype/config_llm.py - -# Configuration for the LLM Asset Classifier Prototype - -# LLM API Endpoint (e.g., for OpenAI, local LLM via LM Studio, etc.) -# Example for OpenAI: "https://api.openai.com/v1/chat/completions" -# Example for LM Studio: "http://localhost:1234/v1/chat/completions" -LLM_API_ENDPOINT = "http://100.65.14.122:1234/v1/chat/completions" # Default to local LM Studio endpoint - -# Optional: LLM Model Name (may be required by some APIs) -LLM_MODEL_NAME = "phi-3.5-mini-instruct" # e.g., "gpt-4o-mini", "llama-3-8b-instruct" - -# Environment variable name for the API key (if required by the API) -LLM_API_KEY_ENV_VAR = "" - -# Path to the prompt template file (relative to the workspace root) -PROMPT_TEMPLATE_PATH = "llm_prototype/prompt_template.txt" - -# Expected internal map types the LLM should classify files into -EXPECTED_MAP_TYPES = [ - "COL", "NRM", "ROUGH", "METAL", "AO", "DISP", "REFL", "MASK", "SSS", "utility" -] - -# Examples/synonyms for each map type to guide the LLM -MAP_TYPE_EXAMPLES = { - "COL": ["Color", "Diffuse", "Albedo"], - "NRM": ["Normal"], - "ROUGH": ["Roughness"], - "METAL": ["Metallic"], - "AO": ["Ambient Occlusion"], - "DISP": ["Displacement", "Height"], - "REFL": ["Reflection"], - "MASK": ["Mask", "Alpha", "Opacity"], - "SSS": ["Subsurface Scattering"], - "utility": ["Utility", "Data", "Helper"] -} - -# Expected asset categories the LLM should classify assets into -EXPECTED_CATEGORIES = [ - "Model", "Surface", "Decal", "Atlas", "Imperfection" -] - -# Examples/synonyms for each asset category to guide the LLM -CATEGORY_EXAMPLES = { - "Model": "Represents a asset-set that has a 3D file (e.g., .obj, .fbx, .gltf). Can include accompanying PBR maps as well", - "Surface": "Represents a set of textures intended for a material (e.g., PBR texture sets with Color, Normal, Roughness maps).", - "Decal": "Represents a texture or set of textures intended to be placed on top of another material (e.g., stickers, grunge overlays). Always has some form of a opacity or mask map", - "Atlas": "Represents a texture or set of textures. Often called atlas or trimsheet", - "Imperfection": "Represents a single texturemap used to add surface imperfections" - } - -# Expected classifications for individual files -EXPECTED_CLASSIFICATIONS = [ - "Map", "Model", "Extra", "Ignored", "Unrecognised" -] - -# Placeholder for the input file list in the prompt template -FILE_LIST_PLACEHOLDER = "{{FILE_LIST_JSON}}" - -# Placeholder for the expected map types list in the prompt template -MAP_TYPES_PLACEHOLDER = "{{EXPECTED_MAP_TYPES}}" - -# Placeholder for the expected categories list in the prompt template -CATEGORIES_PLACEHOLDER = "{{EXPECTED_CATEGORIES}}" - -# Placeholder for the category examples in the prompt template -CATEGORY_EXAMPLES_PLACEHOLDER = "{{CATEGORY_EXAMPLES}}" - -# Placeholder for the expected classifications list in the prompt template -CLASSIFICATIONS_PLACEHOLDER = "{{EXPECTED_CLASSIFICATIONS}}" - -# Placeholder for the expected JSON output schema in the prompt template -OUTPUT_SCHEMA_PLACEHOLDER = "{{OUTPUT_SCHEMA}}" - -# The expected JSON output schema structure (used for validation and prompt) -# This defines the structure the LLM should return AFTER the block -OUTPUT_SCHEMA = """ -[ // Top-level LIST of assets - { // Asset Object - "asset_name": "...", // Determined name for the asset set - "asset_category": "...", // Must be one of: {{CATEGORIES_PLACEHOLDER}} - "asset_archetype": "...", // e.g., Wood, Metal, Fabric, Concrete, Smudge, Scratch, etc. - "file_classifications": [ // List of files belonging ONLY to this asset - { - "input_path": "...", // The original path from the input list - "classification": "...", // Must be one of: {{CLASSIFICATIONS_PLACEHOLDER}} - "map_type": "..." // Must be one of: {{MAP_TYPES_PLACEHOLDER}}, or null if not a Map - } - // ... one entry for each file belonging to this asset - ] - } - // ... potentially more asset objects if more are detected -] -""" \ No newline at end of file diff --git a/llm_prototype/llm_classifier.py b/llm_prototype/llm_classifier.py deleted file mode 100644 index c91770d..0000000 --- a/llm_prototype/llm_classifier.py +++ /dev/null @@ -1,290 +0,0 @@ -# llm_prototype/llm_classifier.py - -import os -import json -import requests -import sys - -# Add the prototype directory to the Python path to import config_llm -sys.path.append(os.path.dirname(__file__)) -import config_llm - -def load_config(): - """Loads configuration from config_llm.py.""" - print("Loading configuration...") - return config_llm - -def load_input_files(input_json_path): - """Loads the list of files from an input JSON file.""" - print(f"Loading input file list from: {input_json_path}") - try: - with open(input_json_path, 'r') as f: - data = json.load(f) - if "files" not in data or not isinstance(data["files"], list): - raise ValueError("Input JSON must contain a 'files' key with a list of strings.") - return data["files"] - except FileNotFoundError: - print(f"Error: Input file not found at {input_json_path}") - sys.exit(1) - except json.JSONDecodeError: - print(f"Error: Could not decode JSON from {input_json_path}") - sys.exit(1) - except ValueError as e: - print(f"Error in input file format: {e}") - sys.exit(1) - -def load_prompt_template(config): - """Loads the prompt template from the specified file.""" - print(f"Loading prompt template from: {config.PROMPT_TEMPLATE_PATH}") - try: - with open(config.PROMPT_TEMPLATE_PATH, 'r', encoding='utf-8') as f: - return f.read() - except FileNotFoundError: - print(f"Error: Prompt template file not found at {config.PROMPT_TEMPLATE_PATH}") - sys.exit(1) - except Exception as e: - print(f"Error loading prompt template: {e}") - sys.exit(1) - - -def format_prompt(template, file_list, config): - """Formats the prompt template with dynamic values.""" - print("Formatting prompt...") - file_list_json = json.dumps(file_list, indent=2) - - # Replace placeholders in the template - prompt = template.replace(config.FILE_LIST_PLACEHOLDER, file_list_json) - prompt = prompt.replace(config.MAP_TYPES_PLACEHOLDER, json.dumps(config.EXPECTED_MAP_TYPES)) - prompt = prompt.replace(config.CATEGORIES_PLACEHOLDER, json.dumps(config.EXPECTED_CATEGORIES)) - prompt = prompt.replace(config.CLASSIFICATIONS_PLACEHOLDER, json.dumps(config.EXPECTED_CLASSIFICATIONS)) - prompt = prompt.replace(config.OUTPUT_SCHEMA_PLACEHOLDER, config.OUTPUT_SCHEMA.strip()) - - return prompt - -def call_llm_api(prompt, config): - """ - Calls the LLM API with the formatted prompt. - Handles API key, headers, and basic error checking. - """ - print(f"Calling LLM API at: {config.LLM_API_ENDPOINT}") - - api_key = os.getenv(config.LLM_API_KEY_ENV_VAR) - headers = { - "Content-Type": "application/json", - } - - # Add Authorization header if API key is provided - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - elif "openai.com" in config.LLM_API_ENDPOINT.lower(): - print(f"Warning: {config.LLM_API_KEY_ENV_VAR} environment variable not set for OpenAI endpoint. API call may fail.") - - - payload = { - "model": config.LLM_MODEL_NAME if config.LLM_MODEL_NAME else "gpt-3.5-turbo", # Use a default if model name is empty - "messages": [ - {"role": "system", "content": "You are a helpful assistant that outputs JSON."}, # System message updated - {"role": "user", "content": prompt} - ] - # Removed "response_format": {"type": "json_object"} as we expect mixed output (text + JSON) - } - - print("\n--- JSON Payload Sent to LLM API ---") - print(json.dumps(payload, indent=2)) - print("------------------------------------\n") - - try: - response = requests.post(config.LLM_API_ENDPOINT, headers=headers, json=payload) - response.raise_for_status() # Raise an HTTPError for bad responses (4xx or 5xx) - - # Check if the response is directly a JSON object (as requested by response_format) - # Some APIs might return the JSON directly if response_format is supported and successful. - try: - json_response = response.json() - # Check if the JSON response looks like a chat completion response - if 'choices' in json_response and len(json_response['choices']) > 0: - return json_response # It's a standard chat completion response - else: - # It might be the raw JSON output if the API returned it directly - print("Received direct JSON response (not standard chat completion format).") - # Wrap it in a structure similar to chat completion for consistent processing - return {"choices": [{"message": {"content": json.dumps(json_response)}}]} - except json.JSONDecodeError: - # If it's not JSON, assume it's plain text content - print("Received non-JSON response. Treating as plain text.") - return {"choices": [{"message": {"content": response.text}}]} - - - except requests.exceptions.RequestException as e: - print(f"Error calling LLM API: {e}") - sys.exit(1) - -def extract_json_from_response(response_data): - """ - Extracts the JSON list part from the LLM's response content by finding - the first '[' and last ']' and parsing the content between them. - Handles responses that might include a thinking block or other text before/after the JSON. - """ - print("Extracting JSON from LLM response...") - - # Look for the content of the first message from the assistant - assistant_message_content = "" - if 'choices' in response_data and len(response_data['choices']) > 0: - message = response_data['choices'][0].get('message', {}) - assistant_message_content = message.get('content', '') - - # Strip markdown code fences if present - if assistant_message_content.strip().startswith("```json"): - assistant_message_content = assistant_message_content.strip()[len("```json"):].strip() - if assistant_message_content.strip().endswith("```"): - assistant_message_content = assistant_message_content.strip()[:-len("```")].strip() - - print("\n--- Processed Assistant Message Content (after stripping fences) ---") - print(assistant_message_content) - print("-------------------------------------------------------------------\n") - - if not assistant_message_content: - print("Error: LLM response content is empty or unexpected format.") - print(f"Full response: {response_data}") - # Attempt to return empty list for validation to catch this - return [] - - # Find the index of the first '[' and the last ']' - first_bracket_index = assistant_message_content.find('[') - last_bracket_index = assistant_message_content.rfind(']') - - if first_bracket_index == -1 or last_bracket_index == -1 or last_bracket_index < first_bracket_index: - print("Error: Could not find a valid JSON list structure (matching '[' and ']') in the LLM response content.") - print(f"Response content snippet: {assistant_message_content[:500]}...") # Print snippet - # Attempt to return empty list for validation to catch this - return [] - - # Extract the potential JSON string between the first '[' and last ']' - json_string = assistant_message_content[first_bracket_index : last_bracket_index + 1] - - # Attempt to parse the extracted string as JSON - try: - parsed_json = json.loads(json_string) - return parsed_json - except json.JSONDecodeError as e: - print(f"Error: Could not decode extracted JSON from LLM response: {e}") - print(f"Attempted to parse (snippet): {json_string[:500]}...") # Print snippet - # Attempt to return empty list for validation to catch this - return [] - - - - -def validate_llm_output(llm_output, config): - """Validates the structure and content of the parsed LLM output JSON.""" - print("Validating LLM output...") - - if not isinstance(llm_output, list): - print("Validation Error: LLM output is not a JSON list.") - return False - - if not llm_output: - print("Validation Warning: LLM output list is empty.") - # Depending on requirements, an empty list might be valid if no assets were found. - # For now, we'll allow it but warn. - - required_asset_keys = ["asset_name", "asset_category", "asset_archetype", "file_classifications"] - required_file_keys = ["input_path", "classification", "map_type"] - - for i, asset in enumerate(llm_output): - if not isinstance(asset, dict): - print(f"Validation Error: Asset item at index {i} is not a dictionary.") - return False - - # Validate required asset keys - for key in required_asset_keys: - if key not in asset: - print(f"Validation Error: Asset item at index {i} is missing required key: '{key}'.") - return False - - # Validate asset_category value - if asset["asset_category"] not in config.EXPECTED_CATEGORIES: - print(f"Validation Error: Asset item at index {i} has invalid asset_category: '{asset['asset_category']}'. Expected one of: {config.EXPECTED_CATEGORIES}") - return False - - # Validate file_classifications list - if not isinstance(asset["file_classifications"], list): - print(f"Validation Error: 'file_classifications' for asset at index {i} is not a list.") - return False - - for j, file_info in enumerate(asset["file_classifications"]): - if not isinstance(file_info, dict): - print(f"Validation Error: File classification item at asset index {i}, file index {j} is not a dictionary.") - return False - - # Validate required file keys - for key in required_file_keys: - if key not in file_info: - print(f"Validation Error: File classification item at asset index {i}, file index {j} is missing required key: '{key}'.") - return False - - # Validate classification value - if file_info["classification"] not in config.EXPECTED_CLASSIFICATIONS: - print(f"Validation Error: File classification item at asset index {i}, file index {j} has invalid classification: '{file_info['classification']}'. Expected one of: {config.EXPECTED_CLASSIFICATIONS}") - return False - - # Validate map_type value if classification is "Map" - if file_info["classification"] == "Map": - if file_info["map_type"] is None or file_info["map_type"] not in config.EXPECTED_MAP_TYPES: - print(f"Validation Error: File classification item at asset index {i}, file index {j} is classified as 'Map' but has invalid or missing map_type: '{file_info['map_type']}'. Expected one of: {config.EXPECTED_MAP_TYPES}") - return False - elif file_info["map_type"] is not None: - print(f"Validation Warning: File classification item at asset index {i}, file index {j} is not classified as 'Map' but has a map_type: '{file_info['map_type']}'. map_type should be null.") - # This is a warning, not a strict error, as some LLMs might include it. - - # Validate input_path is a string - if not isinstance(file_info["input_path"], str) or not file_info["input_path"]: - print(f"Validation Error: File classification item at asset index {i}, file index {j} has invalid or empty input_path: '{file_info['input_path']}'.") - return False - - - print("LLM output validation successful.") - return True - - -def main(): - """Main function to run the LLM classifier prototype.""" - if len(sys.argv) != 2: - print("Usage: python llm_classifier.py ") - sys.exit(1) - - input_json_path = sys.argv[1] - - config = load_config() - file_list = load_input_files(input_json_path) - prompt_template = load_prompt_template(config) - prompt = format_prompt(prompt_template, file_list, config) - - # print("\n--- Generated Prompt ---") - # print(prompt) - # print("------------------------\n") - - llm_response_data = call_llm_api(prompt, config) - - # print("\n--- Raw LLM Response ---") - # print(json.dumps(llm_response_data, indent=2)) - # print("------------------------\n") - - llm_output_json = extract_json_from_response(llm_response_data) - - # Add check for empty extracted JSON - if not llm_output_json: - print("\nError: Extracted JSON list is empty. LLM response or extraction failed.") - sys.exit(1) - - if validate_llm_output(llm_output_json, config): - print("\n--- Validated LLM Output JSON ---") - print(json.dumps(llm_output_json, indent=2)) - print("---------------------------------\n") - print("Prototype run complete. LLM output is valid.") - else: - print("\nPrototype run failed due to LLM output validation errors.") - sys.exit(1) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/llm_prototype/prompt_template.txt b/llm_prototype/prompt_template.txt deleted file mode 100644 index 4c162d4..0000000 --- a/llm_prototype/prompt_template.txt +++ /dev/null @@ -1,98 +0,0 @@ -# llm_prototype/prompt_template.txt - -You are an expert 3D asset file classifier. Your task is to analyze a list of filenames from a single asset source (which may contain multiple distinct assets) and provide a structured JSON output detailing the identified assets and their file classifications. - -Follow these steps: - -1. Analyze the provided list of filenames. -2. Identify distinct asset groups within the list based on naming patterns, file types, or other logical groupings. -3. For each identified asset group, determine: - * A concise `asset_name`. - * The `asset_category`. This must be one of the following exact strings: {{CATEGORIES_PLACEHOLDER}}. Refer to the descriptions below for guidance: {{CATEGORY_EXAMPLES}}. Note: The 'Imperfection' category typically contains a single map file. - * The `asset_archetype` (e.g., Wood, Metal, Fabric, Concrete, Smudge, Scratch, etc.). -4. For each file in the original input list, determine which asset group it belongs to (if any). -4.5. If a file does not belong to a distinct asset group (e.g., preview images, notes), classify it accordingly ("Extra", "Ignored", "Unrecognised") and include it in the "file_classifications" list of the *first* asset object in your output. -5. For each file, determine its `classification`. This must be one of the following exact strings: Map, Model, Extra, Ignored, Unrecognised. (Using placeholder: {{CLASSIFICATIONS_PLACEHOLDER}}) -6. If the classification is "Map", determine the `map_type`. This must be one of the following exact strings: {{EXPECTED_MAP_TYPES}}. If the classification is not "Map", the `map_type` should be `null`. Use the most appropriate map type from the list. For imperfection maps, the map_type should typically be "utility". Common synonyms for these map types include: COL (Color, Diffuse, Albedo), NRM (Normal), ROUGH (Roughness), METAL (Metallic), AO (Ambient Occlusion), DISP (Displacement, Height), REFL (Reflection), MASK (Mask, Alpha, Opacity), SSS (Subsurface Scattering), utility (Utility, Data, Helper). -7. Output your reasoning process within and tags. Explain how you identified the asset groups and classified the files. -8. After the tag, output ONLY the final JSON list structure based on your analysis. **Ensure the JSON is complete and valid.** Do not include any other text outside the JSON. -**STRICT INSTRUCTION: Output ONLY the final JSON list structure after the tag. The output MUST be a valid JSON list `[...]` containing one object `{...}` for EACH distinct asset identified. EACH asset object MUST include the keys "asset_name", "asset_category", "asset_archetype", and "file_classifications". ABSOLUTELY NO markdown code fences (```json) or comments (`//` or `#`) are allowed in the JSON output.** - -Input File List: -{{FILE_LIST_JSON}} - -Category Examples: -{{CATEGORY_EXAMPLES}} - -## Examples - -Here are a few examples of input file lists and the EXACT expected JSON output format. - -**Example 1: Single PBR Surface** -Input: -[ - "Wood_Plank_01_COL_1K.jpg", - "Wood_Plank_01_NRM_1K.jpg", - "Wood_Plank_01_ROUGH_1K.jpg" -] -Expected Output: -[ - { - "asset_name": "Wood_Plank_01", - "asset_category": "Surface", - "asset_archetype": "Wood", - "file_classifications": [ - {"input_path": "Wood_Plank_01_COL_1K.jpg", "classification": "Map", "map_type": "COL"}, - {"input_path": "Wood_Plank_01_NRM_1K.jpg", "classification": "Map", "map_type": "NRM"}, - {"input_path": "Wood_Plank_01_ROUGH_1K.jpg", "classification": "Map", "map_type": "ROUGH"} - ] - } -] - -**Example 2: Multiple Imperfection Maps** -Input: -[ - "DG-01-Smudges.tif", - "DG-02-Scratches.tif", - "DG-03-Smudges-Scratches.tif", - "preview.jpg", - "notes.txt" -] -Expected Output: -[ - { - "asset_name": "Smudges", - "asset_category": "Imperfection", - "asset_archetype": "Smudge", - "file_classifications": [ - {"input_path": "DG-01-Smudges.tif", "classification": "Map", "map_type": "utility"} - ] - }, - { - "asset_name": "Scratches", - "asset_category": "Imperfection", - "asset_archetype": "Scratch", - "file_classifications": [ - {"input_path": "DG-02-Scratches.tif", "classification": "Map", "map_type": "utility"} - ] - }, - { - "asset_name": "Smudges-Scratches", - "asset_category": "Imperfection", - "asset_archetype": "Mixed", - "file_classifications": [ - {"input_path": "DG-03-Smudges-Scratches.tif", "classification": "Map", "map_type": "utility"} - ] - }, - { - "asset_name": "Extra Files", - "asset_category": "Extra", - "asset_archetype": null, - "file_classifications": [ - {"input_path": "preview.jpg", "classification": "Extra", "map_type": null}, - {"input_path": "notes.txt", "classification": "Ignored", "map_type": null} - ] - } -] - -Input File List: diff --git a/llm_prototype/requirements_llm.txt b/llm_prototype/requirements_llm.txt deleted file mode 100644 index b3e628c..0000000 --- a/llm_prototype/requirements_llm.txt +++ /dev/null @@ -1,2 +0,0 @@ -# llm_prototype/requirements_llm.txt -requests \ No newline at end of file diff --git a/llm_prototype/test_inputs/dinesen_example.json b/llm_prototype/test_inputs/dinesen_example.json deleted file mode 100644 index 56959e6..0000000 --- a/llm_prototype/test_inputs/dinesen_example.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "files": [ - "3-HeartOak-RL-2-5m-300mm_COL-1_.jpg", - "3-HeartOak-RL-2-5m-300mm_DISP_.jpg", - "3-HeartOak-RL-2-5m-300mm_GLOSS_.jpg", - "3-HeartOak-RL-2-5m-300mm_NRM_.jpg", - "3-Oak-Classic-RL-2-5m-300mm_COL-1_.jpg", - "3-Oak-Classic-RL-2-5m-300mm_DISP_.jpg", - "3-Oak-Classic-RL-2-5m-300mm_GLOSS_.jpg", - "3-Oak-Classic-RL-2-5m-300mm_NRM_.jpg" - ] -} \ No newline at end of file diff --git a/llm_prototype/test_inputs/imperfections_example.json b/llm_prototype/test_inputs/imperfections_example.json deleted file mode 100644 index 8371711..0000000 --- a/llm_prototype/test_inputs/imperfections_example.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "files": [ - "DG-01-Smudges.tif", - "DG-02-Scratches.tif", - "DG-03-Smudges-Scratches.tif", - "preview.jpg", - "notes.txt" - ] -} \ No newline at end of file diff --git a/main.py b/main.py index e09abb6..2e04852 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,3 @@ -# main.py - import argparse import sys import time @@ -7,20 +5,59 @@ import os import logging from pathlib import Path from concurrent.futures import ProcessPoolExecutor, as_completed -import platform # To potentially adjust worker count defaults -import subprocess # <<< ADDED IMPORT -import shutil # <<< ADDED IMPORT -from typing import List, Dict, Tuple, Optional # Added for type hinting +import subprocess +import shutil +import tempfile +import zipfile +from typing import List, Dict, Tuple, Optional + +# --- Utility Imports --- +from utils.hash_utils import calculate_sha256 +from utils.path_utils import get_next_incrementing_value + +# --- Qt Imports for Application Structure --- +from PySide6.QtCore import QObject, Slot, QThreadPool, QRunnable, Signal +from PySide6.QtCore import Qt +from PySide6.QtWidgets import QApplication + +# --- Backend Imports --- +# Add current directory to sys.path for direct execution +import sys +import os +sys.path.append(os.path.dirname(__file__)) +print(f"DEBUG: sys.path after append: {sys.path}") -# --- Assuming classes are in sibling files --- try: + print("DEBUG: Attempting to import Configuration...") from configuration import Configuration, ConfigurationError - from asset_processor import AssetProcessor, AssetProcessingError - import config as core_config_module # <<< IMPORT config.py HERE + print("DEBUG: Successfully imported Configuration.") + + print("DEBUG: Attempting to import ProcessingEngine...") + from processing_engine import ProcessingEngine + print("DEBUG: Successfully imported ProcessingEngine.") + + print("DEBUG: Attempting to import SourceRule...") + from rule_structure import SourceRule + print("DEBUG: Successfully imported SourceRule.") + + print("DEBUG: Attempting to import MainWindow...") + from gui.main_window import MainWindow + print("DEBUG: Successfully imported MainWindow.") + + print("DEBUG: Attempting to import prepare_processing_workspace...") + from utils.workspace_utils import prepare_processing_workspace + print("DEBUG: Successfully imported prepare_processing_workspace.") + except ImportError as e: - # Provide a more helpful error message if imports fail script_dir = Path(__file__).parent.resolve() + print(f"ERROR: Cannot import Configuration or rule_structure classes.") + print(f"Ensure configuration.py and rule_structure.py are in the same directory or Python path.") print(f"ERROR: Failed to import necessary classes: {e}") + print(f"DEBUG: Exception type: {type(e)}") + print(f"DEBUG: Exception args: {e.args}") + import traceback + print("DEBUG: Full traceback of the ImportError:") + traceback.print_exc() print(f"Ensure 'configuration.py' and 'asset_processor.py' exist in the directory:") print(f" {script_dir}") print("Or that the directory is included in your PYTHONPATH.") @@ -34,7 +71,6 @@ def setup_logging(verbose: bool): log_format = '%(asctime)s [%(levelname)-8s] %(name)s: %(message)s' date_format = '%Y-%m-%d %H:%M:%S' - # Configure root logger # Remove existing handlers to avoid duplication if re-run in same session for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) @@ -44,18 +80,12 @@ def setup_logging(verbose: bool): format=log_format, datefmt=date_format, handlers=[ - logging.StreamHandler(sys.stdout) # Log to console - # Optional: Add FileHandler for persistent logs - # logging.FileHandler("asset_processor.log", mode='a', encoding='utf-8') + logging.StreamHandler(sys.stdout) ] ) - # Get logger specifically for this main script - log = logging.getLogger(__name__) # or use 'main' + log = logging.getLogger(__name__) log.info(f"Logging level set to: {logging.getLevelName(log_level)}") - # Suppress overly verbose messages from libraries if needed (e.g., cv2) - # logging.getLogger('cv2').setLevel(logging.WARNING) -# Use module-level logger after configuration log = logging.getLogger(__name__) @@ -63,7 +93,6 @@ log = logging.getLogger(__name__) # Keep setup_arg_parser as is, it's only used when running main.py directly def setup_arg_parser(): """Sets up and returns the command-line argument parser.""" - # Determine a sensible default worker count default_workers = 1 try: # Use half the cores, but at least 1, max maybe 8-16? Depends on task nature. @@ -71,34 +100,34 @@ def setup_arg_parser(): cores = os.cpu_count() if cores: default_workers = max(1, cores // 2) - # Cap default workers? Maybe not necessary, let user decide via flag. - # default_workers = min(default_workers, 8) # Example cap except NotImplementedError: log.warning("Could not detect CPU count, defaulting workers to 1.") parser = argparse.ArgumentParser( description="Process asset files (ZIPs or folders) into a standardized library format using presets.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message + formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( "input_paths", metavar="INPUT_PATH", type=str, - nargs='+', # Requires one or more input paths - help="Path(s) to the input ZIP file(s) or folder(s) containing assets." + nargs='*', + default=[], + help="Path(s) to the input ZIP file(s) or folder(s) containing assets (Required for CLI mode)." ) parser.add_argument( "-p", "--preset", type=str, - required=True, - help="Name of the configuration preset (e.g., 'poliigon') located in the 'presets' directory (without .json extension)." + required=False, + default=None, + help="Name of the configuration preset (Required for CLI mode)." ) parser.add_argument( "-o", "--output-dir", type=str, - required=False, # No longer required - default=None, # Default is None, will check core_config later - help="Override the default base output directory defined in config.py." # Updated help + required=False, + default=None, + help="Override the default base output directory defined in config.py." ) parser.add_argument( "-w", "--workers", @@ -108,7 +137,7 @@ def setup_arg_parser(): ) parser.add_argument( "-v", "--verbose", - action="store_true", # Makes it a flag, value is True if present + action="store_true", help="Enable detailed DEBUG level logging for troubleshooting." ) parser.add_argument( @@ -128,515 +157,407 @@ def setup_arg_parser(): default=None, help="Path to the .blend file for creating/updating materials. Overrides config.py default." ) - # Potential future flags: - # parser.add_argument("--log-file", type=str, default=None, help="Path to save log output to a file.") + parser.add_argument( + "--gui", + action="store_true", + help="Force launch in GUI mode, ignoring other arguments." + ) return parser -# --- Worker Function --- -def process_single_asset_wrapper(input_path_str: str, preset_name: str, output_dir_str: str, overwrite: bool, verbose: bool) -> Tuple[str, str, Optional[str]]: - """ - Wrapper function for processing a single input path (which might contain multiple assets) - in a separate process. Handles instantiation of Configuration and AssetProcessor, - passes the overwrite flag, catches errors, and interprets the multi-asset status dictionary. +# --- Worker Runnable for Thread Pool --- +class TaskSignals(QObject): + finished = Signal(str, str, object) # rule_input_path, status, result/error - Ensures logging is configured for the worker process. +class ProcessingTask(QRunnable): + """Wraps a call to processing_engine.process for execution in a thread pool.""" - Returns: - Tuple[str, str, Optional[str]]: - - input_path_str: The original input path processed. - - overall_status_string: A single status string summarizing the outcome - ("processed", "skipped", "failed", "partial_success"). - - error_message_or_None: An error message if failures occurred, potentially - listing failed assets. - """ - # Explicitly configure logging for this worker process - worker_log = logging.getLogger(f"Worker_{os.getpid()}") # Log with worker PID - if not logging.root.handlers: - logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)-8s] %(name)s: %(message)s') - worker_log.setLevel(logging.DEBUG if verbose else logging.INFO) - if verbose: - logging.root.setLevel(logging.DEBUG) + def __init__(self, engine: ProcessingEngine, rule: SourceRule, workspace_path: Path, output_base_path: Path): + super().__init__() + self.engine = engine + self.rule = rule + self.workspace_path = workspace_path + self.output_base_path = output_base_path + self.signals = TaskSignals() - input_path_obj = Path(input_path_str) - input_name = input_path_obj.name + @Slot() # Decorator required for QRunnable's run method + def run(self): + """Prepares input files and executes the engine's process method.""" + log.info(f"Worker Thread: Starting processing for rule: {self.rule.input_path}") + log.debug(f"DEBUG: Rule passed to ProcessingTask.run: {self.rule}") + status = "failed" + result_or_error = None + prepared_workspace_path = None # Initialize path for prepared content outside try - try: - worker_log.info(f"Starting processing attempt for input: {input_name}") - config = Configuration(preset_name) - output_base_path = Path(output_dir_str) - - processor = AssetProcessor(input_path_obj, config, output_base_path, overwrite=overwrite) - # processor.process() now returns a Dict[str, List[str]] - status_dict = processor.process() - - # --- Interpret the status dictionary --- - processed_assets = status_dict.get("processed", []) - skipped_assets = status_dict.get("skipped", []) - failed_assets = status_dict.get("failed", []) - - overall_status_string = "failed" # Default - error_message = None - - if failed_assets: - overall_status_string = "failed" - error_message = f"Failed assets within {input_name}: {', '.join(failed_assets)}" - worker_log.error(error_message) # Log the failure details - elif processed_assets: - overall_status_string = "processed" - # Check for partial success (mix of processed/skipped and failed should be caught above) - if skipped_assets: - worker_log.info(f"Input '{input_name}' processed with some assets skipped. Processed: {processed_assets}, Skipped: {skipped_assets}") - else: - worker_log.info(f"Input '{input_name}' processed successfully. Assets: {processed_assets}") - elif skipped_assets: - overall_status_string = "skipped" - worker_log.info(f"Input '{input_name}' skipped (all contained assets already exist). Assets: {skipped_assets}") - else: - # Should not happen if input contained files, but handle as failure. - worker_log.warning(f"Input '{input_name}' resulted in no processed, skipped, or failed assets. Reporting as failed.") - overall_status_string = "failed" - error_message = f"No assets processed, skipped, or failed within {input_name}." - - - return (input_path_str, overall_status_string, error_message) - - except (ConfigurationError, AssetProcessingError) as e: - # Catch errors during processor setup or the process() call itself if it raises before returning dict - worker_log.error(f"Processing failed for input '{input_name}': {type(e).__name__}: {e}") - return (input_path_str, "failed", f"{type(e).__name__}: {e}") - except Exception as e: - # Catch any other unexpected errors - worker_log.exception(f"Unexpected worker failure processing input '{input_name}': {e}") - return (input_path_str, "failed", f"Unexpected Worker Error: {e}") - - -# --- Core Processing Function --- -def run_processing( - valid_inputs: List[str], - preset_name: str, - output_dir_for_processor: str, - overwrite: bool, - num_workers: int, - verbose: bool # Add verbose parameter here -) -> Dict: - """ - Executes the core asset processing logic using a process pool. - - Args: - valid_inputs: List of validated input file/directory paths (strings). - preset_name: Name of the preset to use. - output_dir_for_processor: Absolute path string for the output base directory. - overwrite: Boolean flag to force reprocessing. - num_workers: Maximum number of worker processes. - verbose: Boolean flag for verbose logging. - - Returns: - A dictionary containing processing results: - { - "processed": int, - "skipped": int, - "failed": int, - "results_list": List[Tuple[str, str, Optional[str]]] # (input_path, status, error_msg) - } - """ - log.info(f"Processing {len(valid_inputs)} asset(s) using preset '{preset_name}' with up to {num_workers} worker(s)...") - results_list = [] - successful_processed_count = 0 - skipped_count = 0 - failed_count = 0 - - # Ensure at least one worker - num_workers = max(1, num_workers) - - # Using ProcessPoolExecutor is generally good if AssetProcessor tasks are CPU-bound. - # If tasks are mostly I/O bound, ThreadPoolExecutor might be sufficient. - # Important: Ensure Configuration and AssetProcessor are "pickleable". - try: - with ProcessPoolExecutor(max_workers=num_workers) as executor: - # Create futures - futures = {} - log.debug(f"Submitting {len(valid_inputs)} tasks...") - # Removed the 1-second delay for potentially faster submission in non-CLI use - for i, input_path in enumerate(valid_inputs): - log.debug(f"Submitting task {i+1}/{len(valid_inputs)} for: {Path(input_path).name}") - future = executor.submit( - process_single_asset_wrapper, - input_path, - preset_name, - output_dir_for_processor, - overwrite, - verbose # Pass the verbose flag - ) - futures[future] = input_path # Store future -> input_path mapping - - # Process completed futures - for i, future in enumerate(as_completed(futures), 1): - input_path = futures[future] - asset_name = Path(input_path).name - log.info(f"--- [{i}/{len(valid_inputs)}] Worker finished attempt for: {asset_name} ---") - try: - # Get result tuple: (input_path_str, status_string, error_message_or_None) - result_tuple = future.result() - results_list.append(result_tuple) - input_path_res, status, err_msg = result_tuple - - # Increment counters based on status - if status == "processed": - successful_processed_count += 1 - elif status == "skipped": - skipped_count += 1 - elif status == "failed": - failed_count += 1 - else: # Should not happen, but log as warning/failure - log.warning(f"Unknown status '{status}' received for {asset_name}. Counting as failed.") - failed_count += 1 - - except Exception as e: - # Catch errors if the future itself fails (e.g., worker process crashed hard) - log.exception(f"Critical worker failure for {asset_name}: {e}") - results_list.append((input_path, "failed", f"Worker process crashed: {e}")) - failed_count += 1 # Count crashes as failures - - except Exception as pool_exc: - log.exception(f"An error occurred with the process pool: {pool_exc}") - # Re-raise or handle as appropriate for the calling context (monitor.py) - # For now, log and return current counts - return { - "processed": successful_processed_count, - "skipped": skipped_count, - "failed": failed_count + (len(valid_inputs) - len(results_list)), # Count unprocessed as failed - "results_list": results_list, - "pool_error": str(pool_exc) # Add pool error info - } - - return { - "processed": successful_processed_count, - "skipped": skipped_count, - "failed": failed_count, - "results_list": results_list - } - - -# --- Blender Script Execution Helper --- -def run_blender_script(blender_exe_path: str, blend_file_path: str, python_script_path: str, asset_root_dir: str): - """ - Executes a Python script within Blender in the background. - - Args: - blender_exe_path: Path to the Blender executable. - blend_file_path: Path to the .blend file to open. - python_script_path: Path to the Python script to execute within Blender. - asset_root_dir: Path to the processed asset library root directory (passed to the script). - - Returns: - True if the script executed successfully (return code 0), False otherwise. - """ - log.info(f"Attempting to run Blender script: {Path(python_script_path).name} on {Path(blend_file_path).name}") - - # Ensure paths are absolute strings for subprocess - blender_exe_path = str(Path(blender_exe_path).resolve()) - blend_file_path = str(Path(blend_file_path).resolve()) - python_script_path = str(Path(python_script_path).resolve()) - asset_root_dir = str(Path(asset_root_dir).resolve()) - - # Construct the command arguments - # -b: Run in background (no UI) - # -S: Save the file after running the script - # --python: Execute the specified Python script - # --: Separator, arguments after this are passed to the Python script's sys.argv - command = [ - blender_exe_path, - "-b", # Run in background - blend_file_path, - "--python", python_script_path, - "--", # Pass subsequent arguments to the script - asset_root_dir, - "-S" # Save the blend file after script execution - ] - - log.debug(f"Executing Blender command: {' '.join(command)}") # Log the command for debugging - - try: - # Execute the command - # capture_output=True captures stdout and stderr - # text=True decodes stdout/stderr as text - # check=False prevents raising CalledProcessError on non-zero exit codes - result = subprocess.run(command, capture_output=True, text=True, check=False) - - # Log results - log.info(f"Blender script '{Path(python_script_path).name}' finished with exit code: {result.returncode}") - if result.stdout: - log.debug(f"Blender stdout:\n{result.stdout.strip()}") - if result.stderr: - # Log stderr as warning or error depending on return code - if result.returncode != 0: - log.error(f"Blender stderr:\n{result.stderr.strip()}") - else: - log.warning(f"Blender stderr (Return Code 0):\n{result.stderr.strip()}") # Log stderr even on success as scripts might print warnings - - return result.returncode == 0 - - except FileNotFoundError: - log.error(f"Blender executable not found at: {blender_exe_path}") - return False - except Exception as e: - log.exception(f"An unexpected error occurred while running Blender script '{Path(python_script_path).name}': {e}") - return False - - -# --- Main Execution (for CLI usage) --- -def main(): - """Parses arguments, sets up logging, runs processing, and reports summary.""" - parser = setup_arg_parser() - args = parser.parse_args() - - # Setup logging based on verbosity argument *before* logging status messages - setup_logging(args.verbose) - - start_time = time.time() - log.info("Asset Processor Script Started (CLI Mode)") - - # --- Validate Input Paths --- - valid_inputs = [] - for p_str in args.input_paths: - p = Path(p_str) - if p.exists(): - suffix = p.suffix.lower() - if p.is_dir() or (p.is_file() and suffix in ['.zip', '.rar', '.7z']): - valid_inputs.append(p_str) # Store the original string path - else: - log.warning(f"Input is not a directory or a supported archive type (.zip, .rar, .7z), skipping: {p_str}") - else: - log.warning(f"Input path not found, skipping: {p_str}") - - if not valid_inputs: - log.error("No valid input paths found. Exiting.") - sys.exit(1) # Exit with error code - - # --- Determine Output Directory --- - output_dir_str = args.output_dir # Get value from args (might be None) - if not output_dir_str: - log.debug("Output directory not specified via -o, reading default from config.py.") try: - output_dir_str = getattr(core_config_module, 'OUTPUT_BASE_DIR', None) - if not output_dir_str: - log.error("Output directory not specified with -o and OUTPUT_BASE_DIR not found or empty in config.py. Exiting.") - sys.exit(1) - log.info(f"Using default output directory from config.py: {output_dir_str}") + # --- 1. Prepare Input Workspace using Utility Function --- + # The utility function creates the temp dir, prepares it, and returns its path. + # It raises exceptions on failure (FileNotFoundError, ValueError, zipfile.BadZipFile, OSError). + prepared_workspace_path = prepare_processing_workspace(self.rule.input_path) + log.info(f"Workspace prepared successfully at: {prepared_workspace_path}") + + # --- DEBUG: List files in prepared workspace --- + try: + log.debug(f"Listing contents of prepared workspace: {prepared_workspace_path}") + for item in prepared_workspace_path.rglob('*'): + log.debug(f" Found item: {item.relative_to(prepared_workspace_path)}") + except Exception as list_err: + log.error(f"Error listing prepared workspace contents: {list_err}") + # --- END DEBUG --- + # --- 2. Execute Processing Engine --- + log.info(f"Calling ProcessingEngine.process with rule for input: {self.rule.input_path}, prepared workspace: {prepared_workspace_path}, output: {self.output_base_path}") + log.debug(f" Rule Details: {self.rule}") + + # --- Calculate SHA5 and Incrementing Value --- + config = self.engine.config_obj + archive_path = self.rule.input_path + output_dir = self.output_base_path # This is already a Path object from App.on_processing_requested + + sha5_value = None + try: + archive_path_obj = Path(archive_path) + if archive_path_obj.is_file(): + log.debug(f"Calculating SHA256 for file: {archive_path_obj}") + full_sha = calculate_sha256(archive_path_obj) + if full_sha: + sha5_value = full_sha[:5] + log.info(f"Calculated SHA5 for {archive_path}: {sha5_value}") + else: + log.warning(f"SHA256 calculation returned None for {archive_path}") + elif archive_path_obj.is_dir(): + log.debug(f"Input path {archive_path} is a directory, skipping SHA5 calculation.") + else: + log.warning(f"Input path {archive_path} is not a valid file or directory for SHA5 calculation.") + except FileNotFoundError: + log.error(f"SHA5 calculation failed: File not found at {archive_path}") + except Exception as e: + log.exception(f"Error calculating SHA5 for {archive_path}: {e}") + + next_increment_str = None + try: + # output_dir should already be a Path object + pattern = getattr(config, 'output_directory_pattern', None) + if pattern: + log.debug(f"Calculating next incrementing value for dir: {output_dir} using pattern: {pattern}") + next_increment_str = get_next_incrementing_value(output_dir, pattern) + log.info(f"Calculated next incrementing value for {output_dir}: {next_increment_str}") + else: + log.warning(f"Cannot calculate incrementing value: 'output_directory_pattern' not found in configuration for preset {config.preset_name}") + except Exception as e: + log.exception(f"Error calculating next incrementing value for {output_dir}: {e}") + # --- End Calculation --- + + log.info(f"Calling engine.process with sha5='{sha5_value}', incrementing_value='{next_increment_str}'") + result_or_error = self.engine.process( + self.rule, + workspace_path=prepared_workspace_path, + output_base_path=self.output_base_path, + incrementing_value=next_increment_str, + sha5_value=sha5_value + ) + status = "processed" # Assume success if no exception + log.info(f"Worker Thread: Finished processing for rule: {self.rule.input_path}, Status: {status}") + # Signal emission moved to finally block + + except (FileNotFoundError, ValueError, zipfile.BadZipFile, OSError) as prep_error: + log.exception(f"Worker Thread: Error preparing workspace for rule {self.rule.input_path}: {prep_error}") + status = "failed_preparation" + result_or_error = str(prep_error) + # Signal emission moved to finally block + except Exception as proc_error: + log.exception(f"Worker Thread: Error during engine processing for rule {self.rule.input_path}: {proc_error}") + status = "failed_processing" + result_or_error = str(proc_error) + # Signal emission moved to finally block + finally: + # --- Emit finished signal regardless of success or failure --- + try: + self.signals.finished.emit(str(self.rule.input_path), status, result_or_error) + log.debug(f"Worker Thread: Emitted finished signal for {self.rule.input_path} with status {status}") + except Exception as sig_err: + log.error(f"Worker Thread: Error emitting finished signal for {self.rule.input_path}: {sig_err}") + + # --- 3. Cleanup Workspace --- + # Use the path returned by the utility function for cleanup + if prepared_workspace_path and prepared_workspace_path.exists(): + try: + log.info(f"Cleaning up temporary workspace: {prepared_workspace_path}") + shutil.rmtree(prepared_workspace_path) + except OSError as cleanup_error: + log.error(f"Worker Thread: Failed to cleanup temporary workspace {prepared_workspace_path}: {cleanup_error}") + + + + +# --- Main Application Class (Integrates GUI and Engine) --- +class App(QObject): + # Signal emitted when all queued processing tasks are complete + all_tasks_finished = Signal(int, int, int) # processed_count, skipped_count, failed_count (Placeholder counts for now) + + def __init__(self): + super().__init__() + self.config_obj = None + self.processing_engine = None + self.main_window = None + self.thread_pool = QThreadPool() + self._active_tasks_count = 0 + self._task_results = {"processed": 0, "skipped": 0, "failed": 0} + log.info(f"Maximum threads for pool: {self.thread_pool.maxThreadCount()}") + + self._load_config() + self._init_engine() + self._init_gui() + + def _load_config(self): + """Loads the base configuration using a default preset.""" + # The actual preset name comes from the GUI request later, but the engine + # needs an initial valid configuration object. + try: + # Find the first available preset to use as a default + preset_dir = Path(__file__).parent / "Presets" + default_preset_name = None + if preset_dir.is_dir(): + presets = sorted([f.stem for f in preset_dir.glob("*.json") if f.is_file() and not f.name.startswith('_')]) + if presets: + default_preset_name = presets[0] + log.info(f"Using first available preset as default for initial config: '{default_preset_name}'") + + if not default_preset_name: + # Fallback or raise error if no presets found + log.error("No presets found in the 'Presets' directory. Cannot initialize default configuration.") + # Option 1: Raise an error + raise ConfigurationError("No presets found to load default configuration.") + + self.config_obj = Configuration(preset_name=default_preset_name) + log.info(f"Base configuration loaded using default preset '{default_preset_name}'.") + except ConfigurationError as e: + log.error(f"Fatal: Failed to load base configuration using default preset: {e}") + # In a real app, show this error to the user before exiting + sys.exit(1) except Exception as e: - log.error(f"Could not read OUTPUT_BASE_DIR from config.py: {e}") + log.exception(f"Fatal: Unexpected error loading configuration: {e}") sys.exit(1) - # --- Resolve Output Path (Handles Relative Paths Explicitly) --- - output_path_obj: Path - if os.path.isabs(output_dir_str): - output_path_obj = Path(output_dir_str) - log.info(f"Using absolute output directory: {output_path_obj}") - else: - # Path() interprets relative paths against CWD by default - output_path_obj = Path(output_dir_str) - log.info(f"Using relative output directory '{output_dir_str}'. Resolved against CWD to: {output_path_obj.resolve()}") - - # --- Validate and Setup Output Directory --- - try: - # Resolve to ensure we have an absolute path for consistency and creation - resolved_output_dir = output_path_obj.resolve() - log.info(f"Ensuring output directory exists: {resolved_output_dir}") - resolved_output_dir.mkdir(parents=True, exist_ok=True) - # Use the resolved absolute path string for the processor - output_dir_for_processor = str(resolved_output_dir) - except Exception as e: - log.error(f"Cannot create or access output directory '{resolved_output_dir}': {e}", exc_info=True) - sys.exit(1) - - # --- Check Preset Existence (Basic Check) --- - preset_dir = Path(__file__).parent / "presets" - preset_file = preset_dir / f"{args.preset}.json" - if not preset_file.is_file(): - log.error(f"Preset file not found: {preset_file}") - log.error(f"Ensure a file named '{args.preset}.json' exists in the directory: {preset_dir.resolve()}") - sys.exit(1) - - # --- Execute Processing via the new function --- - processing_results = run_processing( - valid_inputs=valid_inputs, - preset_name=args.preset, - output_dir_for_processor=output_dir_for_processor, - overwrite=args.overwrite, - num_workers=args.workers, - verbose=args.verbose # Pass the verbose flag - ) - - # --- Report Summary --- - duration = time.time() - start_time - successful_processed_count = processing_results["processed"] - skipped_count = processing_results["skipped"] - failed_count = processing_results["failed"] - results_list = processing_results["results_list"] - - log.info("=" * 40) - log.info("Processing Summary") - log.info(f" Duration: {duration:.2f} seconds") - log.info(f" Assets Attempted: {len(valid_inputs)}") - log.info(f" Successfully Processed: {successful_processed_count}") - log.info(f" Skipped (Already Existed): {skipped_count}") - log.info(f" Failed: {failed_count}") - - if processing_results.get("pool_error"): - log.error(f" Process Pool Error: {processing_results['pool_error']}") - # Ensure failed count reflects pool error if it happened - if failed_count == 0 and successful_processed_count == 0 and skipped_count == 0: - failed_count = len(valid_inputs) # Assume all failed if pool died early - - exit_code = 0 - if failed_count > 0: - log.warning("Failures occurred:") - # Iterate through results to show specific errors for failed items - for input_path, status, err_msg in results_list: - if status == "failed": - log.warning(f" - {Path(input_path).name}: {err_msg}") - exit_code = 1 # Exit with error code if failures occurred - else: - # Consider skipped assets as a form of success for the overall run exit code - if successful_processed_count > 0 or skipped_count > 0: - log.info("All assets processed or skipped successfully.") - exit_code = 0 # Exit code 0 indicates success (including skips) + def _init_engine(self): + """Initializes the ProcessingEngine.""" + if self.config_obj: + try: + self.processing_engine = ProcessingEngine(self.config_obj) + log.info("ProcessingEngine initialized.") + except Exception as e: + log.exception(f"Fatal: Failed to initialize ProcessingEngine: {e}") + # Show error and exit + sys.exit(1) else: - # This case might happen if all inputs were invalid initially - log.warning("No assets were processed, skipped, or failed (check input validation logs).") - exit_code = 0 # Still exit 0 as the script itself didn't crash + log.error("Fatal: Cannot initialize ProcessingEngine without configuration.") + sys.exit(1) - # --- Blender Script Execution (Optional) --- - run_nodegroups = False - run_materials = False - nodegroup_blend_path = None - materials_blend_path = None - blender_exe = None - - # 1. Find Blender Executable - try: - blender_exe_config = getattr(core_config_module, 'BLENDER_EXECUTABLE_PATH', None) - if blender_exe_config: - # Check if the path in config exists - if Path(blender_exe_config).is_file(): - blender_exe = str(Path(blender_exe_config).resolve()) - log.info(f"Using Blender executable from config: {blender_exe}") - else: - # Try finding it in PATH if config path is invalid - log.warning(f"Blender path in config not found: '{blender_exe_config}'. Trying to find 'blender' in PATH.") - blender_exe = shutil.which("blender") - if blender_exe: - log.info(f"Found Blender executable in PATH: {blender_exe}") - else: - log.warning("Could not find 'blender' in system PATH.") + def _init_gui(self): + """Initializes the MainWindow and connects signals.""" + if self.processing_engine: + self.main_window = MainWindow() # MainWindow now part of the App + # Connect the signal from the GUI to the App's slot using QueuedConnection + # Connect the signal from the MainWindow (which is triggered by the panel) to the App's slot + connection_success = self.main_window.start_backend_processing.connect(self.on_processing_requested, Qt.ConnectionType.QueuedConnection) + log.info(f"DEBUG: Connection result for processing_requested (Queued): {connection_success}") + if not connection_success: + log.error("*********************************************************") + log.error("FATAL: Failed to connect MainWindow.processing_requested signal to App.on_processing_requested slot!") + log.error("*********************************************************") + # Connect the App's completion signal to the MainWindow's slot + self.all_tasks_finished.connect(self.main_window.on_processing_finished) + log.info("MainWindow initialized and signals connected.") else: - # Try finding it in PATH if not set in config - log.info("BLENDER_EXECUTABLE_PATH not set in config. Trying to find 'blender' in PATH.") - blender_exe = shutil.which("blender") - if blender_exe: - log.info(f"Found Blender executable in PATH: {blender_exe}") - else: - log.warning("Could not find 'blender' in system PATH.") + log.error("Fatal: Cannot initialize MainWindow without ProcessingEngine.") + sys.exit(1) - if not blender_exe: - log.warning("Blender executable not found or configured. Skipping Blender script execution.") + @Slot(list, dict) # Slot to receive List[SourceRule] and processing_settings dict + def on_processing_requested(self, source_rules: list, processing_settings: dict): + log.debug("DEBUG: App.on_processing_requested slot entered.") + """Handles the processing request from the GUI.""" + log.info(f"Received processing request for {len(source_rules)} rule sets.") + log.info(f"DEBUG: Rules received by on_processing_requested: {source_rules}") + log.info(f"VERIFY: App.on_processing_requested received {len(source_rules)} rules.") + for i, rule in enumerate(source_rules): + log.debug(f" VERIFY Rule {i}: Input='{rule.input_path}', Assets={len(rule.assets)}") + if not self.processing_engine: + log.error("Processing engine not available. Cannot process request.") + self.main_window.statusBar().showMessage("Error: Processing Engine not ready.", 5000) + return + if not source_rules: + log.warning("Processing requested with an empty rule list.") + self.main_window.statusBar().showMessage("No rules to process.", 3000) + return - except Exception as e: - log.error(f"Error checking Blender executable path: {e}") - blender_exe = None # Ensure it's None on error + # Reset task counter and results for this batch + self._active_tasks_count = len(source_rules) + self._task_results = {"processed": 0, "skipped": 0, "failed": 0} + log.debug(f"Initialized active task count to: {self._active_tasks_count}") - # 2. Determine Blend File Paths if Blender Exe is available - if blender_exe: - # Nodegroup Blend Path - nodegroup_blend_arg = args.nodegroup_blend - if nodegroup_blend_arg: - p = Path(nodegroup_blend_arg) - if p.is_file() and p.suffix.lower() == '.blend': - nodegroup_blend_path = str(p.resolve()) - log.info(f"Using nodegroup blend file from argument: {nodegroup_blend_path}") - else: - log.warning(f"Invalid nodegroup blend file path from argument: '{nodegroup_blend_arg}'. Ignoring.") - else: - default_ng_path_str = getattr(core_config_module, 'DEFAULT_NODEGROUP_BLEND_PATH', None) - if default_ng_path_str: - p = Path(default_ng_path_str) - if p.is_file() and p.suffix.lower() == '.blend': - nodegroup_blend_path = str(p.resolve()) - log.info(f"Using default nodegroup blend file from config: {nodegroup_blend_path}") - else: - log.warning(f"Invalid default nodegroup blend file path in config: '{default_ng_path_str}'. Ignoring.") + # Update GUI progress bar/status via MainPanelWidget + self.main_window.main_panel_widget.progress_bar.setMaximum(len(source_rules)) + self.main_window.main_panel_widget.progress_bar.setValue(0) + self.main_window.main_panel_widget.progress_bar.setFormat(f"0/{len(source_rules)} tasks") - # Materials Blend Path - materials_blend_arg = args.materials_blend - if materials_blend_arg: - p = Path(materials_blend_arg) - if p.is_file() and p.suffix.lower() == '.blend': - materials_blend_path = str(p.resolve()) - log.info(f"Using materials blend file from argument: {materials_blend_path}") - else: - log.warning(f"Invalid materials blend file path from argument: '{materials_blend_arg}'. Ignoring.") - else: - default_mat_path_str = getattr(core_config_module, 'DEFAULT_MATERIALS_BLEND_PATH', None) - if default_mat_path_str: - p = Path(default_mat_path_str) - if p.is_file() and p.suffix.lower() == '.blend': - materials_blend_path = str(p.resolve()) - log.info(f"Using default materials blend file from config: {materials_blend_path}") - else: - log.warning(f"Invalid default materials blend file path in config: '{default_mat_path_str}'. Ignoring.") + # --- Get paths needed for ProcessingTask --- + try: + # Get output_dir from processing_settings passed from autotest.py + output_base_path_str = processing_settings.get("output_dir") + log.info(f"APP_DEBUG: Received output_dir in processing_settings: {output_base_path_str}") - # 3. Execute Scripts if Paths are Valid - if blender_exe: - script_dir = Path(__file__).parent / "blenderscripts" - nodegroup_script_path = script_dir / "create_nodegroups.py" - materials_script_path = script_dir / "create_materials.py" - asset_output_root = output_dir_for_processor # Use the resolved output dir + if not output_base_path_str: + log.error("Cannot queue tasks: Output directory path is empty in processing_settings.") + # self.main_window.statusBar().showMessage("Error: Output directory cannot be empty.", 5000) # GUI specific + return + output_base_path = Path(output_base_path_str) + # Basic validation - check if it's likely a valid path structure (doesn't guarantee existence/writability here) + if not output_base_path.is_absolute(): + # Or attempt to resolve relative to workspace? For now, require absolute from GUI. + log.warning(f"Output path '{output_base_path}' is not absolute. Processing might fail if relative path is not handled correctly by engine.") + # Consider resolving: output_base_path = Path.cwd() / output_base_path # If relative paths are allowed - if nodegroup_blend_path: - if nodegroup_script_path.is_file(): - log.info("-" * 40) - log.info("Starting Blender Node Group Script Execution...") - success_ng = run_blender_script( - blender_exe_path=blender_exe, - blend_file_path=nodegroup_blend_path, - python_script_path=str(nodegroup_script_path), - asset_root_dir=asset_output_root + # Define workspace path (assuming main.py is in the project root) + workspace_path = Path(__file__).parent.resolve() + log.debug(f"Using Workspace Path: {workspace_path}") + log.debug(f"Using Output Base Path: {output_base_path}") + + except Exception as e: + log.exception(f"Error getting/validating paths for processing task: {e}") + self.main_window.statusBar().showMessage(f"Error preparing paths: {e}", 5000) + return + # --- End Get paths --- + + + # Set max threads based on GUI setting + worker_count = processing_settings.get('workers', 1) + self.thread_pool.setMaxThreadCount(worker_count) + log.info(f"Set thread pool max workers to: {worker_count}") + + # Queue tasks in the thread pool + log.debug("DEBUG: Entering task queuing loop.") + for i, rule in enumerate(source_rules): + if isinstance(rule, SourceRule): + log.info(f"DEBUG Task {i+1}: Rule Input='{rule.input_path}', Supplier ID='{getattr(rule, 'supplier_identifier', 'Not Set')}', Preset='{getattr(rule, 'preset_name', 'Not Set')}'") + log.debug(f"DEBUG: Preparing to queue task {i+1}/{len(source_rules)} for rule: {rule.input_path}") + + # --- Create a new Configuration and Engine instance for this specific task --- + task_engine = None + try: + # Get preset name from the rule, fallback to app's default if missing + preset_name_for_task = getattr(rule, 'preset_name', None) + if not preset_name_for_task: + log.warning(f"Task {i+1} (Rule: {rule.input_path}): SourceRule missing preset_name. Falling back to default preset '{self.config_obj.preset_name}'.") + preset_name_for_task = self.config_obj.preset_name + + task_config = Configuration(preset_name=preset_name_for_task) + task_engine = ProcessingEngine(task_config) + log.debug(f"Task {i+1}: Created new ProcessingEngine instance with preset '{preset_name_for_task}'.") + + except ConfigurationError as config_err: + log.error(f"Task {i+1} (Rule: {rule.input_path}): Failed to load configuration for preset '{preset_name_for_task}': {config_err}. Skipping task.") + self._active_tasks_count -= 1 # Decrement count as this task won't run + self._task_results["failed"] += 1 + # Optionally update GUI status for this specific rule + self.main_window.update_file_status(str(rule.input_path), "failed", f"Config Error: {config_err}") + continue # Skip to the next rule + except Exception as engine_err: + log.exception(f"Task {i+1} (Rule: {rule.input_path}): Failed to initialize ProcessingEngine for preset '{preset_name_for_task}': {engine_err}. Skipping task.") + self._active_tasks_count -= 1 # Decrement count + self._task_results["failed"] += 1 + self.main_window.update_file_status(str(rule.input_path), "failed", f"Engine Init Error: {engine_err}") + continue # Skip to the next rule + + if task_engine is None: # Should not happen if exceptions are caught, but safety check + log.error(f"Task {i+1} (Rule: {rule.input_path}): Engine is None after initialization attempt. Skipping task.") + self._active_tasks_count -= 1 # Decrement count + self._task_results["failed"] += 1 + self.main_window.update_file_status(str(rule.input_path), "failed", "Engine initialization failed (unknown reason).") + continue # Skip to the next rule + # --- End Engine Instantiation --- + + task = ProcessingTask( + engine=task_engine, + rule=rule, + workspace_path=workspace_path, + output_base_path=output_base_path # This is Path(output_base_path_str) ) - if not success_ng: - log.error("Blender node group script execution failed.") - # Optionally change exit code if Blender script fails? - # exit_code = 1 - log.info("Finished Blender Node Group Script Execution.") - log.info("-" * 40) + log.info(f"APP_DEBUG: Passing to ProcessingTask: output_base_path = {output_base_path}") + task.signals.finished.connect(self._on_task_finished) + log.debug(f"DEBUG: Calling thread_pool.start() for task {i+1}") + self.thread_pool.start(task) + log.debug(f"DEBUG: Returned from thread_pool.start() for task {i+1}") else: - log.error(f"Node group script not found: {nodegroup_script_path}") + log.warning(f"Skipping invalid item (index {i}) in rule list: {type(rule)}") - if materials_blend_path: - if materials_script_path.is_file(): - log.info("-" * 40) - log.info("Starting Blender Material Script Execution...") - success_mat = run_blender_script( - blender_exe_path=blender_exe, - blend_file_path=materials_blend_path, - python_script_path=str(materials_script_path), - asset_root_dir=asset_output_root - ) - if not success_mat: - log.error("Blender material script execution failed.") - # Optionally change exit code if Blender script fails? - # exit_code = 1 - log.info("Finished Blender Material Script Execution.") - log.info("-" * 40) - else: - log.error(f"Material script not found: {materials_script_path}") + log.info(f"Queued {len(source_rules)} processing tasks (finished loop).") + # GUI status already updated in MainWindow when signal was emitted - # --- Final Exit --- - log.info("Asset Processor Script Finished.") - sys.exit(exit_code) + # --- Slot to handle completion of individual tasks --- + @Slot(str, str, object) + def _on_task_finished(self, rule_input_path, status, result_or_error): + """Handles the 'finished' signal from a ProcessingTask.""" + log.info(f"Task finished signal received for {rule_input_path}. Status: {status}") + self._active_tasks_count -= 1 + log.debug(f"Active tasks remaining: {self._active_tasks_count}") + + # Update overall results (basic counts for now) + if status == "processed": + self._task_results["processed"] += 1 + elif status == "skipped": # Assuming engine might return 'skipped' status eventually + self._task_results["skipped"] += 1 + else: # Count all other statuses (failed_preparation, failed_processing) as failed + self._task_results["failed"] += 1 + + # Update progress bar via MainPanelWidget + total_tasks = self.main_window.main_panel_widget.progress_bar.maximum() + completed_tasks = total_tasks - self._active_tasks_count + self.main_window.main_panel_widget.update_progress_bar(completed_tasks, total_tasks) # Use MainPanelWidget's method + + # Update status for the specific file in the GUI (if needed) + + if self._active_tasks_count == 0: + log.info("All processing tasks finished.") + # Emit the signal with the final counts + self.all_tasks_finished.emit( + self._task_results["processed"], + self._task_results["skipped"], + self._task_results["failed"] + ) + elif self._active_tasks_count < 0: + log.error("Error: Active task count went below zero!") # Should not happen + + def run(self): + """Shows the main window.""" + if self.main_window: + self.main_window.show() + log.info("Application started. Showing main window.") + else: + log.error("Cannot run application, MainWindow not initialized.") if __name__ == "__main__": - # This ensures the main() function runs only when the script is executed directly - # Important for multiprocessing to work correctly on some platforms (like Windows) - main() \ No newline at end of file + parser = setup_arg_parser() + args = parser.parse_args() + + setup_logging(args.verbose) + + # Determine mode based on presence of required CLI args + if args.input_paths or args.preset: + # If either input_paths or preset is provided, assume CLI mode + # run_cli will handle validation that *both* are actually present + log.info("CLI arguments detected (input_paths or preset), attempting CLI mode.") + run_cli(args) + else: + # If neither input_paths nor preset is provided, run GUI mode + log.info("No required CLI arguments detected, starting GUI mode.") + # --- Run the GUI Application --- + try: + qt_app = QApplication(sys.argv) + + app_instance = App() + app_instance.run() + + sys.exit(qt_app.exec()) + except Exception as gui_exc: + log.exception(f"An error occurred during GUI startup or execution: {gui_exc}") + sys.exit(1) diff --git a/monitor.py b/monitor.py index 56d10d7..b550cd0 100644 --- a/monitor.py +++ b/monitor.py @@ -1,4 +1,3 @@ -# monitor.py import os import sys @@ -6,21 +5,34 @@ import time import logging import re import shutil +import tempfile from pathlib import Path +from concurrent.futures import ThreadPoolExecutor from watchdog.observers.polling import PollingObserver as Observer # Use polling for better compatibility from watchdog.events import FileSystemEventHandler, FileCreatedEvent -# --- Import from local modules --- -try: - # Assuming main.py is in the same directory - from main import run_processing, setup_logging, ConfigurationError, AssetProcessingError -except ImportError as e: - print(f"ERROR: Failed to import required functions/classes from main.py: {e}") - print("Ensure main.py is in the same directory as monitor.py.") - sys.exit(1) +from utils.hash_utils import calculate_sha256 +from utils.path_utils import get_next_incrementing_value + +from configuration import load_config, ConfigurationError +from processing_engine import ProcessingEngine, ProcessingError +from rule_structure import SourceRule +try: + from utils.workspace_utils import prepare_processing_workspace, WorkspaceError +except ImportError: + log = logging.getLogger(__name__) # Need logger early for this message + log.warning("Could not import workspace_utils. Workspace preparation/cleanup might fail.") + # Define dummy functions/exceptions if import fails to avoid NameErrors later, + # but log prominently. + def prepare_processing_workspace(archive_path: Path) -> Path: + log.error("prepare_processing_workspace is not available!") + # Create a dummy temp dir to allow code flow, but it won't be the real one + return Path(tempfile.mkdtemp(prefix="dummy_workspace_")) + class WorkspaceError(Exception): pass + +from utils.prediction_utils import generate_source_rule_from_archive, PredictionError + -# --- Configuration --- -# Read from environment variables with defaults INPUT_DIR = Path(os.environ.get('INPUT_DIR', '/data/input')) OUTPUT_DIR = Path(os.environ.get('OUTPUT_DIR', '/data/output')) PROCESSED_DIR = Path(os.environ.get('PROCESSED_DIR', '/data/processed')) @@ -32,14 +44,13 @@ PROCESS_DELAY = int(os.environ.get('PROCESS_DELAY', '2')) DEFAULT_WORKERS = max(1, os.cpu_count() // 2 if os.cpu_count() else 1) NUM_WORKERS = int(os.environ.get('NUM_WORKERS', str(DEFAULT_WORKERS))) -# --- Logging Setup --- +# Configure logging (ensure logger is available before potential import errors) log_level = getattr(logging, LOG_LEVEL_STR, logging.INFO) -# Use the setup_logging from main.py but configure the level directly -# We don't have a 'verbose' flag here, so call basicConfig directly log_format = '%(asctime)s [%(levelname)-8s] %(name)s: %(message)s' date_format = '%Y-%m-%d %H:%M:%S' logging.basicConfig(level=log_level, format=log_format, datefmt=date_format, handlers=[logging.StreamHandler(sys.stdout)]) -log = logging.getLogger("monitor") +log = logging.getLogger("monitor") # Define logger after basicConfig + log.info(f"Logging level set to: {logging.getLevelName(log_level)}") log.info(f"Monitoring Input Directory: {INPUT_DIR}") log.info(f"Output Directory: {OUTPUT_DIR}") @@ -50,21 +61,8 @@ log.info(f"Processing Delay: {PROCESS_DELAY}s") log.info(f"Max Workers: {NUM_WORKERS}") -# --- Preset Validation --- -PRESET_DIR = Path(__file__).parent / "Presets" -PRESET_FILENAME_REGEX = re.compile(r"^\[?([a-zA-Z0-9_-]+)\]?_.*\.(zip|rar|7z)$", re.IGNORECASE) +SUPPORTED_SUFFIXES = ['.zip', '.rar', '.7z'] -def validate_preset(preset_name: str) -> bool: - """Checks if the preset JSON file exists.""" - if not preset_name: - return False - preset_file = PRESET_DIR / f"{preset_name}.json" - exists = preset_file.is_file() - if not exists: - log.warning(f"Preset file not found: {preset_file}") - return exists - -# --- Watchdog Event Handler --- class ZipHandler(FileSystemEventHandler): """Handles file system events for new ZIP files.""" @@ -73,102 +71,52 @@ class ZipHandler(FileSystemEventHandler): self.output_dir = output_dir.resolve() self.processed_dir = processed_dir.resolve() self.error_dir = error_dir.resolve() - # Ensure target directories exist self.output_dir.mkdir(parents=True, exist_ok=True) self.processed_dir.mkdir(parents=True, exist_ok=True) self.error_dir.mkdir(parents=True, exist_ok=True) - log.info("Handler initialized, target directories ensured.") + + self.executor = ThreadPoolExecutor(max_workers=NUM_WORKERS) + log.info(f"Handler initialized, target directories ensured. ThreadPoolExecutor started with {NUM_WORKERS} workers.") def on_created(self, event: FileCreatedEvent): - """Called when a file or directory is created.""" + """Called when a file or directory is created. Submits task to executor.""" if event.is_directory: return src_path = Path(event.src_path) log.debug(f"File creation event detected: {src_path}") - # Check if the file has a supported archive extension - supported_suffixes = ['.zip', '.rar', '.7z'] - if src_path.suffix.lower() not in supported_suffixes: + if src_path.suffix.lower() not in SUPPORTED_SUFFIXES: log.debug(f"Ignoring file with unsupported extension: {src_path.name}") return - log.info(f"Detected new ZIP file: {src_path.name}. Waiting {PROCESS_DELAY}s before processing...") - time.sleep(PROCESS_DELAY) + log.info(f"Detected new archive: {src_path.name}. Waiting {PROCESS_DELAY}s before queueing...") + time.sleep(PROCESS_DELAY) # Wait for file write to complete - # Re-check if file still exists (might have been temporary) + # Re-check if file still exists (might have been temporary or moved quickly) if not src_path.exists(): log.warning(f"File disappeared after delay: {src_path.name}") return - log.info(f"Processing file: {src_path.name}") + log.info(f"Queueing processing task for: {src_path.name}") + self.executor.submit( + _process_archive_task, + archive_path=src_path, + output_dir=self.output_dir, + processed_dir=self.processed_dir, + error_dir=self.error_dir + ) - # --- Extract Preset Name --- - match = PRESET_FILENAME_REGEX.match(src_path.name) - if not match: - log.warning(f"Filename '{src_path.name}' does not match expected format '[preset]_filename.zip'. Ignoring.") - # Optionally move to an 'ignored' or 'error' directory? For now, leave it. - return + def shutdown(self): + """Shuts down the thread pool executor.""" + log.info("Shutting down thread pool executor...") + self.executor.shutdown(wait=True) + log.info("Executor shut down.") - preset_name = match.group(1) - log.info(f"Extracted preset name: '{preset_name}' from {src_path.name}") - - # --- Validate Preset --- - if not validate_preset(preset_name): - log.error(f"Preset '{preset_name}' is not valid (missing {PRESET_DIR / f'{preset_name}.json'}). Ignoring file {src_path.name}.") - # Move to error dir if preset is invalid? Let's do that. - self.move_file(src_path, self.error_dir, "invalid_preset") - return - - # --- Run Processing --- - try: - log.info(f"Starting asset processing for '{src_path.name}' using preset '{preset_name}'...") - # run_processing expects a list of inputs - results = run_processing( - valid_inputs=[str(src_path)], - preset_name=preset_name, - output_dir_for_processor=str(self.output_dir), # Pass absolute output path - overwrite=False, # Default to no overwrite for monitored files? Or make configurable? Let's default to False. - num_workers=NUM_WORKERS - ) - - # --- Handle Results --- - # Check overall status based on counts - processed = results.get("processed", 0) - skipped = results.get("skipped", 0) - failed = results.get("failed", 0) - pool_error = results.get("pool_error") - - if pool_error: - log.error(f"Processing pool error for {src_path.name}: {pool_error}") - self.move_file(src_path, self.error_dir, "pool_error") - elif failed > 0: - log.error(f"Processing failed for {src_path.name}. Check worker logs for details.") - # Log specific errors if available in results_list - for res_path, status, err_msg in results.get("results_list", []): - if status == "failed": - log.error(f" - Failure reason: {err_msg}") - self.move_file(src_path, self.error_dir, "processing_failed") - elif processed > 0: - log.info(f"Successfully processed {src_path.name}.") - self.move_file(src_path, self.processed_dir, "processed") - elif skipped > 0: - log.info(f"Processing skipped for {src_path.name} (likely already exists).") - self.move_file(src_path, self.processed_dir, "skipped") - else: - # Should not happen if input was valid zip - log.warning(f"Processing finished for {src_path.name} with unexpected status (0 processed, 0 skipped, 0 failed). Moving to error dir.") - self.move_file(src_path, self.error_dir, "unknown_status") - - except (ConfigurationError, AssetProcessingError) as e: - log.error(f"Asset processing error for {src_path.name}: {e}", exc_info=True) - self.move_file(src_path, self.error_dir, "processing_exception") - except Exception as e: - log.exception(f"Unexpected error during processing trigger for {src_path.name}: {e}") - self.move_file(src_path, self.error_dir, "monitor_exception") - - - def move_file(self, src: Path, dest_dir: Path, reason: str): + # move_file remains largely the same, but called from _process_archive_task now + # We make it static or move it outside the class if _process_archive_task is outside + @staticmethod + def move_file(src: Path, dest_dir: Path, reason: str): """Safely moves a file, handling potential name collisions.""" if not src.exists(): log.warning(f"Source file {src} does not exist, cannot move for reason: {reason}.") @@ -190,9 +138,131 @@ class ZipHandler(FileSystemEventHandler): log.exception(f"Failed to move file {src.name} to {dest_dir}: {e}") -# --- Main Monitor Loop --- +def _process_archive_task(archive_path: Path, output_dir: Path, processed_dir: Path, error_dir: Path): + """ + Task executed by the ThreadPoolExecutor to process a single archive file. + """ + log.info(f"[Task:{archive_path.name}] Starting processing.") + temp_workspace_path: Optional[Path] = None + config = None + source_rule = None + move_reason = "unknown_error" # Default reason if early exit + + try: + log.debug(f"[Task:{archive_path.name}] Loading configuration...") + # Assuming load_config() loads the main app config (e.g., app_settings.json) + # and potentially merges preset defaults or paths. Adjust if needed. + config = load_config() # Might need path argument depending on implementation + if not config: + raise ConfigurationError("Failed to load application configuration.") + log.debug(f"[Task:{archive_path.name}] Configuration loaded.") + + log.debug(f"[Task:{archive_path.name}] Generating source rule prediction...") + # This function now handles preset extraction and validation internally + source_rule = generate_source_rule_from_archive(archive_path, config) + log.info(f"[Task:{archive_path.name}] SourceRule generated successfully.") + + log.debug(f"[Task:{archive_path.name}] Preparing processing workspace...") + # This utility should handle extraction and return the temp dir path + temp_workspace_path = prepare_processing_workspace(archive_path) + log.info(f"[Task:{archive_path.name}] Workspace prepared at: {temp_workspace_path}") + + log.debug(f"[Task:{archive_path.name}] Initializing Processing Engine...") + engine = ProcessingEngine(config=config, output_base_dir=output_dir) + log.info(f"[Task:{archive_path.name}] Running Processing Engine...") + + + sha5_value = None + try: + if archive_path.is_file(): + log.debug(f"[Task:{archive_path.name}] Calculating SHA256 for file: {archive_path}") + full_sha = calculate_sha256(archive_path) + if full_sha: + sha5_value = full_sha[:5] + log.info(f"[Task:{archive_path.name}] Calculated SHA5: {sha5_value}") + else: + log.warning(f"[Task:{archive_path.name}] SHA256 calculation returned None for {archive_path}") + # No need to check is_dir here as monitor only processes files based on SUPPORTED_SUFFIXES + else: + log.warning(f"[Task:{archive_path.name}] Input path {archive_path} is not a valid file for SHA5 calculation (unexpected).") + except FileNotFoundError: + log.error(f"[Task:{archive_path.name}] SHA5 calculation failed: File not found at {archive_path}") + except Exception as e: + log.exception(f"[Task:{archive_path.name}] Error calculating SHA5 for {archive_path}: {e}") + + next_increment_str = None + try: + # Assuming config object has 'output_directory_pattern' attribute/key + pattern = getattr(config, 'output_directory_pattern', None) # Use getattr for safety + if pattern: + log.debug(f"[Task:{archive_path.name}] Calculating next incrementing value for dir: {output_dir} using pattern: {pattern}") + next_increment_str = get_next_incrementing_value(output_dir, pattern) + log.info(f"[Task:{archive_path.name}] Calculated next incrementing value: {next_increment_str}") + else: + # Check if config is a dict as fallback (depends on load_config implementation) + if isinstance(config, dict): + pattern = config.get('output_directory_pattern') + if pattern: + log.debug(f"[Task:{archive_path.name}] Calculating next incrementing value for dir: {output_dir} using pattern (from dict): {pattern}") + next_increment_str = get_next_incrementing_value(output_dir, pattern) + log.info(f"[Task:{archive_path.name}] Calculated next incrementing value (from dict): {next_increment_str}") + else: + log.warning(f"[Task:{archive_path.name}] Cannot calculate incrementing value: 'output_directory_pattern' not found in configuration dictionary.") + else: + log.warning(f"[Task:{archive_path.name}] Cannot calculate incrementing value: 'output_directory_pattern' not found in configuration object.") + except Exception as e: + log.exception(f"[Task:{archive_path.name}] Error calculating next incrementing value for {output_dir}: {e}") + + # The engine uses the source_rule to guide processing on the workspace files + log.info(f"[Task:{archive_path.name}] Calling engine.run with sha5='{sha5_value}', incrementing_value='{next_increment_str}'") + engine.run( + workspace_path=temp_workspace_path, + source_rule=source_rule, + incrementing_value=next_increment_str, + sha5_value=sha5_value + ) + log.info(f"[Task:{archive_path.name}] Processing Engine finished successfully.") + move_reason = "processed" + + # If engine.run completes without exception, assume success for now. + # More granular results could be returned by engine.run if needed. + # Moving is handled outside the main try block based on move_reason + + # TODO: Add call to utils.blender_utils.run_blender_script if needed later + + + except FileNotFoundError as e: + log.error(f"[Task:{archive_path.name}] Prerequisite file not found: {e}") + move_reason = "file_not_found" + except (ConfigurationError, PredictionError, WorkspaceError, ProcessingError) as e: + log.error(f"[Task:{archive_path.name}] Processing failed: {e}", exc_info=True) + move_reason = f"{type(e).__name__.lower()}" # e.g., "predictionerror" + except Exception as e: + log.exception(f"[Task:{archive_path.name}] An unexpected error occurred during processing: {e}") + move_reason = "unexpected_exception" + + finally: + log.debug(f"[Task:{archive_path.name}] Moving original archive based on outcome: {move_reason}") + dest_dir = processed_dir if move_reason == "processed" else error_dir + try: + ZipHandler.move_file(archive_path, dest_dir, move_reason) + except Exception as move_err: + log.exception(f"[Task:{archive_path.name}] CRITICAL: Failed to move archive file {archive_path} after processing: {move_err}") + + if temp_workspace_path and temp_workspace_path.exists(): + log.debug(f"[Task:{archive_path.name}] Cleaning up workspace: {temp_workspace_path}") + try: + shutil.rmtree(temp_workspace_path) + log.info(f"[Task:{archive_path.name}] Workspace cleaned up successfully.") + except OSError as e: + log.error(f"[Task:{archive_path.name}] Error removing temporary workspace {temp_workspace_path}: {e}", exc_info=True) + elif temp_workspace_path: + log.warning(f"[Task:{archive_path.name}] Temporary workspace path recorded but not found for cleanup: {temp_workspace_path}") + + log.info(f"[Task:{archive_path.name}] Processing task finished with status: {move_reason}") + + if __name__ == "__main__": - # Ensure input directory exists if not INPUT_DIR.is_dir(): log.error(f"Input directory does not exist or is not a directory: {INPUT_DIR}") log.error("Please create the directory or mount a volume correctly.") @@ -211,11 +281,13 @@ if __name__ == "__main__": # Keep the main thread alive, observer runs in background thread time.sleep(1) except KeyboardInterrupt: - log.info("Keyboard interrupt received, stopping monitor...") + log.info("Keyboard interrupt received, stopping monitor and executor...") observer.stop() + event_handler.shutdown() # Gracefully shutdown the executor except Exception as e: log.exception(f"An unexpected error occurred in the main loop: {e}") observer.stop() + event_handler.shutdown() # Ensure shutdown on other exceptions too observer.join() log.info("Monitor stopped.") \ No newline at end of file diff --git a/processing/pipeline/asset_context.py b/processing/pipeline/asset_context.py new file mode 100644 index 0000000..f6363e5 --- /dev/null +++ b/processing/pipeline/asset_context.py @@ -0,0 +1,106 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional + +from rule_structure import AssetRule, FileRule, SourceRule +from configuration import Configuration + +# Imports needed for new dataclasses +import numpy as np +from typing import Any, Tuple, Union + +# --- Stage Input/Output Dataclasses --- + +# Item types for PrepareProcessingItemsStage output +@dataclass +class MergeTaskDefinition: + """Represents a merge task identified by PrepareProcessingItemsStage.""" + task_data: Dict # The original task data from context.merged_image_tasks + task_key: str # e.g., "merged_task_0" + +# Output for RegularMapProcessorStage +@dataclass +class ProcessedRegularMapData: + processed_image_data: np.ndarray + final_internal_map_type: str + source_file_path: Path + original_bit_depth: Optional[int] + original_dimensions: Optional[Tuple[int, int]] # (width, height) + transformations_applied: List[str] + status: str = "Processed" + error_message: Optional[str] = None + +# Output for MergedTaskProcessorStage +@dataclass +class ProcessedMergedMapData: + merged_image_data: np.ndarray + output_map_type: str # Internal type + source_bit_depths: List[int] + final_dimensions: Optional[Tuple[int, int]] # (width, height) + transformations_applied_to_inputs: Dict[str, List[str]] # Map type -> list of transforms + status: str = "Processed" + error_message: Optional[str] = None + +# Input for InitialScalingStage +@dataclass +class InitialScalingInput: + image_data: np.ndarray + original_dimensions: Optional[Tuple[int, int]] # (width, height) + # Configuration needed + initial_scaling_mode: str + +# Output for InitialScalingStage +@dataclass +class InitialScalingOutput: + scaled_image_data: np.ndarray + scaling_applied: bool + final_dimensions: Tuple[int, int] # (width, height) + +# Input for SaveVariantsStage +@dataclass +class SaveVariantsInput: + image_data: np.ndarray # Final data (potentially scaled) + internal_map_type: str # Final internal type (e.g., MAP_ROUGH, MAP_COL-1) + source_bit_depth_info: List[int] + # Configuration needed + output_filename_pattern_tokens: Dict[str, Any] + image_resolutions: List[int] + file_type_defs: Dict[str, Dict] + output_format_8bit: str + output_format_16bit_primary: str + output_format_16bit_fallback: str + png_compression_level: int + jpg_quality: int + output_filename_pattern: str + resolution_threshold_for_jpg: Optional[int] # Added for JPG conversion + +# Output for SaveVariantsStage +@dataclass +class SaveVariantsOutput: + saved_files_details: List[Dict] + status: str = "Processed" + error_message: Optional[str] = None + +# Add a field to AssetProcessingContext for the prepared items +@dataclass +class AssetProcessingContext: + source_rule: SourceRule + asset_rule: AssetRule + workspace_path: Path + engine_temp_dir: Path + output_base_path: Path + effective_supplier: Optional[str] + asset_metadata: Dict + processed_maps_details: Dict[str, Dict] # Will store final results per item_key + merged_maps_details: Dict[str, Dict] # This might become redundant? Keep for now. + files_to_process: List[FileRule] + loaded_data_cache: Dict + config_obj: Configuration + status_flags: Dict + incrementing_value: Optional[str] + sha5_value: Optional[str] # Keep existing fields + # New field for prepared items + processing_items: Optional[List[Union[FileRule, MergeTaskDefinition]]] = None + # Temporary storage during pipeline execution (managed by orchestrator) + # Keys could be FileRule object hash/id or MergeTaskDefinition task_key + intermediate_results: Optional[Dict[Any, Union[ProcessedRegularMapData, ProcessedMergedMapData, InitialScalingOutput]]] = None \ No newline at end of file diff --git a/processing/pipeline/orchestrator.py b/processing/pipeline/orchestrator.py new file mode 100644 index 0000000..6c8fe7a --- /dev/null +++ b/processing/pipeline/orchestrator.py @@ -0,0 +1,439 @@ +# --- Imports --- +import logging +import shutil +import tempfile +from pathlib import Path +from typing import List, Dict, Optional, Any, Union # Added Any, Union + +import numpy as np # Added numpy + +from configuration import Configuration +from rule_structure import SourceRule, AssetRule, FileRule # Added FileRule + +# Import new context classes and stages +from .asset_context import ( + AssetProcessingContext, + MergeTaskDefinition, + ProcessedRegularMapData, + ProcessedMergedMapData, + InitialScalingInput, + InitialScalingOutput, + SaveVariantsInput, + SaveVariantsOutput, +) +from .stages.base_stage import ProcessingStage +# Import the new stages we created +from .stages.prepare_processing_items import PrepareProcessingItemsStage +from .stages.regular_map_processor import RegularMapProcessorStage +from .stages.merged_task_processor import MergedTaskProcessorStage +from .stages.initial_scaling import InitialScalingStage +from .stages.save_variants import SaveVariantsStage + +log = logging.getLogger(__name__) + +# --- PipelineOrchestrator Class --- + +class PipelineOrchestrator: + """ + Orchestrates the processing of assets based on source rules and a series of processing stages. + Manages the overall flow, including the core item processing sequence. + """ + + def __init__(self, config_obj: Configuration, + pre_item_stages: List[ProcessingStage], + post_item_stages: List[ProcessingStage]): + """ + Initializes the PipelineOrchestrator. + + Args: + config_obj: The main configuration object. + pre_item_stages: Stages to run before the core item processing loop. + post_item_stages: Stages to run after the core item processing loop. + """ + self.config_obj: Configuration = config_obj + self.pre_item_stages: List[ProcessingStage] = pre_item_stages + self.post_item_stages: List[ProcessingStage] = post_item_stages + # Instantiate the core item processing stages internally + self._prepare_stage = PrepareProcessingItemsStage() + self._regular_processor_stage = RegularMapProcessorStage() + self._merged_processor_stage = MergedTaskProcessorStage() + self._scaling_stage = InitialScalingStage() + self._save_stage = SaveVariantsStage() + + def _execute_specific_stages( + self, context: AssetProcessingContext, + stages_to_run: List[ProcessingStage], + stage_group_name: str, + stop_on_skip: bool = True + ) -> AssetProcessingContext: + """Executes a specific list of stages.""" + asset_name = context.asset_rule.asset_name if context.asset_rule else "Unknown" + log.debug(f"Asset '{asset_name}': Executing {stage_group_name} stages...") + for stage in stages_to_run: + stage_name = stage.__class__.__name__ + log.debug(f"Asset '{asset_name}': Executing {stage_group_name} stage: {stage_name}") + try: + # Check if stage expects context directly or specific input + # For now, assume outer stages take context directly + # This might need refinement if outer stages also adopt Input/Output pattern + context = stage.execute(context) + except Exception as e: + log.error(f"Asset '{asset_name}': Error during outer stage '{stage_name}': {e}", exc_info=True) + context.status_flags["asset_failed"] = True + context.status_flags["asset_failed_stage"] = stage_name + context.status_flags["asset_failed_reason"] = str(e) + # Update overall metadata immediately on outer stage failure + context.asset_metadata["status"] = f"Failed: Error in stage {stage_name}" + context.asset_metadata["error_message"] = str(e) + break # Stop processing outer stages for this asset on error + + if stop_on_skip and context.status_flags.get("skip_asset"): + log.info(f"Asset '{asset_name}': Skipped by outer stage '{stage_name}'. Reason: {context.status_flags.get('skip_reason', 'N/A')}") + break # Skip remaining outer stages for this asset + return context + + def process_source_rule( + self, + source_rule: SourceRule, + workspace_path: Path, + output_base_path: Path, + overwrite: bool, + incrementing_value: Optional[str], + sha5_value: Optional[str] # Keep param name consistent for now + ) -> Dict[str, List[str]]: + """ + Processes a single source rule, applying pre-processing stages, + the core item processing loop (Prepare, Process, Scale, Save), + and post-processing stages. + """ + overall_status: Dict[str, List[str]] = { + "processed": [], + "skipped": [], + "failed": [], + } + engine_temp_dir_path: Optional[Path] = None + + try: + # --- Setup Temporary Directory --- + temp_dir_path_str = tempfile.mkdtemp(prefix=self.config_obj.temp_dir_prefix) + engine_temp_dir_path = Path(temp_dir_path_str) + log.debug(f"PipelineOrchestrator created temporary directory: {engine_temp_dir_path}") + + # --- Process Each Asset Rule --- + for asset_rule in source_rule.assets: + asset_name = asset_rule.asset_name + log.info(f"Orchestrator: Processing asset '{asset_name}'") + + # --- Initialize Asset Context --- + context = AssetProcessingContext( + source_rule=source_rule, + asset_rule=asset_rule, + workspace_path=workspace_path, + engine_temp_dir=engine_temp_dir_path, + output_base_path=output_base_path, + effective_supplier=None, + asset_metadata={}, + processed_maps_details={}, # Final results per item + merged_maps_details={}, # Keep for potential backward compat or other uses? + files_to_process=[], # Populated by FileRuleFilterStage (assumed in outer_stages) + loaded_data_cache={}, + config_obj=self.config_obj, + status_flags={"skip_asset": False, "asset_failed": False}, + incrementing_value=incrementing_value, + sha5_value=sha5_value, + processing_items=[], # Initialize new fields + intermediate_results={} + ) + + # --- Execute Pre-Item-Processing Outer Stages --- + # (e.g., MetadataInit, SupplierDet, FileRuleFilter, GlossToRough, NormalInvert) + # Identify which outer stages run before the item loop + # This requires knowing the intended order. Assume all run before for now. + context = self._execute_specific_stages(context, self.pre_item_stages, "pre-item", stop_on_skip=True) + + # Check if asset should be skipped or failed after pre-processing + if context.status_flags.get("asset_failed"): + log.error(f"Asset '{asset_name}': Failed during pre-processing stage '{context.status_flags.get('asset_failed_stage', 'Unknown')}'. Skipping item processing.") + overall_status["failed"].append(f"{asset_name} (Failed in {context.status_flags.get('asset_failed_stage', 'Pre-Processing')})") + continue # Move to the next asset rule + + if context.status_flags.get("skip_asset"): + log.info(f"Asset '{asset_name}': Skipped during pre-processing. Skipping item processing.") + overall_status["skipped"].append(asset_name) + continue # Move to the next asset rule + + # --- Prepare Processing Items --- + log.debug(f"Asset '{asset_name}': Preparing processing items...") + try: + log.info(f"ORCHESTRATOR_TRACE: Asset '{asset_name}': Attempting to call _prepare_stage.execute(). Current context.status_flags: {context.status_flags}") + # Prepare stage modifies context directly + context = self._prepare_stage.execute(context) + log.info(f"ORCHESTRATOR_TRACE: Asset '{asset_name}': Successfully RETURNED from _prepare_stage.execute(). context.processing_items count: {len(context.processing_items) if context.processing_items is not None else 'None'}. context.status_flags: {context.status_flags}") + except Exception as e: + log.error(f"ORCHESTRATOR_TRACE: Asset '{asset_name}': EXCEPTION during _prepare_stage.execute(): {e}", exc_info=True) + context.status_flags["asset_failed"] = True + context.status_flags["asset_failed_stage"] = "PrepareProcessingItemsStage" + context.status_flags["asset_failed_reason"] = str(e) + overall_status["failed"].append(f"{asset_name} (Failed in Prepare Items)") + continue # Move to next asset + + if context.status_flags.get('prepare_items_failed'): + log.error(f"Asset '{asset_name}': Failed during item preparation. Reason: {context.status_flags.get('prepare_items_failed_reason', 'Unknown')}. Skipping item processing loop.") + overall_status["failed"].append(f"{asset_name} (Failed Prepare Items: {context.status_flags.get('prepare_items_failed_reason', 'Unknown')})") + continue # Move to next asset + + if not context.processing_items: + log.info(f"Asset '{asset_name}': No items to process after preparation stage.") + # Status will be determined at the end + + # --- Core Item Processing Loop --- + log.info("ORCHESTRATOR: Starting processing items loop for asset '%s'", asset_name) # Corrected indentation and message + log.info(f"Asset '{asset_name}': Starting core item processing loop for {len(context.processing_items)} items...") + asset_had_item_errors = False + for item_index, item in enumerate(context.processing_items): + item_key: Any = None # Key for storing results (FileRule object or task_key string) + item_log_prefix = f"Asset '{asset_name}', Item {item_index + 1}/{len(context.processing_items)}" + processed_data: Optional[Union[ProcessedRegularMapData, ProcessedMergedMapData]] = None + scaled_data_output: Optional[InitialScalingOutput] = None # Store output object + saved_data: Optional[SaveVariantsOutput] = None + item_status = "Failed" # Default item status + current_image_data: Optional[np.ndarray] = None # Track current image data ref + + try: + # 1. Process (Load/Merge + Transform) + if isinstance(item, FileRule): + if item.item_type == 'EXTRA': + log.debug(f"{item_log_prefix}: Skipping image processing for EXTRA FileRule '{item.file_path}'.") + # Add a basic entry to processed_maps_details to acknowledge it was seen + context.processed_maps_details[item.file_path] = { + "status": "Skipped (EXTRA file)", + "internal_map_type": "EXTRA", + "source_file": str(item.file_path) + } + continue # Skip to the next item + item_key = item.file_path # Use file_path string as key + log.debug(f"{item_log_prefix}: Processing FileRule '{item.file_path}'...") + processed_data = self._regular_processor_stage.execute(context, item) + elif isinstance(item, MergeTaskDefinition): + item_key = item.task_key # Use task_key string as key + log.info(f"{item_log_prefix}: Executing MergedTaskProcessorStage for MergeTask '{item_key}'...") # Log call + processed_data = self._merged_processor_stage.execute(context, item) + # Log status/error from merge processor + if processed_data: + log.info(f"{item_log_prefix}: MergedTaskProcessorStage result - Status: {processed_data.status}, Error: {processed_data.error_message}") + else: + log.warning(f"{item_log_prefix}: MergedTaskProcessorStage returned None for MergeTask '{item_key}'.") + else: + log.warning(f"{item_log_prefix}: Unknown item type '{type(item)}'. Skipping.") + item_key = f"unknown_item_{item_index}" + context.processed_maps_details[item_key] = {"status": "Skipped", "notes": f"Unknown item type {type(item)}"} + asset_had_item_errors = True + continue # Next item + + # Check for processing failure + if not processed_data or processed_data.status != "Processed": + error_msg = processed_data.error_message if processed_data else "Processor returned None" + log.error(f"{item_log_prefix}: Failed during processing stage. Error: {error_msg}") + context.processed_maps_details[item_key] = {"status": "Failed", "notes": f"Processing Error: {error_msg}", "stage": processed_data.__class__.__name__ if processed_data else "UnknownProcessor"} + asset_had_item_errors = True + continue # Next item + + # Store intermediate result & get current image data + context.intermediate_results[item_key] = processed_data + current_image_data = processed_data.processed_image_data if isinstance(processed_data, ProcessedRegularMapData) else processed_data.merged_image_data + current_dimensions = processed_data.original_dimensions if isinstance(processed_data, ProcessedRegularMapData) else processed_data.final_dimensions + + # 2. Scale (Optional) + scaling_mode = getattr(context.config_obj, "INITIAL_SCALING_MODE", "NONE") + if scaling_mode != "NONE" and current_image_data is not None and current_image_data.size > 0: + if isinstance(item, MergeTaskDefinition): # Log scaling call for merge tasks + log.info(f"{item_log_prefix}: Calling InitialScalingStage for MergeTask '{item_key}' (Mode: {scaling_mode})...") + log.debug(f"{item_log_prefix}: Applying initial scaling (Mode: {scaling_mode})...") + scale_input = InitialScalingInput( + image_data=current_image_data, + original_dimensions=current_dimensions, # Pass original/merged dims + initial_scaling_mode=scaling_mode + ) + scaled_data_output = self._scaling_stage.execute(scale_input) + # Update intermediate result and current image data reference + context.intermediate_results[item_key] = scaled_data_output # Overwrite previous intermediate + current_image_data = scaled_data_output.scaled_image_data # Use scaled data for saving + log.debug(f"{item_log_prefix}: Scaling applied: {scaled_data_output.scaling_applied}. New Dims: {scaled_data_output.final_dimensions}") + else: + log.debug(f"{item_log_prefix}: Initial scaling skipped (Mode: NONE or empty image).") + # Create dummy output if scaling skipped, using current dims + final_dims = current_dimensions if current_dimensions else (current_image_data.shape[1], current_image_data.shape[0]) if current_image_data is not None else (0,0) + scaled_data_output = InitialScalingOutput(scaled_image_data=current_image_data, scaling_applied=False, final_dimensions=final_dims) + + + # 3. Save Variants + if current_image_data is None or current_image_data.size == 0: + log.warning(f"{item_log_prefix}: Skipping save stage because image data is empty.") + context.processed_maps_details[item_key] = {"status": "Skipped", "notes": "No image data to save", "stage": "SaveVariantsStage"} + # Don't mark as asset error, just skip this item's saving + continue # Next item + + if isinstance(item, MergeTaskDefinition): # Log save call for merge tasks + log.info(f"{item_log_prefix}: Calling SaveVariantsStage for MergeTask '{item_key}'...") + log.debug(f"{item_log_prefix}: Saving variants...") + # Prepare input for save stage + internal_map_type = processed_data.final_internal_map_type if isinstance(processed_data, ProcessedRegularMapData) else processed_data.output_map_type + source_bit_depth = [processed_data.original_bit_depth] if isinstance(processed_data, ProcessedRegularMapData) and processed_data.original_bit_depth is not None else processed_data.source_bit_depths if isinstance(processed_data, ProcessedMergedMapData) else [8] # Default bit depth if unknown + + # Construct filename tokens (ensure temp dir is used) + output_filename_tokens = { + 'asset_name': asset_name, + 'output_base_directory': context.engine_temp_dir, # Save variants to temp dir + # Add other tokens from context/config as needed by the pattern + 'supplier': context.effective_supplier or 'UnknownSupplier', + } + + # Log the value being read for the threshold before creating the input object + log.info(f"ORCHESTRATOR_DEBUG: Reading RESOLUTION_THRESHOLD_FOR_JPG from config for SaveVariantsInput: {getattr(context.config_obj, 'RESOLUTION_THRESHOLD_FOR_JPG', None)}") + save_input = SaveVariantsInput( + image_data=current_image_data, # Use potentially scaled data + internal_map_type=internal_map_type, + source_bit_depth_info=source_bit_depth, + output_filename_pattern_tokens=output_filename_tokens, + # Pass config values needed by save stage + image_resolutions=context.config_obj.image_resolutions, + file_type_defs=getattr(context.config_obj, "FILE_TYPE_DEFINITIONS", {}), + output_format_8bit=context.config_obj.get_8bit_output_format(), + output_format_16bit_primary=context.config_obj.get_16bit_output_formats()[0], + output_format_16bit_fallback=context.config_obj.get_16bit_output_formats()[1], + png_compression_level=context.config_obj.png_compression_level, + jpg_quality=context.config_obj.jpg_quality, + output_filename_pattern=context.config_obj.output_filename_pattern, + resolution_threshold_for_jpg=getattr(context.config_obj, "resolution_threshold_for_jpg", None) # Corrected case + ) + saved_data = self._save_stage.execute(save_input) + # Log saved_data for merge tasks + if isinstance(item, MergeTaskDefinition): + log.info(f"{item_log_prefix}: SaveVariantsStage result for MergeTask '{item_key}' - Status: {saved_data.status if saved_data else 'N/A'}, Saved Files: {len(saved_data.saved_files_details) if saved_data else 0}") + + # Check save status and finalize item result + if saved_data and saved_data.status.startswith("Processed"): + item_status = saved_data.status # e.g., "Processed" or "Processed (No Output)" + log.info(f"{item_log_prefix}: Item successfully processed and saved. Status: {item_status}") + # Populate final details for this item + final_details = { + "status": item_status, + "saved_files_info": saved_data.saved_files_details, # List of dicts from save util + "internal_map_type": internal_map_type, + "original_dimensions": processed_data.original_dimensions if isinstance(processed_data, ProcessedRegularMapData) else None, + "final_dimensions": scaled_data_output.final_dimensions if scaled_data_output else current_dimensions, + "transformations": processed_data.transformations_applied if isinstance(processed_data, ProcessedRegularMapData) else processed_data.transformations_applied_to_inputs, + # Add source file if regular map + "source_file": str(processed_data.source_file_path) if isinstance(processed_data, ProcessedRegularMapData) else None, + } + # Log final details addition for merge tasks + if isinstance(item, MergeTaskDefinition): + log.info(f"{item_log_prefix}: Adding final details to context.processed_maps_details for MergeTask '{item_key}'. Details: {final_details}") + context.processed_maps_details[item_key] = final_details + else: + error_msg = saved_data.error_message if saved_data else "Save stage returned None" + log.error(f"{item_log_prefix}: Failed during save stage. Error: {error_msg}") + context.processed_maps_details[item_key] = {"status": "Failed", "notes": f"Save Error: {error_msg}", "stage": "SaveVariantsStage"} + asset_had_item_errors = True + item_status = "Failed" # Ensure item status reflects failure + + except Exception as e: + log.exception(f"{item_log_prefix}: Unhandled exception during item processing loop: {e}") + # Ensure details are recorded even on unhandled exception + if item_key is not None: + context.processed_maps_details[item_key] = {"status": "Failed", "notes": f"Unhandled Loop Error: {e}", "stage": "OrchestratorLoop"} + else: + log.error(f"Asset '{asset_name}': Unhandled exception in item loop before item key was set.") + asset_had_item_errors = True + item_status = "Failed" + # Optionally break loop or continue? Continue for now to process other items. + + log.info("ORCHESTRATOR: Finished processing items loop for asset '%s'", asset_name) + log.info(f"Asset '{asset_name}': Finished core item processing loop.") + + # --- Execute Post-Item-Processing Outer Stages --- + # (e.g., OutputOrganization, MetadataFinalizationSave) + # Identify which outer stages run after the item loop + # This needs better handling based on stage purpose. Assume none run after for now. + if not context.status_flags.get("asset_failed"): + log.info("ORCHESTRATOR: Executing post-item-processing outer stages for asset '%s'", asset_name) + context = self._execute_specific_stages(context, self.post_item_stages, "post-item", stop_on_skip=False) + + # --- Final Asset Status Determination --- + final_asset_status = "Unknown" + fail_reason = "" + if context.status_flags.get("asset_failed"): + final_asset_status = "Failed" + fail_reason = f"(Failed in {context.status_flags.get('asset_failed_stage', 'Unknown Stage')}: {context.status_flags.get('asset_failed_reason', 'Unknown Reason')})" + elif context.status_flags.get("skip_asset"): + final_asset_status = "Skipped" + fail_reason = f"(Skipped: {context.status_flags.get('skip_reason', 'Unknown Reason')})" + elif asset_had_item_errors: + final_asset_status = "Failed" + fail_reason = "(One or more items failed)" + elif not context.processing_items: + # No items prepared, no errors -> consider skipped or processed based on definition? + final_asset_status = "Skipped" # Or "Processed (No Items)" + fail_reason = "(No items to process)" + elif not context.processed_maps_details and context.processing_items: + # Items were prepared, but none resulted in processed_maps_details entry + final_asset_status = "Skipped" # Or Failed? + fail_reason = "(All processing items skipped or failed internally)" + elif context.processed_maps_details: + # Check if all items in processed_maps_details are actually processed successfully + all_processed_ok = all( + str(details.get("status", "")).startswith("Processed") + for details in context.processed_maps_details.values() + ) + some_processed_ok = any( + str(details.get("status", "")).startswith("Processed") + for details in context.processed_maps_details.values() + ) + + if all_processed_ok: + final_asset_status = "Processed" + elif some_processed_ok: + final_asset_status = "Partial" # Introduce a partial status? Or just Failed? + fail_reason = "(Some items failed)" + final_asset_status = "Failed" # Treat partial as Failed for overall status + else: # No items processed successfully + final_asset_status = "Failed" + fail_reason = "(All items failed)" + else: + # Should not happen if processing_items existed + final_asset_status = "Failed" + fail_reason = "(Unknown state after item processing)" + + + # Update overall status list + if final_asset_status == "Processed": + overall_status["processed"].append(asset_name) + elif final_asset_status == "Skipped": + overall_status["skipped"].append(f"{asset_name} {fail_reason}") + else: # Failed or Unknown + overall_status["failed"].append(f"{asset_name} {fail_reason}") + + log.info(f"Asset '{asset_name}' final status: {final_asset_status} {fail_reason}") + # Clean up intermediate results for the asset to save memory + context.intermediate_results = {} + + + except Exception as e: + log.error(f"PipelineOrchestrator.process_source_rule failed critically: {e}", exc_info=True) + # Mark all assets from this source rule that weren't finished as failed + processed_or_skipped_or_failed = set(overall_status["processed"]) | \ + set(name.split(" ")[0] for name in overall_status["skipped"]) | \ + set(name.split(" ")[0] for name in overall_status["failed"]) + for asset_rule in source_rule.assets: + if asset_rule.asset_name not in processed_or_skipped_or_failed: + overall_status["failed"].append(f"{asset_rule.asset_name} (Orchestrator Error: {e})") + finally: + # --- Cleanup Temporary Directory --- + if engine_temp_dir_path and engine_temp_dir_path.exists(): + try: + log.debug(f"PipelineOrchestrator cleaning up temporary directory: {engine_temp_dir_path}") + shutil.rmtree(engine_temp_dir_path, ignore_errors=True) + except Exception as e: + log.error(f"Error cleaning up orchestrator temporary directory {engine_temp_dir_path}: {e}", exc_info=True) + + return overall_status \ No newline at end of file diff --git a/processing/pipeline/stages/alpha_extraction_to_mask.py b/processing/pipeline/stages/alpha_extraction_to_mask.py new file mode 100644 index 0000000..87aa3b6 --- /dev/null +++ b/processing/pipeline/stages/alpha_extraction_to_mask.py @@ -0,0 +1,179 @@ +import logging +import uuid +from pathlib import Path +from typing import List, Optional, Dict + +import numpy as np + +from .base_stage import ProcessingStage +from ..asset_context import AssetProcessingContext +from ...utils import image_processing_utils as ipu +from rule_structure import FileRule +from utils.path_utils import sanitize_filename + +logger = logging.getLogger(__name__) + +class AlphaExtractionToMaskStage(ProcessingStage): + """ + Extracts an alpha channel from a suitable source map (e.g., Albedo, Diffuse) + to generate a MASK map if one is not explicitly defined. + """ + # Use MAP_ prefixed types for internal logic checks + SUITABLE_SOURCE_MAP_TYPES = ["MAP_COL", "MAP_ALBEDO", "MAP_BASECOLOR"] # Map types likely to have alpha + + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" + logger.debug(f"Asset '{asset_name_for_log}': Running AlphaExtractionToMaskStage.") + + if context.status_flags.get('skip_asset'): + logger.debug(f"Asset '{asset_name_for_log}': Skipping due to 'skip_asset' flag.") + return context + + if not context.files_to_process or not context.processed_maps_details: + logger.debug( + f"Asset '{asset_name_for_log}': Skipping alpha extraction - " + f"no files to process or no processed map details." + ) + return context + + # A. Check for Existing MASK Map + for file_rule in context.files_to_process: + # Assuming file_rule has 'map_type' and 'file_path' (instead of filename_pattern) + # Check for existing MASK map using the correct item_type field and MAP_ prefix + if file_rule.item_type == "MAP_MASK": + file_path_for_log = file_rule.file_path if hasattr(file_rule, 'file_path') else "Unknown file path" + logger.info( + f"Asset '{asset_name_for_log}': MASK map already defined by FileRule " + f"for '{file_path_for_log}'. Skipping alpha extraction." + ) + return context + + # B. Find Suitable Source Map with Alpha + source_map_details_for_alpha: Optional[Dict] = None + source_file_rule_id_for_alpha: Optional[str] = None # This ID comes from processed_maps_details keys + + for file_rule_id, details in context.processed_maps_details.items(): + # Check for suitable source map using the standardized internal_map_type field + internal_map_type = details.get('internal_map_type') # Use the standardized field + if details.get('status') == 'Processed' and \ + internal_map_type in self.SUITABLE_SOURCE_MAP_TYPES: + try: + temp_path = Path(details['temp_processed_file']) + if not temp_path.exists(): + logger.warning( + f"Asset '{asset_name_for_log}': Temp file {temp_path} for map " + f"{details['map_type']} (ID: {file_rule_id}) does not exist. Cannot check for alpha." + ) + continue + + image_data = ipu.load_image(temp_path) + + if image_data is not None and image_data.ndim == 3 and image_data.shape[2] == 4: + source_map_details_for_alpha = details + source_file_rule_id_for_alpha = file_rule_id + logger.info( + f"Asset '{asset_name_for_log}': Found potential source for alpha extraction: " + f"{temp_path} (MapType: {details['map_type']})" + ) + break + except Exception as e: + logger.warning( + f"Asset '{asset_name_for_log}': Error checking alpha for {details.get('temp_processed_file', 'N/A')}: {e}" + ) + continue + + + if source_map_details_for_alpha is None or source_file_rule_id_for_alpha is None: + logger.info( + f"Asset '{asset_name_for_log}': No suitable source map with alpha channel found " + f"for MASK extraction." + ) + return context + + # C. Extract Alpha Channel + source_image_path = Path(source_map_details_for_alpha['temp_processed_file']) + full_image_data = ipu.load_image(source_image_path) # Reload to ensure we have the original RGBA + + if full_image_data is None or not (full_image_data.ndim == 3 and full_image_data.shape[2] == 4): + logger.error( + f"Asset '{asset_name_for_log}': Failed to reload or verify alpha channel from " + f"{source_image_path} for MASK extraction." + ) + return context + + alpha_channel: np.ndarray = full_image_data[:, :, 3] # Extract alpha (0-255) + + # D. Save New Temporary MASK Map + if alpha_channel.ndim == 2: # Expected + pass + elif alpha_channel.ndim == 3 and alpha_channel.shape[2] == 1: # (H, W, 1) + alpha_channel = alpha_channel.squeeze(axis=2) + else: + logger.error( + f"Asset '{asset_name_for_log}': Extracted alpha channel has unexpected dimensions: " + f"{alpha_channel.shape}. Cannot save." + ) + return context + + mask_temp_filename = ( + f"mask_from_alpha_{sanitize_filename(source_map_details_for_alpha['map_type'])}" + f"_{source_file_rule_id_for_alpha}{source_image_path.suffix}" + ) + mask_temp_path = context.engine_temp_dir / mask_temp_filename + + save_success = ipu.save_image(mask_temp_path, alpha_channel) + + if not save_success: + logger.error( + f"Asset '{asset_name_for_log}': Failed to save extracted alpha mask to {mask_temp_path}." + ) + return context + + logger.info( + f"Asset '{asset_name_for_log}': Extracted alpha and saved as new MASK map: {mask_temp_path}" + ) + + # E. Create New FileRule for the MASK and Update Context + # FileRule does not have id, active, transform_settings, source_map_ids_for_generation + # It has file_path, item_type, item_type_override, etc. + new_mask_file_rule = FileRule( + file_path=mask_temp_path.name, # Use file_path + item_type="MAP_MASK", # This should be the item_type for a mask + map_type="MASK" # Explicitly set map_type if FileRule has it, or handle via item_type + # Other FileRule fields like item_type_override can be set if needed + ) + # If FileRule needs a unique identifier, it should be handled differently, + # perhaps by generating one and storing it in common_metadata or a separate mapping. + # For now, we create a simple FileRule. + + context.files_to_process.append(new_mask_file_rule) + + # For processed_maps_details, we need a unique key. Using a new UUID. + new_mask_processed_map_key = uuid.uuid4().hex + + original_dims = source_map_details_for_alpha.get('original_dimensions') + if original_dims is None and full_image_data is not None: # Fallback if not in details + original_dims = (full_image_data.shape[1], full_image_data.shape[0]) + + + context.processed_maps_details[new_mask_processed_map_key] = { + 'internal_map_type': "MAP_MASK", # Use the standardized MAP_ prefixed field + 'map_type': "MASK", # Keep standard type for metadata/naming consistency if needed + 'source_file': str(source_image_path), + 'temp_processed_file': str(mask_temp_path), + 'original_dimensions': original_dims, + 'processed_dimensions': (alpha_channel.shape[1], alpha_channel.shape[0]), + 'status': 'Processed', + 'notes': ( + f"Generated from alpha of {source_map_details_for_alpha.get('internal_map_type', 'unknown type')} " # Use internal_map_type for notes + f"(Source Detail ID: {source_file_rule_id_for_alpha})" + ), + # 'file_rule_id': new_mask_file_rule_id_str # FileRule doesn't have an ID to link here directly + } + + logger.info( + f"Asset '{asset_name_for_log}': Added new FileRule for generated MASK " + f"and updated processed_maps_details with key '{new_mask_processed_map_key}'." + ) + + return context \ No newline at end of file diff --git a/processing/pipeline/stages/asset_skip_logic.py b/processing/pipeline/stages/asset_skip_logic.py new file mode 100644 index 0000000..0c176fe --- /dev/null +++ b/processing/pipeline/stages/asset_skip_logic.py @@ -0,0 +1,55 @@ +import logging +from .base_stage import ProcessingStage +from ..asset_context import AssetProcessingContext + +class AssetSkipLogicStage(ProcessingStage): + """ + Processing stage to determine if an asset should be skipped based on various conditions. + """ + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + """ + Executes the asset skip logic. + + Args: + context: The asset processing context. + + Returns: + The updated asset processing context. + """ + context.status_flags['skip_asset'] = False # Initialize/reset skip flag + asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" + + # 1. Check for Supplier Error + # Assuming 'supplier_error' might be set by a previous stage (e.g., SupplierDeterminationStage) + # or if effective_supplier is None after attempts to determine it. + if context.effective_supplier is None or context.status_flags.get('supplier_error', False): + logging.info(f"Asset '{asset_name_for_log}': Skipping due to missing or invalid supplier.") + context.status_flags['skip_asset'] = True + context.status_flags['skip_reason'] = "Invalid or missing supplier" + return context + + # 2. Check process_status in asset_rule.common_metadata + process_status = context.asset_rule.common_metadata.get('process_status') + + if process_status == "SKIP": + logging.info(f"Asset '{asset_name_for_log}': Skipping as per common_metadata.process_status 'SKIP'.") + context.status_flags['skip_asset'] = True + context.status_flags['skip_reason'] = "Process status set to SKIP in common_metadata" + return context + + # Assuming context.config_obj.general_settings.overwrite_existing is a valid path. + # This might need adjustment if 'general_settings' or 'overwrite_existing' is not found. + # For now, we'll assume it's correct based on the original code's intent. + if process_status == "PROCESSED" and \ + hasattr(context.config_obj, 'general_settings') and \ + not getattr(context.config_obj.general_settings, 'overwrite_existing', True): # Default to True (allow overwrite) if not found + logging.info( + f"Asset '{asset_name_for_log}': Skipping as it's already 'PROCESSED' (from common_metadata) " + f"and overwrite is disabled." + ) + context.status_flags['skip_asset'] = True + context.status_flags['skip_reason'] = "Already processed (common_metadata), overwrite disabled" + return context + + # If none of the above conditions are met, skip_asset remains False. + return context \ No newline at end of file diff --git a/processing/pipeline/stages/base_stage.py b/processing/pipeline/stages/base_stage.py new file mode 100644 index 0000000..321a0d4 --- /dev/null +++ b/processing/pipeline/stages/base_stage.py @@ -0,0 +1,22 @@ +from abc import ABC, abstractmethod + +from ..asset_context import AssetProcessingContext + + +class ProcessingStage(ABC): + """ + Abstract base class for a stage in the asset processing pipeline. + """ + + @abstractmethod + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + """ + Executes the processing logic of this stage. + + Args: + context: The current asset processing context. + + Returns: + The updated asset processing context. + """ + pass \ No newline at end of file diff --git a/processing/pipeline/stages/file_rule_filter.py b/processing/pipeline/stages/file_rule_filter.py new file mode 100644 index 0000000..b6785c8 --- /dev/null +++ b/processing/pipeline/stages/file_rule_filter.py @@ -0,0 +1,90 @@ +import logging +import fnmatch +from typing import List, Set + +from .base_stage import ProcessingStage +from ..asset_context import AssetProcessingContext +from rule_structure import FileRule + + +class FileRuleFilterStage(ProcessingStage): + """ + Determines which FileRules associated with an AssetRule should be processed. + Populates context.files_to_process, respecting FILE_IGNORE rules. + """ + + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + """ + Executes the file rule filtering logic. + + Args: + context: The AssetProcessingContext for the current asset. + + Returns: + The modified AssetProcessingContext. + """ + asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" + if context.status_flags.get('skip_asset'): + logging.debug(f"Asset '{asset_name_for_log}': Skipping FileRuleFilterStage due to 'skip_asset' flag.") + return context + + context.files_to_process: List[FileRule] = [] + ignore_patterns: Set[str] = set() + + # Step 1: Collect all FILE_IGNORE patterns + if context.asset_rule and context.asset_rule.files: + for file_rule in context.asset_rule.files: + if file_rule.item_type == "FILE_IGNORE": # Removed 'and file_rule.active' + if hasattr(file_rule, 'file_path') and file_rule.file_path: + ignore_patterns.add(file_rule.file_path) + logging.debug( + f"Asset '{asset_name_for_log}': Registering ignore pattern: '{file_rule.file_path}'" + ) + else: + logging.warning(f"Asset '{asset_name_for_log}': FILE_IGNORE rule found without a file_path. Skipping this ignore rule.") + else: + logging.debug(f"Asset '{asset_name_for_log}': No file rules (context.asset_rule.files) to process or asset_rule is None.") + # Still need to return context even if there are no rules + logging.info(f"Asset '{asset_name_for_log}': 0 file rules queued for processing after filtering.") + return context + + + # Step 2: Filter and add processable FileRules + for file_rule in context.asset_rule.files: # Iterate over .files + # Removed 'if not file_rule.active:' check + + if file_rule.item_type == "FILE_IGNORE": + # Already processed, skip. + continue + + is_ignored = False + # Ensure file_rule.file_path exists before using it with fnmatch + current_file_path = file_rule.file_path if hasattr(file_rule, 'file_path') else None + if not current_file_path: + logging.warning(f"Asset '{asset_name_for_log}': FileRule found without a file_path. Skipping this rule for ignore matching.") + # Decide if this rule should be added or skipped if it has no path + # For now, let's assume it might be an error and not add it if it can't be matched. + # If it should be added by default, this logic needs adjustment. + continue + + + for ignore_pat in ignore_patterns: + if fnmatch.fnmatch(current_file_path, ignore_pat): + is_ignored = True + logging.debug( + f"Asset '{asset_name_for_log}': Skipping file rule for '{current_file_path}' " + f"due to matching ignore pattern '{ignore_pat}'." + ) + break + + if not is_ignored: + context.files_to_process.append(file_rule) + logging.debug( + f"Asset '{asset_name_for_log}': Adding file rule for '{current_file_path}' " + f"(type: {file_rule.item_type}) to processing queue." + ) + + logging.info( + f"Asset '{asset_name_for_log}': {len(context.files_to_process)} file rules queued for processing after filtering." + ) + return context \ No newline at end of file diff --git a/processing/pipeline/stages/gloss_to_rough_conversion.py b/processing/pipeline/stages/gloss_to_rough_conversion.py new file mode 100644 index 0000000..9c2f948 --- /dev/null +++ b/processing/pipeline/stages/gloss_to_rough_conversion.py @@ -0,0 +1,195 @@ +import logging +from pathlib import Path +import numpy as np +from typing import List +import dataclasses + +from .base_stage import ProcessingStage +from ..asset_context import AssetProcessingContext +from rule_structure import FileRule +from ...utils import image_processing_utils as ipu +from utils.path_utils import sanitize_filename + +logger = logging.getLogger(__name__) + +class GlossToRoughConversionStage(ProcessingStage): + """ + Processing stage to convert glossiness maps to roughness maps. + Iterates through FileRules, identifies GLOSS maps, loads their + corresponding temporary processed images, inverts them, and saves + them as new temporary ROUGHNESS maps. Updates the FileRule and + context.processed_maps_details accordingly. + """ + + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + """ + Executes the gloss to roughness conversion logic. + + Args: + context: The AssetProcessingContext containing asset and processing details. + + Returns: + The updated AssetProcessingContext. + """ + asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" + if context.status_flags.get('skip_asset'): + logger.debug(f"Asset '{asset_name_for_log}': Skipping GlossToRoughConversionStage due to skip_asset flag.") + return context + + if not context.processed_maps_details: # files_to_process might be empty if only gloss maps existed and all are converted + logger.debug( + f"Asset '{asset_name_for_log}': processed_maps_details is empty in GlossToRoughConversionStage. Skipping." + ) + return context + + # Start with a copy of the current file rules. We will modify this list. + new_files_to_process: List[FileRule] = list(context.files_to_process) if context.files_to_process else [] + processed_a_gloss_map = False + successful_conversion_statuses = ['BasePOTSaved', 'Processed_With_Variants', 'Processed_No_Variants'] + + logger.info(f"Asset '{asset_name_for_log}': Starting Gloss to Roughness Conversion Stage. Examining {len(context.processed_maps_details)} processed map entries.") + + # Iterate using the index (map_key_index) as the key, which is now standard. + for map_key_index, map_details in context.processed_maps_details.items(): + # Use the standardized internal_map_type field + internal_map_type = map_details.get('internal_map_type', '') + map_status = map_details.get('status') + original_temp_path_str = map_details.get('temp_processed_file') + # source_file_rule_idx from details should align with map_key_index. + # We primarily use map_key_index for accessing FileRule from context.files_to_process. + source_file_rule_idx_from_details = map_details.get('source_file_rule_index') + processing_tag = map_details.get('processing_tag') + + if map_key_index != source_file_rule_idx_from_details: + logger.warning( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: Mismatch between map key index and 'source_file_rule_index' ({source_file_rule_idx_from_details}) in details. " + f"Using map_key_index ({map_key_index}) for FileRule lookup. This might indicate a data consistency issue from previous stage." + ) + + if not processing_tag: + logger.warning(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: 'processing_tag' is missing in map_details. Using a fallback for temp filename. This is unexpected.") + processing_tag = f"mki_{map_key_index}_fallback_tag" + + + # Check if the map is a GLOSS map using the standardized internal_map_type + if not internal_map_type.startswith("MAP_GLOSS"): + # logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: Type '{internal_map_type}' is not GLOSS. Skipping.") + continue + + logger.info(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Identified potential GLOSS map (Type: {internal_map_type}).") + + if map_status not in successful_conversion_statuses: + logger.warning( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): Status '{map_status}' is not one of {successful_conversion_statuses}. " + f"Skipping conversion for this map." + ) + continue + + if not original_temp_path_str: + logger.warning( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): 'temp_processed_file' missing in details. " + f"Skipping conversion." + ) + continue + + original_temp_path = Path(original_temp_path_str) + if not original_temp_path.exists(): + logger.error( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): Temporary file {original_temp_path_str} " + f"does not exist. Skipping conversion." + ) + continue + + # Use map_key_index directly to access the FileRule + # Ensure map_key_index is a valid index for context.files_to_process + if not isinstance(map_key_index, int) or map_key_index < 0 or map_key_index >= len(context.files_to_process): + logger.error( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): Invalid map_key_index ({map_key_index}) for accessing files_to_process (len: {len(context.files_to_process)}). " + f"Skipping conversion." + ) + continue + + original_file_rule = context.files_to_process[map_key_index] + source_file_path_for_log = original_file_rule.file_path if hasattr(original_file_rule, 'file_path') else "Unknown source path" + logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Processing GLOSS map from '{original_temp_path_str}' (Original FileRule path: '{source_file_path_for_log}') for conversion.") + + image_data = ipu.load_image(str(original_temp_path)) + if image_data is None: + logger.error( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Failed to load image data from {original_temp_path_str}. " + f"Skipping conversion." + ) + continue + + # Perform Inversion + inverted_image_data: np.ndarray + if np.issubdtype(image_data.dtype, np.floating): + inverted_image_data = 1.0 - image_data + inverted_image_data = np.clip(inverted_image_data, 0.0, 1.0) + logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Inverted float image data.") + elif np.issubdtype(image_data.dtype, np.integer): + max_val = np.iinfo(image_data.dtype).max + inverted_image_data = max_val - image_data + logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Inverted integer image data (max_val: {max_val}).") + else: + logger.error( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Unsupported image data type {image_data.dtype} " + f"for GLOSS map. Cannot invert. Skipping conversion." + ) + continue + + # Save New Temporary (Roughness) Map + new_temp_filename = f"rough_from_gloss_{processing_tag}{original_temp_path.suffix}" + new_temp_path = context.engine_temp_dir / new_temp_filename + + save_success = ipu.save_image(str(new_temp_path), inverted_image_data) + + if save_success: + logger.info( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Converted GLOSS map {original_temp_path_str} " + f"to ROUGHNESS map {new_temp_path}." + ) + + update_dict = {'item_type': "MAP_ROUGH", 'item_type_override': "MAP_ROUGH"} + + modified_file_rule: Optional[FileRule] = None + if hasattr(original_file_rule, 'model_copy') and callable(original_file_rule.model_copy): # Pydantic + modified_file_rule = original_file_rule.model_copy(update=update_dict) + elif dataclasses.is_dataclass(original_file_rule): # Dataclass + modified_file_rule = dataclasses.replace(original_file_rule, **update_dict) + else: + logger.error(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Original FileRule is neither Pydantic nor dataclass. Cannot modify. Skipping update for this rule.") + continue + + new_files_to_process[map_key_index] = modified_file_rule # Replace using map_key_index + + # Update context.processed_maps_details for this map_key_index + map_details['temp_processed_file'] = str(new_temp_path) + map_details['original_map_type_before_conversion'] = internal_map_type # Store the original internal type + map_details['internal_map_type'] = "MAP_ROUGH" # Use the standardized MAP_ prefixed field + map_details['map_type'] = "Roughness" # Keep standard type for metadata/naming consistency if needed + map_details['status'] = "Converted_To_Rough" + map_details['notes'] = map_details.get('notes', '') + "; Converted from GLOSS by GlossToRoughConversionStage" + if 'base_pot_resolution_name' in map_details: + map_details['processed_resolution_name'] = map_details['base_pot_resolution_name'] + + processed_a_gloss_map = True + else: + logger.error( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Failed to save inverted ROUGHNESS map to {new_temp_path}. " + f"Original GLOSS FileRule remains." + ) + + context.files_to_process = new_files_to_process + + if processed_a_gloss_map: + logger.info( + f"Asset '{asset_name_for_log}': Gloss to Roughness conversion stage finished. Processed one or more maps and updated file list and map details." + ) + else: + logger.info( + f"Asset '{asset_name_for_log}': No gloss maps were converted in GlossToRoughConversionStage. " + f"File list for next stage contains original non-gloss maps and any gloss maps that failed or were ineligible for conversion." + ) + + return context \ No newline at end of file diff --git a/processing/pipeline/stages/initial_scaling.py b/processing/pipeline/stages/initial_scaling.py new file mode 100644 index 0000000..6fc27ab --- /dev/null +++ b/processing/pipeline/stages/initial_scaling.py @@ -0,0 +1,83 @@ +import logging +from typing import Tuple + +import cv2 # Assuming cv2 is available for interpolation flags +import numpy as np + +from .base_stage import ProcessingStage +# Import necessary context classes and utils +from ..asset_context import InitialScalingInput, InitialScalingOutput +from ...utils import image_processing_utils as ipu + +log = logging.getLogger(__name__) + +class InitialScalingStage(ProcessingStage): + """ + Applies initial scaling (e.g., Power-of-Two downscaling) to image data + if configured via the InitialScalingInput. + """ + + def execute(self, input_data: InitialScalingInput) -> InitialScalingOutput: + """ + Applies scaling based on input_data.initial_scaling_mode. + """ + log.debug(f"Initial Scaling Stage: Mode '{input_data.initial_scaling_mode}'.") + + image_to_scale = input_data.image_data + original_dims_wh = input_data.original_dimensions + scaling_mode = input_data.initial_scaling_mode + scaling_applied = False + final_image_data = image_to_scale # Default to original if no scaling happens + + if image_to_scale is None or image_to_scale.size == 0: + log.warning("Initial Scaling Stage: Input image data is None or empty. Skipping.") + # Return original (empty) data and indicate no scaling + return InitialScalingOutput( + scaled_image_data=np.array([]), + scaling_applied=False, + final_dimensions=(0, 0) + ) + + if original_dims_wh is None: + log.warning("Initial Scaling Stage: Original dimensions not provided. Using current image shape.") + h_pre_scale, w_pre_scale = image_to_scale.shape[:2] + original_dims_wh = (w_pre_scale, h_pre_scale) + else: + w_pre_scale, h_pre_scale = original_dims_wh + + + if scaling_mode == "POT_DOWNSCALE": + pot_w = ipu.get_nearest_power_of_two_downscale(w_pre_scale) + pot_h = ipu.get_nearest_power_of_two_downscale(h_pre_scale) + + if (pot_w, pot_h) != (w_pre_scale, h_pre_scale): + log.info(f"Initial Scaling: Applying POT Downscale from ({w_pre_scale},{h_pre_scale}) to ({pot_w},{pot_h}).") + # Use INTER_AREA for downscaling generally + resized_img = ipu.resize_image(image_to_scale, pot_w, pot_h, interpolation=cv2.INTER_AREA) + if resized_img is not None: + final_image_data = resized_img + scaling_applied = True + log.debug("Initial Scaling: POT Downscale applied successfully.") + else: + log.warning("Initial Scaling: POT Downscale resize failed. Using original data.") + # final_image_data remains image_to_scale + else: + log.info("Initial Scaling: POT Downscale - Image already POT or smaller. No scaling needed.") + # final_image_data remains image_to_scale + + elif scaling_mode == "NONE": + log.info("Initial Scaling: Mode is NONE. No scaling applied.") + # final_image_data remains image_to_scale + else: + log.warning(f"Initial Scaling: Unknown INITIAL_SCALING_MODE '{scaling_mode}'. Defaulting to NONE.") + # final_image_data remains image_to_scale + + # Determine final dimensions + final_h, final_w = final_image_data.shape[:2] + final_dims_wh = (final_w, final_h) + + return InitialScalingOutput( + scaled_image_data=final_image_data, + scaling_applied=scaling_applied, + final_dimensions=final_dims_wh + ) \ No newline at end of file diff --git a/processing/pipeline/stages/merged_task_processor.py b/processing/pipeline/stages/merged_task_processor.py new file mode 100644 index 0000000..46507b8 --- /dev/null +++ b/processing/pipeline/stages/merged_task_processor.py @@ -0,0 +1,329 @@ +import logging +import re +from pathlib import Path +from typing import List, Optional, Tuple, Dict, Any + +import cv2 +import numpy as np + +from .base_stage import ProcessingStage +# Import necessary context classes and utils +from ..asset_context import AssetProcessingContext, MergeTaskDefinition, ProcessedMergedMapData +from ...utils import image_processing_utils as ipu + +log = logging.getLogger(__name__) + +class MergedTaskProcessorStage(ProcessingStage): + """ + Processes a single merge task defined in the configuration. + Loads inputs, applies transformations to inputs, handles fallbacks/resizing, + performs the merge, and returns the merged data. + """ + + def _find_input_map_details_in_context( + self, + required_map_type: str, + processed_map_details_context: Dict[str, Dict[str, Any]], + log_prefix_for_find: str + ) -> Optional[Dict[str, Any]]: + """ + Finds the details of a required input map from the context's processed_maps_details. + Prefers exact match for full types (e.g. MAP_TYPE-1), or base type / base type + "-1" for base types (e.g. MAP_TYPE). + Returns the details dictionary for the found map if it has saved_files_info. + """ + # Try exact match first (e.g., rule asks for "MAP_NRM-1" or "MAP_NRM" if that's how it was processed) + for item_key, details in processed_map_details_context.items(): + if details.get('internal_map_type') == required_map_type: + if details.get('saved_files_info') and isinstance(details['saved_files_info'], list) and len(details['saved_files_info']) > 0: + log.debug(f"{log_prefix_for_find}: Found exact match for '{required_map_type}' with key '{item_key}'.") + return details + log.warning(f"{log_prefix_for_find}: Found exact match for '{required_map_type}' (key '{item_key}') but no saved_files_info.") + return None # Found type but no usable files + + # If exact match not found, and required_map_type is a base type (e.g. "MAP_NRM") + # try to find the primary suffixed version "MAP_NRM-1" or the base type itself if it was processed without a suffix. + if not re.search(r'-\d+$', required_map_type): # if it's a base type like MAP_XXX + # Prefer "MAP_XXX-1" as the primary variant if suffixed types exist + primary_suffixed_type = f"{required_map_type}-1" + for item_key, details in processed_map_details_context.items(): + if details.get('internal_map_type') == primary_suffixed_type: + if details.get('saved_files_info') and isinstance(details['saved_files_info'], list) and len(details['saved_files_info']) > 0: + log.debug(f"{log_prefix_for_find}: Found primary suffixed match '{primary_suffixed_type}' for base '{required_map_type}' with key '{item_key}'.") + return details + log.warning(f"{log_prefix_for_find}: Found primary suffixed match '{primary_suffixed_type}' (key '{item_key}') but no saved_files_info.") + return None # Found type but no usable files + + log.debug(f"{log_prefix_for_find}: No suitable match found for '{required_map_type}' via exact or primary suffixed type search.") + return None + + def execute( + self, + context: AssetProcessingContext, + merge_task: MergeTaskDefinition # Specific item passed by orchestrator + ) -> ProcessedMergedMapData: + """ + Processes the given MergeTaskDefinition item. + """ + asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" + task_key = merge_task.task_key + task_data = merge_task.task_data + log_prefix = f"Asset '{asset_name_for_log}', Task '{task_key}'" + log.info(f"{log_prefix}: Processing Merge Task.") + + # Initialize output object with default failure state + result = ProcessedMergedMapData( + merged_image_data=np.array([]), # Placeholder + output_map_type=task_data.get('output_map_type', 'UnknownMergeOutput'), + source_bit_depths=[], + final_dimensions=None, + transformations_applied_to_inputs={}, + status="Failed", + error_message="Initialization error" + ) + + try: + # --- Configuration & Task Data --- + config = context.config_obj + file_type_definitions = getattr(config, "FILE_TYPE_DEFINITIONS", {}) + invert_normal_green = config.invert_normal_green_globally + merge_dimension_mismatch_strategy = getattr(config, "MERGE_DIMENSION_MISMATCH_STRATEGY", "USE_LARGEST") + workspace_path = context.workspace_path # Base for resolving relative input paths + + # input_map_sources_from_task is no longer used for paths. Paths are sourced from context.processed_maps_details. + target_dimensions_hw = task_data.get('source_dimensions') # Expected dimensions (h, w) for fallback creation, must be in config. + merge_inputs_config = task_data.get('inputs', {}) # e.g., {'R': 'MAP_AO', 'G': 'MAP_ROUGH', ...} + merge_defaults = task_data.get('defaults', {}) # e.g., {'R': 255, 'G': 255, ...} + merge_channels_order = task_data.get('channel_order', 'RGB') # e.g., 'RGB', 'RGBA' + + # Target dimensions are crucial if fallbacks are needed. + # Merge inputs config is essential. + # Merge inputs config is essential. Check directly in task_data. + inputs_from_task_data = task_data.get('inputs') + if not isinstance(inputs_from_task_data, dict) or not inputs_from_task_data: + result.error_message = "Merge task data is incomplete (missing or invalid 'inputs' dictionary in task_data)." + log.error(f"{log_prefix}: {result.error_message}") + return result + if not target_dimensions_hw and any(merge_defaults.get(ch) is not None for ch in merge_inputs_config.keys()): + log.warning(f"{log_prefix}: Merge task has defaults defined, but 'source_dimensions' (target_dimensions_hw) is missing in task_data. Fallback image creation might fail if needed.") + # Not returning error yet, as fallbacks might not be triggered. + + loaded_inputs_for_merge: Dict[str, np.ndarray] = {} # Channel char -> image data + actual_input_dimensions: List[Tuple[int, int]] = [] # List of (h, w) for loaded files + input_source_bit_depths: Dict[str, int] = {} # Channel char -> bit depth + all_transform_notes: Dict[str, List[str]] = {} # Channel char -> list of transform notes + + # --- Load, Transform, and Prepare Inputs --- + log.debug(f"{log_prefix}: Loading and preparing inputs...") + for channel_char, required_map_type_from_rule in merge_inputs_config.items(): + # Validate that the required input map type starts with "MAP_" + if not required_map_type_from_rule.startswith("MAP_"): + result.error_message = ( + f"Invalid input map type '{required_map_type_from_rule}' for channel '{channel_char}'. " + f"Input map types for merging must start with 'MAP_'." + ) + log.error(f"{log_prefix}: {result.error_message}") + return result # Fail the task if an input type is invalid + + input_image_data: Optional[np.ndarray] = None + input_source_desc = f"Fallback for {required_map_type_from_rule}" + input_log_prefix = f"{log_prefix}, Input '{required_map_type_from_rule}' (Channel '{channel_char}')" + channel_transform_notes: List[str] = [] + + # 1. Attempt to load from context.processed_maps_details + found_input_map_details = self._find_input_map_details_in_context( + required_map_type_from_rule, context.processed_maps_details, input_log_prefix + ) + + if found_input_map_details: + # Assuming the first saved file is the primary one for merging. + # This might need refinement if specific variants (resolutions/formats) are required. + primary_saved_file_info = found_input_map_details['saved_files_info'][0] + input_file_path_str = primary_saved_file_info.get('path') + + if input_file_path_str: + input_file_path = Path(input_file_path_str) # Path is absolute from SaveVariantsStage + if input_file_path.is_file(): + try: + input_image_data = ipu.load_image(str(input_file_path)) + if input_image_data is not None: + log.info(f"{input_log_prefix}: Loaded from context: {input_file_path}") + actual_input_dimensions.append(input_image_data.shape[:2]) # (h, w) + input_source_desc = str(input_file_path) + # Bit depth from the saved variant info + input_source_bit_depths[channel_char] = primary_saved_file_info.get('bit_depth', 8) + else: + log.warning(f"{input_log_prefix}: Failed to load image from {input_file_path} (found in context). Attempting fallback.") + input_image_data = None # Ensure fallback is triggered + except Exception as e: + log.warning(f"{input_log_prefix}: Error loading image from {input_file_path} (found in context): {e}. Attempting fallback.") + input_image_data = None # Ensure fallback is triggered + else: + log.warning(f"{input_log_prefix}: Input file path '{input_file_path}' (from context) not found. Attempting fallback.") + input_image_data = None # Ensure fallback is triggered + else: + log.warning(f"{input_log_prefix}: Found map type '{required_map_type_from_rule}' in context, but 'path' is missing in saved_files_info. Attempting fallback.") + input_image_data = None # Ensure fallback is triggered + else: + log.info(f"{input_log_prefix}: Input map type '{required_map_type_from_rule}' not found in context.processed_maps_details. Attempting fallback.") + input_image_data = None # Ensure fallback is triggered + + # 2. Apply Fallback if needed + if input_image_data is None: + fallback_value = merge_defaults.get(channel_char) + if fallback_value is not None: + try: + if not target_dimensions_hw: + result.error_message = f"Cannot create fallback for channel '{channel_char}': 'source_dimensions' (target_dimensions_hw) not defined in task_data." + log.error(f"{log_prefix}: {result.error_message}") + return result # Critical failure if dimensions for fallback are missing + h, w = target_dimensions_hw + # Infer shape/dtype for fallback (simplified) + num_channels = 1 if isinstance(fallback_value, (int, float)) else len(fallback_value) if isinstance(fallback_value, (list, tuple)) else 1 + dtype = np.uint8 # Default dtype + shape = (h, w) if num_channels == 1 else (h, w, num_channels) + + input_image_data = np.full(shape, fallback_value, dtype=dtype) + log.warning(f"{input_log_prefix}: Using fallback value {fallback_value} (Target Dims: {target_dimensions_hw}).") + input_source_desc = f"Fallback value {fallback_value}" + input_source_bit_depths[channel_char] = 8 # Assume 8-bit for fallbacks + channel_transform_notes.append(f"Used fallback value {fallback_value}") + except Exception as e: + result.error_message = f"Error creating fallback for channel '{channel_char}': {e}" + log.error(f"{log_prefix}: {result.error_message}") + return result # Critical failure + else: + result.error_message = f"Missing input '{required_map_type_from_rule}' and no fallback default provided for channel '{channel_char}'." + log.error(f"{log_prefix}: {result.error_message}") + return result # Critical failure + + # 3. Apply Transformations to the loaded/fallback input + if input_image_data is not None: + input_image_data, _, transform_notes = ipu.apply_common_map_transformations( + input_image_data.copy(), # Transform a copy + required_map_type_from_rule, # Use the type required by the rule + invert_normal_green, + file_type_definitions, + input_log_prefix + ) + channel_transform_notes.extend(transform_notes) + else: + # This case should be prevented by fallback logic, but as a safeguard: + result.error_message = f"Input data for channel '{channel_char}' is None after load/fallback attempt." + log.error(f"{log_prefix}: {result.error_message} This indicates an internal logic error.") + return result + + loaded_inputs_for_merge[channel_char] = input_image_data + all_transform_notes[channel_char] = channel_transform_notes + + result.transformations_applied_to_inputs = all_transform_notes # Store notes + + # --- Handle Dimension Mismatches (using transformed inputs) --- + log.debug(f"{log_prefix}: Handling dimension mismatches...") + unique_dimensions = set(actual_input_dimensions) + target_merge_dims_hw = target_dimensions_hw # Default + + if len(unique_dimensions) > 1: + log.warning(f"{log_prefix}: Mismatched dimensions found among loaded inputs: {unique_dimensions}. Applying strategy: {merge_dimension_mismatch_strategy}") + mismatch_note = f"Mismatched input dimensions ({unique_dimensions}), applied {merge_dimension_mismatch_strategy}" + # Add note to all relevant inputs? Or just a general note? Add general for now. + # result.status_notes.append(mismatch_note) # Need a place for general notes + + if merge_dimension_mismatch_strategy == "ERROR_SKIP": + result.error_message = "Dimension mismatch and strategy is ERROR_SKIP." + log.error(f"{log_prefix}: {result.error_message}") + return result + elif merge_dimension_mismatch_strategy == "USE_LARGEST": + max_h = max(h for h, w in unique_dimensions) + max_w = max(w for h, w in unique_dimensions) + target_merge_dims_hw = (max_h, max_w) + elif merge_dimension_mismatch_strategy == "USE_FIRST": + target_merge_dims_hw = actual_input_dimensions[0] if actual_input_dimensions else target_dimensions_hw + # Add other strategies or default to USE_LARGEST + + log.info(f"{log_prefix}: Resizing inputs to target merge dimensions: {target_merge_dims_hw}") + # Resize loaded inputs (not fallbacks unless they were treated as having target dims) + for channel_char, img_data in loaded_inputs_for_merge.items(): + # Only resize if it was a loaded input that contributed to the mismatch check + if img_data.shape[:2] in unique_dimensions and img_data.shape[:2] != target_merge_dims_hw: + resized_img = ipu.resize_image(img_data, target_merge_dims_hw[1], target_merge_dims_hw[0]) # w, h + if resized_img is None: + result.error_message = f"Failed to resize input for channel '{channel_char}' to {target_merge_dims_hw}." + log.error(f"{log_prefix}: {result.error_message}") + return result + loaded_inputs_for_merge[channel_char] = resized_img + log.debug(f"{log_prefix}: Resized input for channel '{channel_char}'.") + + # If target_merge_dims_hw is still None (no source_dimensions and no mismatch), use first loaded input's dimensions + if target_merge_dims_hw is None and actual_input_dimensions: + target_merge_dims_hw = actual_input_dimensions[0] + log.info(f"{log_prefix}: Using dimensions from first loaded input: {target_merge_dims_hw}") + + # --- Perform Merge --- + log.debug(f"{log_prefix}: Performing merge operation for channels '{merge_channels_order}'.") + try: + # Final check for valid dimensions before unpacking + if not isinstance(target_merge_dims_hw, tuple) or len(target_merge_dims_hw) != 2: + result.error_message = "Could not determine valid target dimensions for merge operation." + log.error(f"{log_prefix}: {result.error_message} (target_merge_dims_hw: {target_merge_dims_hw})") + return result + + output_channels = len(merge_channels_order) + h, w = target_merge_dims_hw # Use the potentially adjusted dimensions + + # Determine output dtype (e.g., based on inputs or config) - Assume uint8 for now + output_dtype = np.uint8 + + if output_channels == 1: + # Assume the first channel in order is the one to use + channel_char_to_use = merge_channels_order[0] + source_img = loaded_inputs_for_merge[channel_char_to_use] + # Ensure it's grayscale (take first channel if it's multi-channel) + if len(source_img.shape) == 3: + merged_image = source_img[:, :, 0].copy().astype(output_dtype) + else: + merged_image = source_img.copy().astype(output_dtype) + elif output_channels > 1: + merged_image = np.zeros((h, w, output_channels), dtype=output_dtype) + for i, channel_char in enumerate(merge_channels_order): + source_img = loaded_inputs_for_merge.get(channel_char) + if source_img is not None: + # Extract the correct channel (e.g., R from RGB, or use grayscale directly) + if len(source_img.shape) == 3: + # Simple approach: take the first channel if source is color. Needs refinement if specific channel mapping (R->R, G->G etc.) is needed. + merged_image[:, :, i] = source_img[:, :, 0] + else: # Grayscale source + merged_image[:, :, i] = source_img + else: + # This case should have been caught by fallback logic earlier + result.error_message = f"Internal error: Missing prepared input for channel '{channel_char}' during final merge assembly." + log.error(f"{log_prefix}: {result.error_message}") + return result + else: + result.error_message = f"Invalid channel_order '{merge_channels_order}' in merge config." + log.error(f"{log_prefix}: {result.error_message}") + return result + + result.merged_image_data = merged_image + result.final_dimensions = (merged_image.shape[1], merged_image.shape[0]) # w, h + result.source_bit_depths = list(input_source_bit_depths.values()) # Collect bit depths used + log.info(f"{log_prefix}: Successfully merged inputs into image with shape {result.merged_image_data.shape}") + + except Exception as e: + log.exception(f"{log_prefix}: Error during merge operation: {e}") + result.error_message = f"Merge operation failed: {e}" + return result + + # --- Success --- + result.status = "Processed" + result.error_message = None + log.info(f"{log_prefix}: Successfully processed merge task.") + + except Exception as e: + log.exception(f"{log_prefix}: Unhandled exception during processing: {e}") + result.status = "Failed" + result.error_message = f"Unhandled exception: {e}" + # Ensure image data is empty on failure + if result.merged_image_data is None or result.merged_image_data.size == 0: + result.merged_image_data = np.array([]) + + return result \ No newline at end of file diff --git a/processing/pipeline/stages/metadata_finalization_save.py b/processing/pipeline/stages/metadata_finalization_save.py new file mode 100644 index 0000000..78373fc --- /dev/null +++ b/processing/pipeline/stages/metadata_finalization_save.py @@ -0,0 +1,219 @@ +import datetime +import json +import logging +from pathlib import Path +from typing import Any, Dict + +from ..asset_context import AssetProcessingContext +from .base_stage import ProcessingStage +from utils.path_utils import generate_path_from_pattern, sanitize_filename + + +logger = logging.getLogger(__name__) + +class MetadataFinalizationAndSaveStage(ProcessingStage): + """ + This stage finalizes the asset_metadata (e.g., setting processing end time, + final status) and saves it as a JSON file. + """ + + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + """ + Finalizes metadata, determines output path, and saves the metadata JSON file. + """ + asset_name_for_log = "Unknown Asset" + if hasattr(context, 'asset_rule') and context.asset_rule and hasattr(context.asset_rule, 'asset_name'): + asset_name_for_log = context.asset_rule.asset_name + + if not hasattr(context, 'asset_metadata') or not context.asset_metadata: + if context.status_flags.get('skip_asset'): + logger.info( + f"Asset '{asset_name_for_log}': " + f"Skipped before metadata initialization. No metadata file will be saved." + ) + else: + logger.warning( + f"Asset '{asset_name_for_log}': " + f"asset_metadata not initialized. Skipping metadata finalization and save." + ) + return context + + # Check Skip Flag + if context.status_flags.get('skip_asset'): + context.asset_metadata['status'] = "Skipped" + # context.asset_metadata['processing_end_time'] = datetime.datetime.now().isoformat() + context.asset_metadata['notes'] = context.status_flags.get('skip_reason', 'Skipped early in pipeline') + logger.info( + f"Asset '{asset_name_for_log}': Marked as skipped. Reason: {context.asset_metadata['notes']}" + ) + # Assuming we save metadata for skipped assets if it was initialized. + # If not, the logic to skip saving would be here or before path generation. + # However, if we are here, asset_metadata IS initialized. + + # A. Finalize Metadata + # context.asset_metadata['processing_end_time'] = datetime.datetime.now().isoformat() + + # Determine final status (if not already set to Skipped) + if context.asset_metadata.get('status') != "Skipped": + has_errors = any( + context.status_flags.get(error_flag) + for error_flag in ['file_processing_error', 'merge_error', 'critical_error', + 'individual_map_processing_failed', 'metadata_save_error'] # Added more flags + ) + if has_errors: + context.asset_metadata['status'] = "Failed" + else: + context.asset_metadata['status'] = "Processed" + + # Add details of processed and merged maps + # Restructure processed_map_details before assigning + restructured_processed_maps = {} + # getattr(context, 'processed_maps_details', {}) is the source (plural 'maps') + original_processed_maps = getattr(context, 'processed_maps_details', {}) + + # Define keys to remove at the top level of each map entry + map_keys_to_remove = [ + "status", "source_file_path", "temp_processed_file", # Assuming "source_file_path" is the correct key + "original_resolution_name", "base_pot_resolution_name", "processed_resolution_name" + ] + # Define keys to remove from each variant + variant_keys_to_remove = ["temp_path", "dimensions"] + + for map_key, map_detail_original in original_processed_maps.items(): + # Create a new dictionary for the modified map entry + new_map_entry = {} + for key, value in map_detail_original.items(): + if key not in map_keys_to_remove: + new_map_entry[key] = value + + if "variants" in map_detail_original and isinstance(map_detail_original["variants"], dict): + new_variants_dict = {} + for variant_name, variant_data_original in map_detail_original["variants"].items(): + new_variant_entry = {} + for key, value in variant_data_original.items(): + if key not in variant_keys_to_remove: + new_variant_entry[key] = value + + # Add 'path_to_file' + # This path is expected to be set by OutputOrganizationStage in the context. + # It should be a Path object representing the path relative to the metadata directory, + # or an absolute Path that make_serializable can convert. + # Using 'final_output_path_for_metadata' as the key from context. + if 'final_output_path_for_metadata' in variant_data_original: + new_variant_entry['path_to_file'] = variant_data_original['final_output_path_for_metadata'] + else: + # Log a warning if the expected path is not found + logger.warning( + f"Asset '{asset_name_for_log}': 'final_output_path_for_metadata' " + f"missing for variant '{variant_name}' in map '{map_key}'. " + f"Metadata will be incomplete for this variant's path." + ) + new_variant_entry['path_to_file'] = "ERROR_PATH_NOT_FOUND" # Placeholder + new_variants_dict[variant_name] = new_variant_entry + new_map_entry["variants"] = new_variants_dict + + restructured_processed_maps[map_key] = new_map_entry + + # Assign the restructured details. Note: 'processed_map_details' (singular 'map') is the key in asset_metadata. + # context.asset_metadata['processed_map_details'] = restructured_processed_maps + # context.asset_metadata['merged_map_details'] = getattr(context, 'merged_maps_details', {}) + + # (Optional) Add a list of all temporary files + # context.asset_metadata['temporary_files'] = getattr(context, 'temporary_files', []) # Assuming this is populated elsewhere + + # B. Determine Metadata Output Path + # asset_name_for_log is defined at the top of the function if asset_metadata exists + + source_rule_identifier_for_path = "unknown_source" + if hasattr(context, 'source_rule') and context.source_rule: + if hasattr(context.source_rule, 'supplier_identifier') and context.source_rule.supplier_identifier: + source_rule_identifier_for_path = context.source_rule.supplier_identifier + elif hasattr(context.source_rule, 'input_path') and context.source_rule.input_path: + source_rule_identifier_for_path = Path(context.source_rule.input_path).stem # Use stem of input path if no identifier + else: + source_rule_identifier_for_path = "unknown_source_details" + + # Use the configured metadata filename from config_obj + metadata_filename_from_config = getattr(context.config_obj, 'metadata_filename', "metadata.json") + # Ensure asset_name_for_log is safe for filenames + safe_asset_name = sanitize_filename(asset_name_for_log) # asset_name_for_log is defined at the top + final_metadata_filename = f"{safe_asset_name}_{metadata_filename_from_config}" + + # Output path pattern should come from config_obj, not asset_rule + output_path_pattern_from_config = getattr(context.config_obj, 'output_directory_pattern', "[supplier]/[assetname]") + + sha_value = getattr(context, 'sha5_value', None) # Prefer sha5_value if explicitly set on context + if sha_value is None: # Fallback to sha256_value if that was the intended attribute + sha_value = getattr(context, 'sha256_value', None) + + token_data = { + "assetname": asset_name_for_log, + "supplier": context.effective_supplier if context.effective_supplier else source_rule_identifier_for_path, + "sourcerulename": source_rule_identifier_for_path, + "incrementingvalue": getattr(context, 'incrementing_value', None), + "sha5": sha_value, # Assuming pattern uses [sha5] or similar for sha_value + "maptype": "metadata", # Added maptype to token_data + "filename": final_metadata_filename # Added filename to token_data + # Add other tokens if your output_path_pattern_from_config expects them + } + # Clean None values, as generate_path_from_pattern might not handle them well for all tokens + token_data_cleaned = {k: v for k, v in token_data.items() if v is not None} + + # Generate the relative directory path using the pattern and tokens + relative_dir_path_str = generate_path_from_pattern( + pattern_string=output_path_pattern_from_config, # This pattern should resolve to a directory + token_data=token_data_cleaned + ) + + # Construct the full path by joining the base output path, the generated relative directory, and the final filename + metadata_save_path = Path(context.output_base_path) / Path(relative_dir_path_str) / Path(final_metadata_filename) + + # C. Save Metadata File + try: + metadata_save_path.parent.mkdir(parents=True, exist_ok=True) + + def make_serializable(data: Any) -> Any: + if isinstance(data, Path): + # metadata_save_path is available from the outer scope + metadata_dir = metadata_save_path.parent + try: + # Attempt to make the path relative if it's absolute and under the same root + if data.is_absolute(): + # Check if the path can be made relative (e.g., same drive on Windows) + # This check might need to be more robust depending on os.path.relpath behavior + # For pathlib, relative_to will raise ValueError if not possible. + return str(data.relative_to(metadata_dir)) + else: + # If it's already relative, assume it's correct or handle as needed + return str(data) + except ValueError: + # If paths are on different drives or cannot be made relative, + # log a warning and return the absolute path as a string. + # This can happen if an output path was explicitly set to an unrelated directory. + logger.warning( + f"Asset '{asset_name_for_log}': Could not make path {data} " + f"relative to {metadata_dir}. Storing as absolute." + ) + return str(data) + if isinstance(data, datetime.datetime): # Ensure datetime is serializable + return data.isoformat() + if isinstance(data, dict): + return {k: make_serializable(v) for k, v in data.items()} + if isinstance(data, list): + return [make_serializable(i) for i in data] + return data + + # final_output_files is populated by OutputOrganizationStage. Explicitly remove it as per user request. + context.asset_metadata.pop('final_output_files', None) + serializable_metadata = make_serializable(context.asset_metadata) + + with open(metadata_save_path, 'w') as f: + json.dump(serializable_metadata, f, indent=4) + logger.info(f"Asset '{asset_name_for_log}': Metadata saved to {metadata_save_path}") # Use asset_name_for_log + context.asset_metadata['metadata_file_path'] = str(metadata_save_path) + except Exception as e: + logger.error(f"Asset '{asset_name_for_log}': Failed to save metadata to {metadata_save_path}. Error: {e}") # Use asset_name_for_log + context.asset_metadata['status'] = "Failed (Metadata Save Error)" + context.status_flags['metadata_save_error'] = True + + return context \ No newline at end of file diff --git a/processing/pipeline/stages/metadata_initialization.py b/processing/pipeline/stages/metadata_initialization.py new file mode 100644 index 0000000..e77ef96 --- /dev/null +++ b/processing/pipeline/stages/metadata_initialization.py @@ -0,0 +1,175 @@ +import datetime +import logging + +from .base_stage import ProcessingStage +from ..asset_context import AssetProcessingContext # Adjusted import path assuming asset_context is in processing.pipeline +# If AssetProcessingContext is directly under 'processing', the import would be: +# from ...asset_context import AssetProcessingContext +# Based on the provided file structure, asset_context.py is in processing/pipeline/ +# So, from ...asset_context import AssetProcessingContext is likely incorrect. +# It should be: from ..asset_context import AssetProcessingContext +# Correcting this based on typical Python package structure and the location of base_stage.py + +# Re-evaluating import based on common structure: +# If base_stage.py is in processing/pipeline/stages/ +# and asset_context.py is in processing/pipeline/ +# then the import for AssetProcessingContext from metadata_initialization.py (in stages) would be: +# from ..asset_context import AssetProcessingContext + +# Let's assume the following structure for clarity: +# processing/ +# L-- pipeline/ +# L-- __init__.py +# L-- asset_context.py +# L-- base_stage.py (Mistake here, base_stage is in stages, so it's ..base_stage) +# L-- stages/ +# L-- __init__.py +# L-- metadata_initialization.py +# L-- base_stage.py (Corrected: base_stage.py is here) + +# Corrected imports based on the plan and typical structure: +# base_stage.py is in processing/pipeline/stages/ +# asset_context.py is in processing/pipeline/ + +# from ..base_stage import ProcessingStage # This would mean base_stage is one level up from stages (i.e. in pipeline) +# The plan says: from ..base_stage import ProcessingStage +# This implies that metadata_initialization.py is in a subdirectory of where base_stage.py is. +# However, the file path for metadata_initialization.py is processing/pipeline/stages/metadata_initialization.py +# And base_stage.py is listed as processing/pipeline/stages/base_stage.py in the open tabs. +# So, the import should be: +# from .base_stage import ProcessingStage + +# AssetProcessingContext is at processing/pipeline/asset_context.py +# So from processing/pipeline/stages/metadata_initialization.py, it would be: +# from ..asset_context import AssetProcessingContext + +# Final check on imports based on instructions: +# `from ..base_stage import ProcessingStage` -> This means base_stage.py is in `processing/pipeline/` +# `from ...asset_context import AssetProcessingContext` -> This means asset_context.py is in `processing/` +# Let's verify the location of these files from the environment details. +# processing/pipeline/asset_context.py +# processing/pipeline/stages/base_stage.py +# +# So, from processing/pipeline/stages/metadata_initialization.py: +# To import ProcessingStage from processing/pipeline/stages/base_stage.py: +# from .base_stage import ProcessingStage +# To import AssetProcessingContext from processing/pipeline/asset_context.py: +# from ..asset_context import AssetProcessingContext + +# The instructions explicitly state: +# `from ..base_stage import ProcessingStage` +# `from ...asset_context import AssetProcessingContext` +# This implies a different structure than what seems to be in the file tree. +# I will follow the explicit import instructions from the task. +# This means: +# base_stage.py is expected at `processing/pipeline/base_stage.py` +# asset_context.py is expected at `processing/asset_context.py` + +# Given the file tree: +# processing/pipeline/asset_context.py +# processing/pipeline/stages/base_stage.py +# The imports in `processing/pipeline/stages/metadata_initialization.py` should be: +# from .base_stage import ProcessingStage +# from ..asset_context import AssetProcessingContext + +# I will use the imports that align with the provided file structure. + + + +logger = logging.getLogger(__name__) + +class MetadataInitializationStage(ProcessingStage): + """ + Initializes metadata structures within the AssetProcessingContext. + This stage sets up asset_metadata, processed_maps_details, and + merged_maps_details. + """ + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + logger.debug(f"METADATA_INIT_DEBUG: Entry - context.output_base_path = {context.output_base_path}") # Added + """ + Executes the metadata initialization logic. + + Args: + context: The AssetProcessingContext for the current asset. + + Returns: + The modified AssetProcessingContext. + """ + if context.status_flags.get('skip_asset', False): + logger.debug(f"Asset '{context.asset_rule.asset_name if context.asset_rule else 'Unknown'}': Skipping metadata initialization as 'skip_asset' is True.") + return context + + logger.debug(f"Asset '{context.asset_rule.asset_name if context.asset_rule else 'Unknown'}': Initializing metadata.") + + context.asset_metadata = {} + context.processed_maps_details = {} + context.merged_maps_details = {} + + # Populate Initial asset_metadata + if context.asset_rule: + context.asset_metadata['asset_name'] = context.asset_rule.asset_name + # Attempt to get 'id' from common_metadata or use asset_name as a fallback + asset_id_val = context.asset_rule.common_metadata.get('id', context.asset_rule.common_metadata.get('asset_id')) + if asset_id_val is None: + logger.warning(f"Asset '{context.asset_rule.asset_name}': No 'id' or 'asset_id' found in common_metadata. Using asset_name as asset_id.") + asset_id_val = context.asset_rule.asset_name + context.asset_metadata['asset_id'] = str(asset_id_val) + + # Assuming source_path, output_path_pattern, tags, custom_fields might also be in common_metadata + context.asset_metadata['source_path'] = str(context.asset_rule.common_metadata.get('source_path', 'N/A')) + context.asset_metadata['output_path_pattern'] = context.asset_rule.common_metadata.get('output_path_pattern', 'N/A') + context.asset_metadata['tags'] = list(context.asset_rule.common_metadata.get('tags', [])) + context.asset_metadata['custom_fields'] = dict(context.asset_rule.common_metadata.get('custom_fields', {})) + else: + # Handle cases where asset_rule might be None, though typically it should be set + logger.warning("AssetRule is not set in context during metadata initialization.") + context.asset_metadata['asset_name'] = "Unknown Asset" + context.asset_metadata['asset_id'] = "N/A" + context.asset_metadata['source_path'] = "N/A" + context.asset_metadata['output_path_pattern'] = "N/A" + context.asset_metadata['tags'] = [] + context.asset_metadata['custom_fields'] = {} + + + if context.source_rule: + # SourceRule also doesn't have 'name' or 'id' directly. + # Using 'input_path' as a proxy for name, and a placeholder for id. + source_rule_name_val = context.source_rule.input_path if context.source_rule.input_path else "Unknown Source Rule Path" + source_rule_id_val = context.source_rule.high_level_sorting_parameters.get('id', "N/A_SR_ID") # Check high_level_sorting_parameters + logger.debug(f"SourceRule: using input_path '{source_rule_name_val}' as name, and '{source_rule_id_val}' as id.") + context.asset_metadata['source_rule_name'] = source_rule_name_val + context.asset_metadata['source_rule_id'] = str(source_rule_id_val) + else: + logger.warning("SourceRule is not set in context during metadata initialization.") + context.asset_metadata['source_rule_name'] = "Unknown Source Rule" + context.asset_metadata['source_rule_id'] = "N/A" + + context.asset_metadata['effective_supplier'] = context.effective_supplier + context.asset_metadata['processing_start_time'] = datetime.datetime.now().isoformat() + context.asset_metadata['status'] = "Pending" + + if context.config_obj and hasattr(context.config_obj, 'general_settings') and \ + hasattr(context.config_obj.general_settings, 'app_version'): + context.asset_metadata['version'] = context.config_obj.general_settings.app_version + else: + logger.warning("App version not found in config_obj.general_settings. Setting version to 'N/A'.") + context.asset_metadata['version'] = "N/A" # Default or placeholder + + if context.incrementing_value is not None: + context.asset_metadata['incrementing_value'] = context.incrementing_value + + # The plan mentions sha5_value, which is likely a typo for sha256 or similar. + # Implementing as 'sha5_value' per instructions, but noting the potential typo. + if hasattr(context, 'sha5_value') and context.sha5_value is not None: # Check attribute existence + context.asset_metadata['sha5_value'] = context.sha5_value + elif hasattr(context, 'sha256_value') and context.sha256_value is not None: # Fallback if sha5 was a typo + logger.debug("sha5_value not found, using sha256_value if available for metadata.") + context.asset_metadata['sha256_value'] = context.sha256_value + + + logger.info(f"Asset '{context.asset_metadata.get('asset_name', 'Unknown')}': Metadata initialized.") + # Example of how you might log the full metadata for debugging: + # logger.debug(f"Initialized metadata: {context.asset_metadata}") + + logger.debug(f"METADATA_INIT_DEBUG: Exit - context.output_base_path = {context.output_base_path}") # Added + return context \ No newline at end of file diff --git a/processing/pipeline/stages/normal_map_green_channel.py b/processing/pipeline/stages/normal_map_green_channel.py new file mode 100644 index 0000000..636c1ec --- /dev/null +++ b/processing/pipeline/stages/normal_map_green_channel.py @@ -0,0 +1,155 @@ +import logging +import numpy as np +from pathlib import Path +from typing import List + +from .base_stage import ProcessingStage +from ..asset_context import AssetProcessingContext +from rule_structure import FileRule +from ...utils import image_processing_utils as ipu +from utils.path_utils import sanitize_filename + +logger = logging.getLogger(__name__) + +class NormalMapGreenChannelStage(ProcessingStage): + """ + Processing stage to invert the green channel of normal maps if configured. + This is often needed when converting between DirectX (Y-) and OpenGL (Y+) normal map formats. + """ + + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + """ + Identifies NORMAL maps, checks configuration for green channel inversion, + performs inversion if needed, saves a new temporary file, and updates + the AssetProcessingContext. + """ + asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" + if context.status_flags.get('skip_asset'): + logger.debug(f"Asset '{asset_name_for_log}': Skipping NormalMapGreenChannelStage due to skip_asset flag.") + return context + + if not context.processed_maps_details: # Check processed_maps_details primarily + logger.debug( + f"Asset '{asset_name_for_log}': No processed_maps_details in NormalMapGreenChannelStage. Skipping." + ) + return context + + processed_a_normal_map = False + + # Iterate through processed maps, as FileRule objects don't have IDs directly + for map_id_hex, map_details in context.processed_maps_details.items(): + # Check if the map is a processed normal map using the standardized internal_map_type + internal_map_type = map_details.get('internal_map_type') + if internal_map_type and internal_map_type.startswith("MAP_NRM") and map_details.get('status') == 'Processed': + + # Check configuration for inversion + # Assuming general_settings is an attribute of config_obj and might be a dict or an object + should_invert = False + if hasattr(context.config_obj, 'general_settings'): + if isinstance(context.config_obj.general_settings, dict): + should_invert = context.config_obj.general_settings.get('invert_normal_map_green_channel_globally', False) + elif hasattr(context.config_obj.general_settings, 'invert_normal_map_green_channel_globally'): + should_invert = getattr(context.config_obj.general_settings, 'invert_normal_map_green_channel_globally', False) + + original_temp_path_str = map_details.get('temp_processed_file') + if not original_temp_path_str: + logger.warning(f"Asset '{asset_name_for_log}': Normal map (ID: {map_id_hex}) missing 'temp_processed_file' in details. Skipping.") + continue + + original_temp_path = Path(original_temp_path_str) + original_filename_for_log = original_temp_path.name + + if not should_invert: + logger.debug( + f"Asset '{asset_name_for_log}': Normal map green channel inversion not enabled. " + f"Skipping for {original_filename_for_log} (ID: {map_id_hex})." + ) + continue + + if not original_temp_path.exists(): + logger.error( + f"Asset '{asset_name_for_log}': Temporary file {original_temp_path} for normal map " + f"{original_filename_for_log} (ID: {map_id_hex}) does not exist. Cannot invert green channel." + ) + continue + + image_data = ipu.load_image(original_temp_path) + + if image_data is None: + logger.error( + f"Asset '{asset_name_for_log}': Failed to load image from {original_temp_path} " + f"for normal map {original_filename_for_log} (ID: {map_id_hex})." + ) + continue + + if image_data.ndim != 3 or image_data.shape[2] < 2: # Must have at least R, G channels + logger.error( + f"Asset '{asset_name_for_log}': Image {original_temp_path} for normal map " + f"{original_filename_for_log} (ID: {map_id_hex}) is not a valid RGB/normal map " + f"(ndim={image_data.ndim}, channels={image_data.shape[2] if image_data.ndim == 3 else 'N/A'}) " + f"for green channel inversion." + ) + continue + + # Perform Green Channel Inversion + modified_image_data = image_data.copy() + try: + if np.issubdtype(modified_image_data.dtype, np.floating): + modified_image_data[:, :, 1] = 1.0 - modified_image_data[:, :, 1] + elif np.issubdtype(modified_image_data.dtype, np.integer): + max_val = np.iinfo(modified_image_data.dtype).max + modified_image_data[:, :, 1] = max_val - modified_image_data[:, :, 1] + else: + logger.error( + f"Asset '{asset_name_for_log}': Unsupported image data type " + f"{modified_image_data.dtype} for normal map {original_temp_path}. Cannot invert green channel." + ) + continue + except IndexError: + logger.error( + f"Asset '{asset_name_for_log}': Image {original_temp_path} for normal map " + f"{original_filename_for_log} (ID: {map_id_hex}) does not have a green channel (index 1) " + f"or has unexpected dimensions ({modified_image_data.shape}). Cannot invert." + ) + continue + + # Save New Temporary (Modified Normal) Map + # Sanitize map_details.get('map_type') in case it's missing, though it should be 'NORMAL' here + map_type_for_filename = sanitize_filename(map_details.get('map_type', 'NORMAL')) + new_temp_filename = f"normal_g_inv_{map_type_for_filename}_{map_id_hex}{original_temp_path.suffix}" + new_temp_path = context.engine_temp_dir / new_temp_filename + + save_success = ipu.save_image(new_temp_path, modified_image_data) + + if save_success: + logger.info( + f"Asset '{asset_name_for_log}': Inverted green channel for NORMAL map " + f"{original_filename_for_log}, saved to {new_temp_path.name}." + ) + # Update processed_maps_details for this map_id_hex + context.processed_maps_details[map_id_hex]['temp_processed_file'] = str(new_temp_path) + current_notes = context.processed_maps_details[map_id_hex].get('notes', '') + context.processed_maps_details[map_id_hex]['notes'] = \ + f"{current_notes}; Green channel inverted by NormalMapGreenChannelStage".strip('; ') + + processed_a_normal_map = True + else: + logger.error( + f"Asset '{asset_name_for_log}': Failed to save inverted normal map to {new_temp_path} " + f"for original {original_filename_for_log}." + ) + # No need to explicitly manage new_files_to_process list in this loop, + # as we are modifying the temp_processed_file path within processed_maps_details. + # The existing FileRule objects in context.files_to_process (if any) would + # be linked to these details by a previous stage (e.g. IndividualMapProcessing) + # if that stage populates a 'file_rule_id' in map_details. + + # context.files_to_process remains unchanged by this stage directly, + # as we modify the data pointed to by processed_maps_details. + + if processed_a_normal_map: + logger.info(f"Asset '{asset_name_for_log}': NormalMapGreenChannelStage processed relevant normal maps.") + else: + logger.debug(f"Asset '{asset_name_for_log}': No normal maps found or processed in NormalMapGreenChannelStage.") + + return context \ No newline at end of file diff --git a/processing/pipeline/stages/output_organization.py b/processing/pipeline/stages/output_organization.py new file mode 100644 index 0000000..db92653 --- /dev/null +++ b/processing/pipeline/stages/output_organization.py @@ -0,0 +1,307 @@ +import logging +import shutil +from pathlib import Path +from typing import List, Dict, Optional + +from .base_stage import ProcessingStage +from ..asset_context import AssetProcessingContext +from utils.path_utils import generate_path_from_pattern, sanitize_filename, get_filename_friendly_map_type # Absolute import +from rule_structure import FileRule # Assuming these are needed for type hints if not directly in context + +log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) + +class OutputOrganizationStage(ProcessingStage): + """ + Organizes output files by copying temporary processed files to their final destinations. + """ + + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + asset_name_for_log_early = context.asset_rule.asset_name if hasattr(context, 'asset_rule') and context.asset_rule else "Unknown Asset (early)" + log.info(f"OUTPUT_ORG_DEBUG: Stage execution started for asset '{asset_name_for_log_early}'.") + logger.debug(f"OUTPUT_ORG_DEBUG: Entry - context.output_base_path = {context.output_base_path}") # Modified + log.info(f"OUTPUT_ORG_DEBUG: Received context.config_obj.output_directory_base (raw from config) = {getattr(context.config_obj, 'output_directory_base', 'N/A')}") + # resolved_base = "N/A" + # if hasattr(context.config_obj, '_settings') and context.config_obj._settings.get('OUTPUT_BASE_DIR'): + # base_dir_from_settings = context.config_obj._settings.get('OUTPUT_BASE_DIR') + # Path resolution logic might be complex + # log.info(f"OUTPUT_ORG_DEBUG: Received context.config_obj._settings.OUTPUT_BASE_DIR (resolved guess) = {resolved_base}") + log.info(f"OUTPUT_ORG_DEBUG: context.processed_maps_details at start: {context.processed_maps_details}") + """ + Copies temporary processed and merged files to their final output locations + based on path patterns and updates AssetProcessingContext. + """ + asset_name_for_log = context.asset_rule.asset_name if hasattr(context, 'asset_rule') and context.asset_rule else "Unknown Asset" + logger.debug(f"Asset '{asset_name_for_log}': Starting output organization stage.") + + if context.status_flags.get('skip_asset'): + logger.info(f"Asset '{asset_name_for_log}': Output organization skipped as 'skip_asset' is True.") + return context + + current_status = context.asset_metadata.get('status', '') + if current_status.startswith("Failed") or current_status == "Skipped": + logger.info(f"Asset '{asset_name_for_log}': Output organization skipped due to prior status: {current_status}.") + return context + + final_output_files: List[str] = [] + overwrite_existing = context.config_obj.overwrite_existing + + output_dir_pattern = getattr(context.config_obj, 'output_directory_pattern', "[supplier]/[assetname]") + output_filename_pattern_config = getattr(context.config_obj, 'output_filename_pattern', "[assetname]_[maptype]_[resolution].[ext]") + + + # A. Organize Processed Individual Maps + if context.processed_maps_details: + logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(context.processed_maps_details)} processed individual map entries.") + for processed_map_key, details in context.processed_maps_details.items(): + map_status = details.get('status') + # Retrieve the internal map type first + internal_map_type = details.get('internal_map_type', 'unknown_map_type') + # Convert internal type to filename-friendly type using the helper + file_type_definitions = getattr(context.config_obj, "FILE_TYPE_DEFINITIONS", {}) + base_map_type = get_filename_friendly_map_type(internal_map_type, file_type_definitions) # Final filename-friendly type + + # --- Handle maps processed by the SaveVariantsStage (identified by having saved_files_info) --- + saved_files_info = details.get('saved_files_info') # This is a list of dicts from SaveVariantsOutput + + # Check if 'saved_files_info' exists and is a non-empty list. + # This indicates the item was processed by SaveVariantsStage. + if saved_files_info and isinstance(saved_files_info, list) and len(saved_files_info) > 0: + logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(saved_files_info)} variants for map key '{processed_map_key}' (map type: {base_map_type}) from SaveVariantsStage.") + + # Use base_map_type (e.g., "COL") as the key for the map entry + map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(base_map_type, {}) + # map_type is now the key, so no need to store it inside the entry + # map_metadata_entry['map_type'] = base_map_type + map_metadata_entry.setdefault('variant_paths', {}) # Initialize if not present + + processed_any_variant_successfully = False + failed_any_variant = False + + for variant_index, variant_detail in enumerate(saved_files_info): + # Extract info from the save utility's output structure + temp_variant_path_str = variant_detail.get('path') # Key is 'path' + if not temp_variant_path_str: + logger.warning(f"Asset '{asset_name_for_log}': Variant {variant_index} for map '{processed_map_key}' is missing 'path' in saved_files_info. Skipping.") + # Optionally update variant_detail status if it's mutable and tracked, otherwise just skip + continue + + temp_variant_path = Path(temp_variant_path_str) + if not temp_variant_path.is_file(): + logger.warning(f"Asset '{asset_name_for_log}': Temporary variant file '{temp_variant_path}' for map '{processed_map_key}' not found. Skipping.") + continue + + variant_resolution_key = variant_detail.get('resolution_key', f"varRes{variant_index}") + variant_ext = variant_detail.get('format', temp_variant_path.suffix.lstrip('.')) # Use 'format' key + + token_data_variant = { + "assetname": asset_name_for_log, + "supplier": context.effective_supplier or "DefaultSupplier", + "maptype": base_map_type, + "resolution": variant_resolution_key, + "ext": variant_ext, + "incrementingvalue": getattr(context, 'incrementing_value', None), + "sha5": getattr(context, 'sha5_value', None) + } + token_data_variant_cleaned = {k: v for k, v in token_data_variant.items() if v is not None} + output_filename_variant = generate_path_from_pattern(output_filename_pattern_config, token_data_variant_cleaned) + + try: + relative_dir_path_str_variant = generate_path_from_pattern( + pattern_string=output_dir_pattern, + token_data=token_data_variant_cleaned + ) + logger.debug(f"OUTPUT_ORG_DEBUG: Variants - Using context.output_base_path = {context.output_base_path} for final_variant_path construction.") # Added + final_variant_path = Path(context.output_base_path) / Path(relative_dir_path_str_variant) / Path(output_filename_variant) + logger.debug(f"OUTPUT_ORG_DEBUG: Variants - Constructed final_variant_path = {final_variant_path}") # Added + final_variant_path.parent.mkdir(parents=True, exist_ok=True) + + if final_variant_path.exists() and not overwrite_existing: + logger.info(f"Asset '{asset_name_for_log}': Output variant file {final_variant_path} for map '{processed_map_key}' (res: {variant_resolution_key}) exists and overwrite is disabled. Skipping copy.") + # Optionally update variant_detail status if needed + else: + shutil.copy2(temp_variant_path, final_variant_path) + logger.info(f"Asset '{asset_name_for_log}': Copied variant {temp_variant_path} to {final_variant_path} for map '{processed_map_key}'.") + final_output_files.append(str(final_variant_path)) + # Optionally update variant_detail status if needed + + # Store relative path in metadata + # Store only the filename, as it's relative to the metadata.json location + map_metadata_entry['variant_paths'][variant_resolution_key] = output_filename_variant + processed_any_variant_successfully = True + + except Exception as e: + logger.error(f"Asset '{asset_name_for_log}': Failed to copy variant {temp_variant_path} for map key '{processed_map_key}' (res: {variant_resolution_key}). Error: {e}", exc_info=True) + context.status_flags['output_organization_error'] = True + context.asset_metadata['status'] = "Failed (Output Organization Error - Variant)" + # Optionally update variant_detail status if needed + failed_any_variant = True + + # Update parent map detail status based on variant outcomes + if failed_any_variant: + details['status'] = 'Organization Failed (Save Utility Variants)' + elif processed_any_variant_successfully: + details['status'] = 'Organized (Save Utility Variants)' + else: # No variants were successfully copied (e.g., all skipped due to existing file or missing temp file) + details['status'] = 'Organization Skipped (No Save Utility Variants Copied/Needed)' + + # --- Handle older/other processing statuses (like single file processing) --- + elif map_status in ['Processed', 'Processed_No_Variants', 'Converted_To_Rough']: # Add other single-file statuses if needed + temp_file_path_str = details.get('temp_processed_file') + if not temp_file_path_str: + logger.warning(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (status '{map_status}') due to missing 'temp_processed_file'.") + details['status'] = 'Organization Skipped (Missing Temp File)' + continue + + temp_file_path = Path(temp_file_path_str) + if not temp_file_path.is_file(): + logger.warning(f"Asset '{asset_name_for_log}': Temporary file '{temp_file_path}' for map '{processed_map_key}' not found. Skipping.") + details['status'] = 'Organization Skipped (Temp File Not Found)' + continue + + resolution_str = details.get('processed_resolution_name', details.get('original_resolution_name', 'resX')) + + token_data = { + "assetname": asset_name_for_log, + "supplier": context.effective_supplier or "DefaultSupplier", + "maptype": base_map_type, + "resolution": resolution_str, + "ext": temp_file_path.suffix.lstrip('.'), + "incrementingvalue": getattr(context, 'incrementing_value', None), + "sha5": getattr(context, 'sha5_value', None) + } + token_data_cleaned = {k: v for k, v in token_data.items() if v is not None} + + output_filename = generate_path_from_pattern(output_filename_pattern_config, token_data_cleaned) + + try: + relative_dir_path_str = generate_path_from_pattern( + pattern_string=output_dir_pattern, + token_data=token_data_cleaned + ) + logger.debug(f"OUTPUT_ORG_DEBUG: SingleFile - Using context.output_base_path = {context.output_base_path} for final_path construction.") # Added + final_path = Path(context.output_base_path) / Path(relative_dir_path_str) / Path(output_filename) + logger.debug(f"OUTPUT_ORG_DEBUG: SingleFile - Constructed final_path = {final_path}") # Added + final_path.parent.mkdir(parents=True, exist_ok=True) + + if final_path.exists() and not overwrite_existing: + logger.info(f"Asset '{asset_name_for_log}': Output file {final_path} for map '{processed_map_key}' exists and overwrite is disabled. Skipping copy.") + details['status'] = 'Organized (Exists, Skipped Copy)' + else: + shutil.copy2(temp_file_path, final_path) + logger.info(f"Asset '{asset_name_for_log}': Copied {temp_file_path} to {final_path} for map '{processed_map_key}'.") + final_output_files.append(str(final_path)) + details['status'] = 'Organized' + + details['final_output_path'] = str(final_path) + + # Update asset_metadata for metadata.json + # Use base_map_type (e.g., "COL") as the key for the map entry + map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(base_map_type, {}) + # map_type is now the key, so no need to store it inside the entry + # map_metadata_entry['map_type'] = base_map_type + # Store single path in variant_paths, keyed by its resolution string + # Store only the filename, as it's relative to the metadata.json location + map_metadata_entry.setdefault('variant_paths', {})[resolution_str] = output_filename + # Remove old cleanup logic, as variant_paths is now the standard + # if 'variant_paths' in map_metadata_entry: + # del map_metadata_entry['variant_paths'] + + except Exception as e: + logger.error(f"Asset '{asset_name_for_log}': Failed to copy {temp_file_path} for map key '{processed_map_key}'. Error: {e}", exc_info=True) + context.status_flags['output_organization_error'] = True + context.asset_metadata['status'] = "Failed (Output Organization Error)" + details['status'] = 'Organization Failed' + + # --- Handle other statuses (Skipped, Failed, etc.) --- + else: # Catches statuses not explicitly handled above + logger.debug(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (status: '{map_status}') for organization as it's not a recognized final processed state or variant state.") + continue + else: + logger.debug(f"Asset '{asset_name_for_log}': No processed individual maps to organize.") + + # B. Organize Merged Maps (OBSOLETE BLOCK - Merged maps are handled by the main loop processing context.processed_maps_details) + # The log "No merged maps to organize" will no longer appear from here. + # If merged maps are not appearing, the issue is likely that they are not being added + # to context.processed_maps_details with 'saved_files_info' by the orchestrator/SaveVariantsStage. + + # C. Organize Extra Files (e.g., previews, text files) + logger.debug(f"Asset '{asset_name_for_log}': Checking for EXTRA files to organize.") + extra_files_organized_count = 0 + if hasattr(context, 'files_to_process') and context.files_to_process: + extra_subdir_name = getattr(context.config_obj, 'extra_files_subdir', 'Extra') # Default to 'Extra' + + for file_rule in context.files_to_process: + if file_rule.item_type == 'EXTRA': + source_file_path = context.workspace_path / file_rule.file_path + if not source_file_path.is_file(): + logger.warning(f"Asset '{asset_name_for_log}': EXTRA file '{source_file_path}' not found. Skipping.") + continue + + # Basic token data for the asset's base output directory + # We don't use map_type, resolution, or ext for the base directory of extras. + # However, generate_path_from_pattern might expect them or handle their absence. + # For the base asset directory, only assetname and supplier are typically primary. + base_token_data = { + "assetname": asset_name_for_log, + "supplier": context.effective_supplier or "DefaultSupplier", + # Add other tokens if your output_directory_pattern uses them at the asset level + "incrementingvalue": getattr(context, 'incrementing_value', None), + "sha5": getattr(context, 'sha5_value', None) + } + base_token_data_cleaned = {k: v for k, v in base_token_data.items() if v is not None} + + try: + asset_base_output_dir_str = generate_path_from_pattern( + pattern_string=output_dir_pattern, # Uses the same pattern as other maps for base dir + token_data=base_token_data_cleaned + ) + # Destination: /// + logger.debug(f"OUTPUT_ORG_DEBUG: ExtraFiles - Using context.output_base_path = {context.output_base_path} for final_dest_path construction.") # Added + final_dest_path = (Path(context.output_base_path) / + Path(asset_base_output_dir_str) / + Path(extra_subdir_name) / + source_file_path.name) # Use original filename + logger.debug(f"OUTPUT_ORG_DEBUG: ExtraFiles - Constructed final_dest_path = {final_dest_path}") # Added + + final_dest_path.parent.mkdir(parents=True, exist_ok=True) + + if final_dest_path.exists() and not overwrite_existing: + logger.info(f"Asset '{asset_name_for_log}': EXTRA file destination {final_dest_path} exists and overwrite is disabled. Skipping copy.") + else: + shutil.copy2(source_file_path, final_dest_path) + logger.info(f"Asset '{asset_name_for_log}': Copied EXTRA file {source_file_path} to {final_dest_path}") + final_output_files.append(str(final_dest_path)) + extra_files_organized_count += 1 + + # Optionally, add more detailed tracking for extra files in context.asset_metadata + # For example: + # if 'extra_files_details' not in context.asset_metadata: + # context.asset_metadata['extra_files_details'] = [] + # context.asset_metadata['extra_files_details'].append({ + # 'source_path': str(source_file_path), + # 'destination_path': str(final_dest_path), + # 'status': 'Organized' + # }) + + except Exception as e: + logger.error(f"Asset '{asset_name_for_log}': Failed to copy EXTRA file {source_file_path} to destination. Error: {e}", exc_info=True) + context.status_flags['output_organization_error'] = True + context.asset_metadata['status'] = "Failed (Output Organization Error - Extra Files)" + # Optionally, update status for the specific file_rule if tracked + + if extra_files_organized_count > 0: + logger.info(f"Asset '{asset_name_for_log}': Successfully organized {extra_files_organized_count} EXTRA file(s).") + else: + logger.debug(f"Asset '{asset_name_for_log}': No EXTRA files were processed or found to organize.") + + + context.asset_metadata['final_output_files'] = final_output_files + + if context.status_flags.get('output_organization_error'): + logger.error(f"Asset '{asset_name_for_log}': Output organization encountered errors. Status: {context.asset_metadata['status']}") + else: + logger.info(f"Asset '{asset_name_for_log}': Output organization complete. {len(final_output_files)} files placed.") + + logger.debug(f"Asset '{asset_name_for_log}': Output organization stage finished.") + return context \ No newline at end of file diff --git a/processing/pipeline/stages/prepare_processing_items.py b/processing/pipeline/stages/prepare_processing_items.py new file mode 100644 index 0000000..cdfc2ac --- /dev/null +++ b/processing/pipeline/stages/prepare_processing_items.py @@ -0,0 +1,105 @@ +import logging +from typing import List, Union, Optional + +from .base_stage import ProcessingStage +from ..asset_context import AssetProcessingContext, MergeTaskDefinition +from rule_structure import FileRule # Assuming FileRule is imported correctly + +log = logging.getLogger(__name__) + +class PrepareProcessingItemsStage(ProcessingStage): + """ + Identifies and prepares a unified list of items (FileRule, MergeTaskDefinition) + to be processed in subsequent stages. Performs initial validation. + """ + + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + """ + Populates context.processing_items with FileRule and MergeTaskDefinition objects. + """ + asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" + log.info(f"Asset '{asset_name_for_log}': Preparing processing items...") + + if context.status_flags.get('skip_asset', False): + log.info(f"Asset '{asset_name_for_log}': Skipping item preparation due to skip_asset flag.") + context.processing_items = [] + return context + + items_to_process: List[Union[FileRule, MergeTaskDefinition]] = [] + preparation_failed = False + + # --- Add regular files --- + if context.files_to_process: + # Validate source path early for regular files + source_path_valid = True + if not context.source_rule or not context.source_rule.input_path: + log.error(f"Asset '{asset_name_for_log}': SourceRule or SourceRule.input_path is not set. Cannot process regular files.") + source_path_valid = False + preparation_failed = True # Mark as failed if source path is missing + context.status_flags['prepare_items_failed_reason'] = "SourceRule.input_path missing" + elif not context.workspace_path or not context.workspace_path.is_dir(): + log.error(f"Asset '{asset_name_for_log}': Workspace path '{context.workspace_path}' is not a valid directory. Cannot process regular files.") + source_path_valid = False + preparation_failed = True # Mark as failed if workspace path is bad + context.status_flags['prepare_items_failed_reason'] = "Workspace path invalid" + + if source_path_valid: + for file_rule in context.files_to_process: + # Basic validation for FileRule itself + if not file_rule.file_path: + log.warning(f"Asset '{asset_name_for_log}': Skipping FileRule with empty file_path.") + continue # Skip this specific rule, but don't fail the whole stage + items_to_process.append(file_rule) + log.debug(f"Asset '{asset_name_for_log}': Added {len(context.files_to_process)} potential FileRule items.") + else: + log.warning(f"Asset '{asset_name_for_log}': Skipping addition of all FileRule items due to invalid source/workspace path.") + + + # --- Add merged tasks --- + # --- Add merged tasks from global configuration --- + # merged_image_tasks are expected to be loaded into context.config_obj + # by the Configuration class from app_settings.json. + + merged_tasks_list = getattr(context.config_obj, 'map_merge_rules', None) + + if merged_tasks_list and isinstance(merged_tasks_list, list): + log.debug(f"Asset '{asset_name_for_log}': Found {len(merged_tasks_list)} merge tasks in global config.") + for task_idx, task_data in enumerate(merged_tasks_list): + if isinstance(task_data, dict): + task_key = f"merged_task_{task_idx}" + # Basic validation for merge task data: requires output_map_type and an inputs dictionary + if not task_data.get('output_map_type') or not isinstance(task_data.get('inputs'), dict): + log.warning(f"Asset '{asset_name_for_log}', Task Index {task_idx}: Skipping merge task due to missing 'output_map_type' or valid 'inputs' dictionary. Task data: {task_data}") + continue # Skip this specific task + log.debug(f"Asset '{asset_name_for_log}', Preparing Merge Task Index {task_idx}: Raw task_data: {task_data}") + merge_def = MergeTaskDefinition(task_data=task_data, task_key=task_key) + log.debug(f"Asset '{asset_name_for_log}': Created MergeTaskDefinition object: {merge_def}") + log.info(f"Asset '{asset_name_for_log}': Successfully CREATED MergeTaskDefinition: Key='{merge_def.task_key}', OutputType='{merge_def.task_data.get('output_map_type', 'N/A')}'") + items_to_process.append(merge_def) + else: + log.warning(f"Asset '{asset_name_for_log}': Item at index {task_idx} in config_obj.merged_image_tasks is not a dictionary. Skipping. Item: {task_data}") + # The log for "Added X potential MergeTaskDefinition items" will be covered by the final log. + elif merged_tasks_list is None: + log.debug(f"Asset '{asset_name_for_log}': 'merged_image_tasks' not found in config_obj. No global merge tasks to add.") + elif not isinstance(merged_tasks_list, list): + log.warning(f"Asset '{asset_name_for_log}': 'merged_image_tasks' in config_obj is not a list. Skipping global merge tasks. Type: {type(merged_tasks_list)}") + else: # Empty list + log.debug(f"Asset '{asset_name_for_log}': 'merged_image_tasks' in config_obj is empty. No global merge tasks to add.") + + + if not items_to_process: + log.info(f"Asset '{asset_name_for_log}': No valid items found to process after preparation.") + + log.debug(f"Asset '{asset_name_for_log}': Final items_to_process before assigning to context: {items_to_process}") + context.processing_items = items_to_process + context.intermediate_results = {} # Initialize intermediate results storage + + if preparation_failed: + # Set a flag indicating failure during preparation, even if some items might have been added before failure + context.status_flags['prepare_items_failed'] = True + log.error(f"Asset '{asset_name_for_log}': Item preparation failed. Reason: {context.status_flags.get('prepare_items_failed_reason', 'Unknown')}") + # Optionally, clear items if failure means nothing should proceed + # context.processing_items = [] + + log.info(f"Asset '{asset_name_for_log}': Finished preparing items. Found {len(context.processing_items)} valid items.") + return context \ No newline at end of file diff --git a/processing/pipeline/stages/regular_map_processor.py b/processing/pipeline/stages/regular_map_processor.py new file mode 100644 index 0000000..964aaf8 --- /dev/null +++ b/processing/pipeline/stages/regular_map_processor.py @@ -0,0 +1,213 @@ +import logging +import re +from pathlib import Path +from typing import List, Optional, Tuple, Dict + +import cv2 +import numpy as np + +from .base_stage import ProcessingStage # Assuming base_stage is in the same directory +from ..asset_context import AssetProcessingContext, ProcessedRegularMapData +from rule_structure import FileRule, AssetRule +from processing.utils import image_processing_utils as ipu # Absolute import +from utils.path_utils import get_filename_friendly_map_type # Absolute import + +log = logging.getLogger(__name__) + + +class RegularMapProcessorStage(ProcessingStage): + """ + Processes a single regular texture map defined by a FileRule. + Loads the image, determines map type, applies transformations, + and returns the processed data. + """ + + # --- Helper Methods (Adapted from IndividualMapProcessingStage) --- + + def _get_suffixed_internal_map_type( + self, + asset_rule: Optional[AssetRule], + current_file_rule: FileRule, + initial_internal_map_type: str, + respect_variant_map_types: List[str], + asset_name_for_log: str + ) -> str: + """ + Determines the potentially suffixed internal map type (e.g., MAP_COL-1). + """ + final_internal_map_type = initial_internal_map_type # Default + + base_map_type_match = re.match(r"(MAP_[A-Z]{3})", initial_internal_map_type) + if not base_map_type_match or not asset_rule or not asset_rule.files: + return final_internal_map_type # Cannot determine suffix without base type or asset rule files + + true_base_map_type = base_map_type_match.group(1) # This is "MAP_XXX" + + # Find all FileRules in the asset with the same base map type + peers_of_same_base_type = [] + for fr_asset in asset_rule.files: + fr_asset_item_type = fr_asset.item_type_override or fr_asset.item_type or "UnknownMapType" + fr_asset_base_match = re.match(r"(MAP_[A-Z]{3})", fr_asset_item_type) + if fr_asset_base_match and fr_asset_base_match.group(1) == true_base_map_type: + peers_of_same_base_type.append(fr_asset) + + num_occurrences = len(peers_of_same_base_type) + current_instance_index = 0 # 1-based index + + try: + # Find the index based on the FileRule object itself (requires object identity) + current_instance_index = peers_of_same_base_type.index(current_file_rule) + 1 + except ValueError: + # Fallback: try matching by file_path if object identity fails (less reliable) + try: + current_instance_index = [fr.file_path for fr in peers_of_same_base_type].index(current_file_rule.file_path) + 1 + log.warning(f"Asset '{asset_name_for_log}', FileRule path '{current_file_rule.file_path}': Found peer index using file_path fallback for suffixing.") + except (ValueError, AttributeError): # Catch AttributeError if file_path is None + log.warning( + f"Asset '{asset_name_for_log}', FileRule path '{current_file_rule.file_path}' (Initial Type: '{initial_internal_map_type}', Base: '{true_base_map_type}'): " + f"Could not find its own instance in the list of {num_occurrences} peers from asset_rule.files using object identity or path. Suffixing may be incorrect." + ) + # Keep index 0, suffix logic below will handle it + + # Determine Suffix + map_type_for_respect_check = true_base_map_type.replace("MAP_", "") # e.g., "COL" + is_in_respect_list = map_type_for_respect_check in respect_variant_map_types + + suffix_to_append = "" + if num_occurrences > 1: + if current_instance_index > 0: + suffix_to_append = f"-{current_instance_index}" + else: + # If index is still 0 (not found), don't add suffix to avoid ambiguity + log.warning(f"Asset '{asset_name_for_log}', FileRule path '{current_file_rule.file_path}': Index for multi-occurrence map type '{true_base_map_type}' (count: {num_occurrences}) not determined. Omitting numeric suffix.") + elif num_occurrences == 1 and is_in_respect_list: + suffix_to_append = "-1" # Add suffix even for single instance if in respect list + + if suffix_to_append: + final_internal_map_type = true_base_map_type + suffix_to_append + + if final_internal_map_type != initial_internal_map_type: + log.debug(f"Asset '{asset_name_for_log}', FileRule path '{current_file_rule.file_path}': Suffixed internal map type determined: '{initial_internal_map_type}' -> '{final_internal_map_type}'") + + return final_internal_map_type + + + # --- Execute Method --- + + def execute( + self, + context: AssetProcessingContext, + file_rule: FileRule # Specific item passed by orchestrator + ) -> ProcessedRegularMapData: + """ + Processes the given FileRule item. + """ + asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" + log_prefix = f"Asset '{asset_name_for_log}', File '{file_rule.file_path}'" + log.info(f"{log_prefix}: Processing Regular Map.") + + # Initialize output object with default failure state + result = ProcessedRegularMapData( + processed_image_data=np.array([]), # Placeholder + final_internal_map_type="Unknown", + source_file_path=Path(file_rule.file_path or "InvalidPath"), + original_bit_depth=None, + original_dimensions=None, + transformations_applied=[], + status="Failed", + error_message="Initialization error" + ) + + try: + # --- Configuration --- + config = context.config_obj + file_type_definitions = getattr(config, "FILE_TYPE_DEFINITIONS", {}) + respect_variant_map_types = getattr(config, "respect_variant_map_types", []) + invert_normal_green = config.invert_normal_green_globally + + # --- Determine Map Type (with suffix) --- + initial_internal_map_type = file_rule.item_type_override or file_rule.item_type or "UnknownMapType" + if not initial_internal_map_type or initial_internal_map_type == "UnknownMapType": + result.error_message = "Map type (item_type) not defined in FileRule." + log.error(f"{log_prefix}: {result.error_message}") + return result # Early exit + + # Explicitly skip if the determined type doesn't start with "MAP_" + if not initial_internal_map_type.startswith("MAP_"): + result.status = "Skipped (Invalid Type)" + result.error_message = f"FileRule item_type '{initial_internal_map_type}' does not start with 'MAP_'. Skipping processing." + log.warning(f"{log_prefix}: {result.error_message}") + return result # Early exit + + processing_map_type = self._get_suffixed_internal_map_type( + context.asset_rule, file_rule, initial_internal_map_type, respect_variant_map_types, asset_name_for_log + ) + result.final_internal_map_type = processing_map_type # Store initial suffixed type + + # --- Find and Load Source File --- + if not file_rule.file_path: # Should have been caught by Prepare stage, but double-check + result.error_message = "FileRule has empty file_path." + log.error(f"{log_prefix}: {result.error_message}") + return result + + source_base_path = context.workspace_path + potential_source_path = source_base_path / file_rule.file_path + source_file_path_found: Optional[Path] = None + + if potential_source_path.is_file(): + source_file_path_found = potential_source_path + log.info(f"{log_prefix}: Found source file: {source_file_path_found}") + else: + # Optional: Add globbing fallback if needed, similar to original stage + log.warning(f"{log_prefix}: Source file not found directly at '{potential_source_path}'. Add globbing if necessary.") + result.error_message = f"Source file not found at '{potential_source_path}'" + log.error(f"{log_prefix}: {result.error_message}") + return result + + result.source_file_path = source_file_path_found # Update result with found path + + # Load image + source_image_data = ipu.load_image(str(source_file_path_found)) + if source_image_data is None: + result.error_message = f"Failed to load image from '{source_file_path_found}'." + log.error(f"{log_prefix}: {result.error_message}") + return result + + original_height, original_width = source_image_data.shape[:2] + result.original_dimensions = (original_width, original_height) + log.debug(f"{log_prefix}: Loaded image {result.original_dimensions[0]}x{result.original_dimensions[1]}.") + + # Get original bit depth + try: + result.original_bit_depth = ipu.get_image_bit_depth(str(source_file_path_found)) + log.info(f"{log_prefix}: Determined source bit depth: {result.original_bit_depth}") + except Exception as e: + log.warning(f"{log_prefix}: Could not determine source bit depth for {source_file_path_found}: {e}. Setting to None.") + result.original_bit_depth = None # Indicate failure to determine + + # --- Apply Transformations --- + transformed_image_data, final_map_type, transform_notes = ipu.apply_common_map_transformations( + source_image_data.copy(), # Pass a copy to avoid modifying original load + processing_map_type, + invert_normal_green, + file_type_definitions, + log_prefix + ) + result.processed_image_data = transformed_image_data + result.final_internal_map_type = final_map_type # Update if Gloss->Rough changed it + result.transformations_applied = transform_notes + + # --- Success --- + result.status = "Processed" + result.error_message = None + log.info(f"{log_prefix}: Successfully processed regular map. Final type: '{result.final_internal_map_type}'.") + + except Exception as e: + log.exception(f"{log_prefix}: Unhandled exception during processing: {e}") + result.status = "Failed" + result.error_message = f"Unhandled exception: {e}" + # Ensure image data is empty on failure if it wasn't set + if result.processed_image_data is None or result.processed_image_data.size == 0: + result.processed_image_data = np.array([]) + + return result \ No newline at end of file diff --git a/processing/pipeline/stages/save_variants.py b/processing/pipeline/stages/save_variants.py new file mode 100644 index 0000000..482b1cc --- /dev/null +++ b/processing/pipeline/stages/save_variants.py @@ -0,0 +1,89 @@ +import logging +from typing import List, Dict, Optional # Added Optional + +import numpy as np + +from .base_stage import ProcessingStage +# Import necessary context classes and utils +from ..asset_context import SaveVariantsInput, SaveVariantsOutput +from processing.utils import image_saving_utils as isu # Absolute import +from utils.path_utils import get_filename_friendly_map_type # Absolute import + +log = logging.getLogger(__name__) + + +class SaveVariantsStage(ProcessingStage): + """ + Takes final processed image data and configuration, calls the + save_image_variants utility, and returns the results. + """ + + def execute(self, input_data: SaveVariantsInput) -> SaveVariantsOutput: + """ + Calls isu.save_image_variants with data from input_data. + """ + internal_map_type = input_data.internal_map_type + log_prefix = f"Save Variants Stage (Type: {internal_map_type})" + log.info(f"{log_prefix}: Starting.") + + # Initialize output object with default failure state + result = SaveVariantsOutput( + saved_files_details=[], + status="Failed", + error_message="Initialization error" + ) + + if input_data.image_data is None or input_data.image_data.size == 0: + result.error_message = "Input image data is None or empty." + log.error(f"{log_prefix}: {result.error_message}") + return result + + try: + # --- Prepare arguments for save_image_variants --- + + # Get the filename-friendly base map type using the helper + # This assumes the save utility expects the friendly type. Adjust if needed. + base_map_type_friendly = get_filename_friendly_map_type( + internal_map_type, input_data.file_type_defs + ) + log.debug(f"{log_prefix}: Using filename-friendly base type '{base_map_type_friendly}' for saving.") + + save_args = { + "source_image_data": input_data.image_data, + "base_map_type": base_map_type_friendly, # Use the friendly type + "source_bit_depth_info": input_data.source_bit_depth_info, + "image_resolutions": input_data.image_resolutions, + "file_type_defs": input_data.file_type_defs, + "output_format_8bit": input_data.output_format_8bit, + "output_format_16bit_primary": input_data.output_format_16bit_primary, + "output_format_16bit_fallback": input_data.output_format_16bit_fallback, + "png_compression_level": input_data.png_compression_level, + "jpg_quality": input_data.jpg_quality, + "output_filename_pattern_tokens": input_data.output_filename_pattern_tokens, + "output_filename_pattern": input_data.output_filename_pattern, + "resolution_threshold_for_jpg": input_data.resolution_threshold_for_jpg, # Added + } + + log.debug(f"{log_prefix}: Calling save_image_variants utility.") + saved_files_details: List[Dict] = isu.save_image_variants(**save_args) + + if saved_files_details: + log.info(f"{log_prefix}: Save utility completed successfully. Saved {len(saved_files_details)} variants.") + result.saved_files_details = saved_files_details + result.status = "Processed" + result.error_message = None + else: + # This might not be an error, maybe no variants were configured? + log.warning(f"{log_prefix}: Save utility returned no saved file details. This might be expected if no resolutions/formats matched.") + result.saved_files_details = [] + result.status = "Processed (No Output)" # Indicate processing happened but nothing saved + result.error_message = "Save utility reported no files saved (check configuration/resolutions)." + + + except Exception as e: + log.exception(f"{log_prefix}: Error calling or executing save_image_variants: {e}") + result.status = "Failed" + result.error_message = f"Save utility call failed: {e}" + result.saved_files_details = [] # Ensure empty list on error + + return result \ No newline at end of file diff --git a/processing/pipeline/stages/supplier_determination.py b/processing/pipeline/stages/supplier_determination.py new file mode 100644 index 0000000..15f5e5d --- /dev/null +++ b/processing/pipeline/stages/supplier_determination.py @@ -0,0 +1,67 @@ +import logging + +from .base_stage import ProcessingStage +from ..asset_context import AssetProcessingContext + +class SupplierDeterminationStage(ProcessingStage): + """ + Determines the effective supplier for an asset based on asset and source rules. + """ + + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + """ + Determines and validates the effective supplier for the asset. + + Args: + context: The asset processing context. + + Returns: + The updated asset processing context. + """ + effective_supplier = None + logger = logging.getLogger(__name__) # Using a logger specific to this module + asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" + + # 1. Check source_rule.supplier_override (highest precedence) + if context.source_rule and context.source_rule.supplier_override: + effective_supplier = context.source_rule.supplier_override + logger.debug(f"Asset '{asset_name_for_log}': Supplier override from source_rule found: '{effective_supplier}'.") + # 2. If not overridden, check source_rule.supplier_identifier + elif context.source_rule and context.source_rule.supplier_identifier: + effective_supplier = context.source_rule.supplier_identifier + logger.debug(f"Asset '{asset_name_for_log}': Supplier identifier from source_rule found: '{effective_supplier}'.") + + # 3. Validation + if not effective_supplier: + logger.error(f"Asset '{asset_name_for_log}': No supplier defined in source_rule (override or identifier).") + context.effective_supplier = None + if 'status_flags' not in context: # Ensure status_flags exists + context.status_flags = {} + context.status_flags['supplier_error'] = True + # Assuming context.config_obj.suppliers is a valid way to get the list of configured suppliers. + # This might need further investigation if errors occur here later. + elif context.config_obj and hasattr(context.config_obj, 'suppliers') and effective_supplier not in context.config_obj.suppliers: + logger.warning( + f"Asset '{asset_name_for_log}': Determined supplier '{effective_supplier}' not found in global supplier configuration. " + f"Available: {list(context.config_obj.suppliers.keys()) if context.config_obj.suppliers else 'None'}" + ) + context.effective_supplier = None + if 'status_flags' not in context: # Ensure status_flags exists + context.status_flags = {} + context.status_flags['supplier_error'] = True + else: + context.effective_supplier = effective_supplier + logger.info(f"Asset '{asset_name_for_log}': Effective supplier set to '{effective_supplier}'.") + # Optionally clear the error flag if previously set and now resolved. + if 'supplier_error' in context.status_flags: + del context.status_flags['supplier_error'] + + # merged_image_tasks are loaded from app_settings.json into Configuration object, + # not from supplier-specific presets. + # Ensure the attribute exists on context for PrepareProcessingItemsStage, + # which will get it from context.config_obj. + if not hasattr(context, 'merged_image_tasks'): + context.merged_image_tasks = [] + + + return context \ No newline at end of file diff --git a/processing/utils/__init__.py b/processing/utils/__init__.py new file mode 100644 index 0000000..5f3ceb7 --- /dev/null +++ b/processing/utils/__init__.py @@ -0,0 +1 @@ +# This file makes the 'utils' directory a Python package. \ No newline at end of file diff --git a/processing/utils/image_processing_utils.py b/processing/utils/image_processing_utils.py new file mode 100644 index 0000000..70da34a --- /dev/null +++ b/processing/utils/image_processing_utils.py @@ -0,0 +1,515 @@ +import cv2 +import numpy as np +from pathlib import Path +import math +from typing import Optional, Union, List, Tuple, Dict + +# --- Basic Power-of-Two Utilities --- + +def is_power_of_two(n: int) -> bool: + """Checks if a number is a power of two.""" + return (n > 0) and (n & (n - 1) == 0) + +def get_nearest_pot(value: int) -> int: + """Finds the nearest power of two to the given value.""" + if value <= 0: + return 1 # POT must be positive, return 1 as a fallback + if is_power_of_two(value): + return value + + lower_pot = 1 << (value.bit_length() - 1) + upper_pot = 1 << value.bit_length() + + if (value - lower_pot) < (upper_pot - value): + return lower_pot + else: + return upper_pot + +def get_nearest_power_of_two_downscale(value: int) -> int: + """ + Finds the nearest power of two that is less than or equal to the given value. + If the value is already a power of two, it returns the value itself. + Returns 1 if the value is less than 1. + """ + if value < 1: + return 1 + if is_power_of_two(value): + return value + # Find the largest power of two strictly less than value, + # unless value itself is POT. + # (1 << (value.bit_length() - 1)) achieves this. + # Example: value=7 (0111, bl=3), 1<<2 = 4. + # Example: value=8 (1000, bl=4), 1<<3 = 8. + # Example: value=9 (1001, bl=4), 1<<3 = 8. + return 1 << (value.bit_length() - 1) +# --- Dimension Calculation --- + +def calculate_target_dimensions( + original_width: int, + original_height: int, + target_width: Optional[int] = None, + target_height: Optional[int] = None, + resize_mode: str = "fit", # e.g., "fit", "stretch", "max_dim_pot" + ensure_pot: bool = False, + allow_upscale: bool = False, + target_max_dim_for_pot_mode: Optional[int] = None # Specific for "max_dim_pot" +) -> Tuple[int, int]: + """ + Calculates target dimensions based on various modes and constraints. + + Args: + original_width: Original width of the image. + original_height: Original height of the image. + target_width: Desired target width. + target_height: Desired target height. + resize_mode: + - "fit": Scales to fit within target_width/target_height, maintaining aspect ratio. + Requires at least one of target_width or target_height. + - "stretch": Scales to exactly target_width and target_height, ignoring aspect ratio. + Requires both target_width and target_height. + - "max_dim_pot": Scales to fit target_max_dim_for_pot_mode while maintaining aspect ratio, + then finds nearest POT for each dimension. Requires target_max_dim_for_pot_mode. + ensure_pot: If True, final dimensions will be adjusted to the nearest power of two. + allow_upscale: If False, dimensions will not exceed original dimensions unless ensure_pot forces it. + target_max_dim_for_pot_mode: Max dimension to use when resize_mode is "max_dim_pot". + + Returns: + A tuple (new_width, new_height). + """ + if original_width <= 0 or original_height <= 0: + # Fallback for invalid original dimensions + fallback_dim = 1 + if ensure_pot: + if target_width and target_height: + fallback_dim = get_nearest_pot(max(target_width, target_height, 1)) + elif target_width: + fallback_dim = get_nearest_pot(target_width) + elif target_height: + fallback_dim = get_nearest_pot(target_height) + elif target_max_dim_for_pot_mode: + fallback_dim = get_nearest_pot(target_max_dim_for_pot_mode) + else: # Default POT if no target given + fallback_dim = 256 + return (fallback_dim, fallback_dim) + return (target_width or 1, target_height or 1) + + + w, h = original_width, original_height + + if resize_mode == "max_dim_pot": + if target_max_dim_for_pot_mode is None: + raise ValueError("target_max_dim_for_pot_mode must be provided for 'max_dim_pot' resize_mode.") + + # Logic adapted from old processing_engine.calculate_target_dimensions + ratio = w / h + if ratio > 1: # Width is dominant + scaled_w = target_max_dim_for_pot_mode + scaled_h = max(1, round(scaled_w / ratio)) + else: # Height is dominant or square + scaled_h = target_max_dim_for_pot_mode + scaled_w = max(1, round(scaled_h * ratio)) + + # Upscale check for this mode is implicitly handled by target_max_dim + # If ensure_pot is true (as it was in the original logic), it's applied here + # For this mode, ensure_pot is effectively always true for the final step + w = get_nearest_pot(scaled_w) + h = get_nearest_pot(scaled_h) + return int(w), int(h) + + elif resize_mode == "fit": + if target_width is None and target_height is None: + raise ValueError("At least one of target_width or target_height must be provided for 'fit' mode.") + + if target_width and target_height: + ratio_orig = w / h + ratio_target = target_width / target_height + if ratio_orig > ratio_target: # Original is wider than target aspect + w_new = target_width + h_new = max(1, round(w_new / ratio_orig)) + else: # Original is taller or same aspect + h_new = target_height + w_new = max(1, round(h_new * ratio_orig)) + elif target_width: + w_new = target_width + h_new = max(1, round(w_new / (w / h))) + else: # target_height is not None + h_new = target_height + w_new = max(1, round(h_new * (w / h))) + w, h = w_new, h_new + + elif resize_mode == "stretch": + if target_width is None or target_height is None: + raise ValueError("Both target_width and target_height must be provided for 'stretch' mode.") + w, h = target_width, target_height + + else: + raise ValueError(f"Unsupported resize_mode: {resize_mode}") + + if not allow_upscale: + if w > original_width: w = original_width + if h > original_height: h = original_height + + if ensure_pot: + w = get_nearest_pot(w) + h = get_nearest_pot(h) + # Re-check upscale if POT adjustment made it larger than original and not allowed + if not allow_upscale: + if w > original_width: w = get_nearest_pot(original_width) # Get closest POT to original + if h > original_height: h = get_nearest_pot(original_height) + + + return int(max(1, w)), int(max(1, h)) + + +# --- Image Statistics --- + +def get_image_bit_depth(image_path_str: str) -> Optional[int]: + """ + Determines the bit depth of an image file. + """ + try: + # Use IMREAD_UNCHANGED to preserve original bit depth + img = cv2.imread(image_path_str, cv2.IMREAD_UNCHANGED) + if img is None: + # logger.error(f"Failed to read image for bit depth: {image_path_str}") # Use print for utils + print(f"Warning: Failed to read image for bit depth: {image_path_str}") + return None + + dtype_to_bit_depth = { + np.dtype('uint8'): 8, + np.dtype('uint16'): 16, + np.dtype('float32'): 32, # Typically for EXR etc. + np.dtype('int8'): 8, # Unlikely for images but good to have + np.dtype('int16'): 16, # Unlikely + # Add other dtypes if necessary + } + bit_depth = dtype_to_bit_depth.get(img.dtype) + if bit_depth is None: + # logger.warning(f"Unknown dtype {img.dtype} for image {image_path_str}, cannot determine bit depth.") # Use print for utils + print(f"Warning: Unknown dtype {img.dtype} for image {image_path_str}, cannot determine bit depth.") + pass # Return None + return bit_depth + except Exception as e: + # logger.error(f"Error getting bit depth for {image_path_str}: {e}") # Use print for utils + print(f"Error getting bit depth for {image_path_str}: {e}") + return None + +def calculate_image_stats(image_data: np.ndarray) -> Optional[Dict]: + """ + Calculates min, max, mean for a given numpy image array. + Handles grayscale and multi-channel images. Converts to float64 for calculation. + Normalizes uint8/uint16 data to 0-1 range before calculating stats. + """ + if image_data is None: + return None + try: + data_float = image_data.astype(np.float64) + + if image_data.dtype == np.uint16: + data_float /= 65535.0 + elif image_data.dtype == np.uint8: + data_float /= 255.0 + + stats = {} + if len(data_float.shape) == 2: # Grayscale (H, W) + stats["min"] = float(np.min(data_float)) + stats["max"] = float(np.max(data_float)) + stats["mean"] = float(np.mean(data_float)) + stats["median"] = float(np.median(data_float)) + elif len(data_float.shape) == 3: # Color (H, W, C) + stats["min"] = [float(v) for v in np.min(data_float, axis=(0, 1))] + stats["max"] = [float(v) for v in np.max(data_float, axis=(0, 1))] + stats["mean"] = [float(v) for v in np.mean(data_float, axis=(0, 1))] + stats["median"] = [float(v) for v in np.median(data_float, axis=(0, 1))] + else: + return None # Unsupported shape + return stats + except Exception: + return {"error": "Error calculating image stats"} + +# --- Aspect Ratio String --- + +def normalize_aspect_ratio_change(original_width: int, original_height: int, resized_width: int, resized_height: int, decimals: int = 2) -> str: + """ + Calculates the aspect ratio change string (e.g., "EVEN", "X133"). + """ + if original_width <= 0 or original_height <= 0: + return "InvalidInput" + if resized_width <= 0 or resized_height <= 0: + return "InvalidResize" + + width_change_percentage = ((resized_width - original_width) / original_width) * 100 + height_change_percentage = ((resized_height - original_height) / original_height) * 100 + + normalized_width_change = width_change_percentage / 100 + normalized_height_change = height_change_percentage / 100 + + normalized_width_change = min(max(normalized_width_change + 1, 0), 2) + normalized_height_change = min(max(normalized_height_change + 1, 0), 2) + + epsilon = 1e-9 + if abs(normalized_width_change) < epsilon and abs(normalized_height_change) < epsilon: + closest_value_to_one = 1.0 + elif abs(normalized_width_change) < epsilon: + closest_value_to_one = abs(normalized_height_change) + elif abs(normalized_height_change) < epsilon: + closest_value_to_one = abs(normalized_width_change) + else: + closest_value_to_one = min(abs(normalized_width_change), abs(normalized_height_change)) + + scale_factor = 1 / (closest_value_to_one + epsilon) if abs(closest_value_to_one) < epsilon else 1 / closest_value_to_one + + scaled_normalized_width_change = scale_factor * normalized_width_change + scaled_normalized_height_change = scale_factor * normalized_height_change + + output_width = round(scaled_normalized_width_change, decimals) + output_height = round(scaled_normalized_height_change, decimals) + + if abs(output_width - 1.0) < epsilon: output_width = 1 + if abs(output_height - 1.0) < epsilon: output_height = 1 + + # Helper to format the number part + def format_value(val, dec): + # Multiply by 10^decimals, convert to int to keep trailing zeros in effect + # e.g. val=1.1, dec=2 -> 1.1 * 100 = 110 + # e.g. val=1.0, dec=2 -> 1.0 * 100 = 100 (though this might become "1" if it's exactly 1.0 before this) + # The existing logic already handles output_width/height being 1.0 to produce "EVEN" or skip a component. + # This formatting is for when output_width/height is NOT 1.0. + return str(int(round(val * (10**dec)))) + + if abs(output_width - output_height) < epsilon: # Handles original square or aspect maintained + output = "EVEN" + elif output_width != 1 and abs(output_height - 1.0) < epsilon : # Width changed, height maintained relative to width + output = f"X{format_value(output_width, decimals)}" + elif output_height != 1 and abs(output_width - 1.0) < epsilon: # Height changed, width maintained relative to height + output = f"Y{format_value(output_height, decimals)}" + else: # Both changed relative to each other + output = f"X{format_value(output_width, decimals)}Y{format_value(output_height, decimals)}" + return output + +# --- Image Loading, Conversion, Resizing --- + +def load_image(image_path: Union[str, Path], read_flag: int = cv2.IMREAD_UNCHANGED) -> Optional[np.ndarray]: + """Loads an image from the specified path. Converts BGR/BGRA to RGB/RGBA if color.""" + try: + img = cv2.imread(str(image_path), read_flag) + if img is None: + # print(f"Warning: Failed to load image: {image_path}") # Optional: for debugging utils + return None + + # Ensure RGB/RGBA for color images + if len(img.shape) == 3: + if img.shape[2] == 4: # BGRA from OpenCV + img = cv2.cvtColor(img, cv2.COLOR_BGRA2RGBA) + elif img.shape[2] == 3: # BGR from OpenCV + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + return img + except Exception: # as e: + # print(f"Error loading image {image_path}: {e}") # Optional: for debugging utils + return None + +def convert_bgr_to_rgb(image: np.ndarray) -> np.ndarray: + """Converts an image from BGR/BGRA to RGB/RGBA color space.""" + if image is None or len(image.shape) < 3: + return image # Return as is if not a color image or None + + if image.shape[2] == 4: # BGRA + return cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA) # Keep alpha, convert to RGBA + elif image.shape[2] == 3: # BGR + return cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + return image # Return as is if not 3 or 4 channels + +def convert_rgb_to_bgr(image: np.ndarray) -> np.ndarray: + """Converts an image from RGB/RGBA to BGR/BGRA color space.""" + if image is None or len(image.shape) < 3: + return image # Return as is if not a color image or None + + if image.shape[2] == 4: # RGBA + return cv2.cvtColor(image, cv2.COLOR_RGBA2BGRA) + elif image.shape[2] == 3: # RGB + return cv2.cvtColor(image, cv2.COLOR_RGB2BGR) + return image # Return as is if not 3 or 4 channels + + +def resize_image(image: np.ndarray, target_width: int, target_height: int, interpolation: Optional[int] = None) -> np.ndarray: + """Resizes an image to target_width and target_height.""" + if image is None: + raise ValueError("Cannot resize a None image.") + if target_width <= 0 or target_height <= 0: + raise ValueError("Target width and height must be positive.") + + original_height, original_width = image.shape[:2] + + if interpolation is None: + # Default interpolation: Lanczos for downscaling, Cubic for upscaling/same + if (target_width * target_height) < (original_width * original_height): + interpolation = cv2.INTER_LANCZOS4 + else: + interpolation = cv2.INTER_CUBIC + + return cv2.resize(image, (target_width, target_height), interpolation=interpolation) + +# --- Image Saving --- + +def save_image( + image_path: Union[str, Path], + image_data: np.ndarray, + output_format: Optional[str] = None, # e.g. "png", "jpg", "exr" + output_dtype_target: Optional[np.dtype] = None, # e.g. np.uint8, np.uint16, np.float16 + params: Optional[List[int]] = None, + convert_to_bgr_before_save: bool = True # True for most formats except EXR +) -> bool: + """ + Saves image data to a file. Handles data type and color space conversions. + + Args: + image_path: Path to save the image. + image_data: NumPy array of the image. + output_format: Desired output format (e.g., 'png', 'jpg'). If None, derived from extension. + output_dtype_target: Target NumPy dtype for saving (e.g., np.uint8, np.uint16). + If None, tries to use image_data.dtype or a sensible default. + params: OpenCV imwrite parameters (e.g., [cv2.IMWRITE_JPEG_QUALITY, 90]). + convert_to_bgr_before_save: If True and image is 3-channel, converts RGB to BGR. + Set to False for formats like EXR that expect RGB. + + Returns: + True if saving was successful, False otherwise. + """ + if image_data is None: + return False + + img_to_save = image_data.copy() + path_obj = Path(image_path) + path_obj.parent.mkdir(parents=True, exist_ok=True) + + # 1. Data Type Conversion + if output_dtype_target is not None: + if output_dtype_target == np.uint8 and img_to_save.dtype != np.uint8: + if img_to_save.dtype == np.uint16: img_to_save = (img_to_save.astype(np.float32) / 65535.0 * 255.0).astype(np.uint8) + elif img_to_save.dtype in [np.float16, np.float32, np.float64]: img_to_save = (np.clip(img_to_save, 0.0, 1.0) * 255.0).astype(np.uint8) + else: img_to_save = img_to_save.astype(np.uint8) + elif output_dtype_target == np.uint16 and img_to_save.dtype != np.uint16: + if img_to_save.dtype == np.uint8: img_to_save = (img_to_save.astype(np.float32) / 255.0 * 65535.0).astype(np.uint16) # More accurate + elif img_to_save.dtype in [np.float16, np.float32, np.float64]: img_to_save = (np.clip(img_to_save, 0.0, 1.0) * 65535.0).astype(np.uint16) + else: img_to_save = img_to_save.astype(np.uint16) + elif output_dtype_target == np.float16 and img_to_save.dtype != np.float16: + if img_to_save.dtype == np.uint16: img_to_save = (img_to_save.astype(np.float32) / 65535.0).astype(np.float16) + elif img_to_save.dtype == np.uint8: img_to_save = (img_to_save.astype(np.float32) / 255.0).astype(np.float16) + elif img_to_save.dtype in [np.float32, np.float64]: img_to_save = img_to_save.astype(np.float16) + # else: cannot convert to float16 easily + elif output_dtype_target == np.float32 and img_to_save.dtype != np.float32: + if img_to_save.dtype == np.uint16: img_to_save = (img_to_save.astype(np.float32) / 65535.0) + elif img_to_save.dtype == np.uint8: img_to_save = (img_to_save.astype(np.float32) / 255.0) + elif img_to_save.dtype == np.float16: img_to_save = img_to_save.astype(np.float32) + + + # 2. Color Space Conversion (Internal RGB/RGBA -> BGR/BGRA for OpenCV) + # Input `image_data` is assumed to be in RGB/RGBA format (due to `load_image` changes). + # OpenCV's `imwrite` typically expects BGR/BGRA for formats like PNG, JPG. + # EXR format usually expects RGB/RGBA. + # The `convert_to_bgr_before_save` flag controls this behavior. + current_format = output_format if output_format else path_obj.suffix.lower().lstrip('.') + + if convert_to_bgr_before_save and current_format != 'exr': + # If image is 3-channel (RGB) or 4-channel (RGBA), convert to BGR/BGRA. + if len(img_to_save.shape) == 3 and (img_to_save.shape[2] == 3 or img_to_save.shape[2] == 4): + img_to_save = convert_rgb_to_bgr(img_to_save) # Handles RGB->BGR and RGBA->BGRA + # If `convert_to_bgr_before_save` is False or format is 'exr', + # the image (assumed RGB/RGBA) is saved as is. + + # 3. Save Image + try: + if params: + cv2.imwrite(str(path_obj), img_to_save, params) + else: + cv2.imwrite(str(path_obj), img_to_save) + return True + except Exception: # as e: + # print(f"Error saving image {path_obj}: {e}") # Optional: for debugging utils + return False + +# --- Common Map Transformations --- + +import re +import logging + +ipu_log = logging.getLogger(__name__) + +def apply_common_map_transformations( + image_data: np.ndarray, + processing_map_type: str, # The potentially suffixed internal type + invert_normal_green: bool, + file_type_definitions: Dict[str, Dict], + log_prefix: str +) -> Tuple[np.ndarray, str, List[str]]: + """ + Applies common in-memory transformations (Gloss-to-Rough, Normal Green Invert). + Returns potentially transformed image data, potentially updated map type, and notes. + """ + transformation_notes = [] + current_image_data = image_data # Start with original data + updated_processing_map_type = processing_map_type # Start with original type + + # Gloss-to-Rough + # Check if the base type is Gloss (before suffix) + base_map_type_match = re.match(r"(MAP_GLOSS)", processing_map_type) + if base_map_type_match: + ipu_log.info(f"{log_prefix}: Applying Gloss-to-Rough conversion.") + inversion_succeeded = False + if np.issubdtype(current_image_data.dtype, np.floating): + current_image_data = 1.0 - current_image_data + current_image_data = np.clip(current_image_data, 0.0, 1.0) + ipu_log.debug(f"{log_prefix}: Inverted float image data for Gloss->Rough.") + inversion_succeeded = True + elif np.issubdtype(current_image_data.dtype, np.integer): + max_val = np.iinfo(current_image_data.dtype).max + current_image_data = max_val - current_image_data + ipu_log.debug(f"{log_prefix}: Inverted integer image data (max_val: {max_val}) for Gloss->Rough.") + inversion_succeeded = True + else: + ipu_log.error(f"{log_prefix}: Unsupported image data type {current_image_data.dtype} for GLOSS map. Cannot invert.") + transformation_notes.append("Gloss-to-Rough FAILED (unsupported dtype)") + + if inversion_succeeded: + # Update the type string itself (e.g., MAP_GLOSS-1 -> MAP_ROUGH-1) + updated_processing_map_type = processing_map_type.replace("GLOSS", "ROUGH") + ipu_log.info(f"{log_prefix}: Map type updated: '{processing_map_type}' -> '{updated_processing_map_type}'") + transformation_notes.append("Gloss-to-Rough applied") + + # Normal Green Invert + # Check if the base type is Normal (before suffix) + base_map_type_match_nrm = re.match(r"(MAP_NRM)", processing_map_type) + if base_map_type_match_nrm and invert_normal_green: + ipu_log.info(f"{log_prefix}: Applying Normal Map Green Channel Inversion (Global Setting).") + current_image_data = invert_normal_map_green_channel(current_image_data) + transformation_notes.append("Normal Green Inverted (Global)") + + return current_image_data, updated_processing_map_type, transformation_notes + +# --- Normal Map Utilities --- + +def invert_normal_map_green_channel(normal_map: np.ndarray) -> np.ndarray: + """ + Inverts the green channel of a normal map. + Assumes the normal map is in RGB or RGBA format (channel order R, G, B, A). + """ + if normal_map is None or len(normal_map.shape) < 3 or normal_map.shape[2] < 3: + # Not a valid color image with at least 3 channels + return normal_map + + # Ensure data is mutable + inverted_map = normal_map.copy() + + # Invert the green channel (index 1) + # Handle different data types + if np.issubdtype(inverted_map.dtype, np.floating): + inverted_map[:, :, 1] = 1.0 - inverted_map[:, :, 1] + elif np.issubdtype(inverted_map.dtype, np.integer): + max_val = np.iinfo(inverted_map.dtype).max + inverted_map[:, :, 1] = max_val - inverted_map[:, :, 1] + else: + # Unsupported dtype, return original + print(f"Warning: Unsupported dtype {inverted_map.dtype} for normal map green channel inversion.") + return normal_map + + return inverted_map \ No newline at end of file diff --git a/processing/utils/image_saving_utils.py b/processing/utils/image_saving_utils.py new file mode 100644 index 0000000..9147fc5 --- /dev/null +++ b/processing/utils/image_saving_utils.py @@ -0,0 +1,297 @@ +import logging +import cv2 +import numpy as np +from pathlib import Path +from typing import List, Dict, Any, Tuple, Optional + +# Potentially import ipu from ...utils import image_processing_utils as ipu +# Assuming ipu is available in the same utils directory or parent +try: + from . import image_processing_utils as ipu +except ImportError: + # Fallback for different import structures if needed, adjust based on actual project structure + # For this project structure, the relative import should work. + logging.warning("Could not import image_processing_utils using relative path. Attempting absolute import.") + try: + from processing.utils import image_processing_utils as ipu + except ImportError: + logging.error("Could not import image_processing_utils.") + ipu = None # Handle case where ipu is not available + +logger = logging.getLogger(__name__) + +def save_image_variants( + source_image_data: np.ndarray, + base_map_type: str, # Filename-friendly map type + source_bit_depth_info: List[Optional[int]], + image_resolutions: Dict[str, int], + file_type_defs: Dict[str, Dict[str, Any]], + output_format_8bit: str, + output_format_16bit_primary: str, + output_format_16bit_fallback: str, + png_compression_level: int, + jpg_quality: int, + output_filename_pattern_tokens: Dict[str, Any], # Must include 'output_base_directory': Path and 'asset_name': str + output_filename_pattern: str, + resolution_threshold_for_jpg: Optional[int] = None, # Added + # Consider adding ipu or relevant parts of it if not importing globally +) -> List[Dict[str, Any]]: + """ + Centralizes image saving logic, generating and saving various resolution variants + according to configuration. + + Args: + source_image_data (np.ndarray): High-res image data (in memory, potentially transformed). + base_map_type (str): Final map type (e.g., "COL", "ROUGH", "NORMAL", "MAP_NRMRGH"). + This is the filename-friendly map type. + source_bit_depth_info (List[Optional[int]]): List of original source bit depth(s) + (e.g., [8], [16], [8, 16]). Can contain None. + image_resolutions (Dict[str, int]): Dictionary mapping resolution keys (e.g., "4K") + to max dimensions (e.g., 4096). + file_type_defs (Dict[str, Dict[str, Any]]): Dictionary defining properties for map types, + including 'bit_depth_rule'. + output_format_8bit (str): File extension for 8-bit output (e.g., "jpg", "png"). + output_format_16bit_primary (str): Primary file extension for 16-bit output (e.g., "png", "tif"). + output_format_16bit_fallback (str): Fallback file extension for 16-bit output. + png_compression_level (int): Compression level for PNG output (0-9). + jpg_quality (int): Quality level for JPG output (0-100). + output_filename_pattern_tokens (Dict[str, Any]): Dictionary of tokens for filename + pattern replacement. Must include + 'output_base_directory' (Path) and + 'asset_name' (str). + output_filename_pattern (str): Pattern string for generating output filenames + (e.g., "[assetname]_[maptype]_[resolution].[ext]"). + + Returns: + List[Dict[str, Any]]: A list of dictionaries, each containing details about a saved file. + Example: [{'path': str, 'resolution_key': str, 'format': str, + 'bit_depth': int, 'dimensions': (w,h)}, ...] + """ + if ipu is None: + logger.error("image_processing_utils is not available. Cannot save images.") + return [] + + saved_file_details = [] + source_h, source_w = source_image_data.shape[:2] + source_max_dim = max(source_h, source_w) + + # 1. Use provided configuration inputs (already available as function arguments) + logger.info(f"SaveImageVariants: Starting for map type: {base_map_type}. Source shape: {source_image_data.shape}, Source bit depths: {source_bit_depth_info}") + logger.debug(f"SaveImageVariants: Resolutions: {image_resolutions}, File Type Defs: {file_type_defs.keys()}, Output Formats: 8bit={output_format_8bit}, 16bit_pri={output_format_16bit_primary}, 16bit_fall={output_format_16bit_fallback}") + logger.debug(f"SaveImageVariants: PNG Comp: {png_compression_level}, JPG Qual: {jpg_quality}") + logger.debug(f"SaveImageVariants: Output Tokens: {output_filename_pattern_tokens}, Output Pattern: {output_filename_pattern}") + logger.debug(f"SaveImageVariants: Received resolution_threshold_for_jpg: {resolution_threshold_for_jpg}") # Log received threshold + + # 2. Determine Target Bit Depth + target_bit_depth = 8 # Default + bit_depth_rule = file_type_defs.get(base_map_type, {}).get('bit_depth_rule', 'force_8bit') + if bit_depth_rule not in ['force_8bit', 'respect_inputs']: + logger.warning(f"Unknown bit_depth_rule '{bit_depth_rule}' for map type '{base_map_type}'. Defaulting to 'force_8bit'.") + bit_depth_rule = 'force_8bit' + + if bit_depth_rule == 'respect_inputs': + # Check if any source bit depth is > 8, ignoring None + if any(depth is not None and depth > 8 for depth in source_bit_depth_info): + target_bit_depth = 16 + else: + target_bit_depth = 8 + logger.info(f"Bit depth rule 'respect_inputs' applied. Source bit depths: {source_bit_depth_info}. Target bit depth: {target_bit_depth}") + else: # force_8bit + target_bit_depth = 8 + logger.info(f"Bit depth rule 'force_8bit' applied. Target bit depth: {target_bit_depth}") + + + # 3. Determine Output File Format(s) + if target_bit_depth == 8: + output_ext = output_format_8bit.lstrip('.').lower() + elif target_bit_depth == 16: + # Prioritize primary, fallback to fallback if primary is not supported/desired + # For now, just use primary. More complex logic might be needed later. + output_ext = output_format_16bit_primary.lstrip('.').lower() + # Basic fallback logic example (can be expanded) + if output_ext not in ['png', 'tif']: # Assuming common 16-bit formats + output_ext = output_format_16bit_fallback.lstrip('.').lower() + logger.warning(f"Primary 16-bit format '{output_format_16bit_primary}' might not be suitable. Using fallback '{output_format_16bit_fallback}'.") + else: + logger.error(f"Unsupported target bit depth: {target_bit_depth}. Defaulting to 8-bit format.") + output_ext = output_format_8bit.lstrip('.').lower() + + current_output_ext = output_ext # Store the initial extension based on bit depth + + logger.info(f"SaveImageVariants: Determined target bit depth: {target_bit_depth}, Initial output format: {current_output_ext} for map type {base_map_type}") + + # 4. Generate and Save Resolution Variants + # Sort resolutions by max dimension descending + sorted_resolutions = sorted(image_resolutions.items(), key=lambda item: item[1], reverse=True) + + for res_key, res_max_dim in sorted_resolutions: + logger.info(f"SaveImageVariants: Processing variant {res_key} ({res_max_dim}px) for {base_map_type}") + + # --- Prevent Upscaling --- + # Skip this resolution variant if its target dimension is larger than the source image's largest dimension. + if res_max_dim > source_max_dim: + logger.info(f"SaveImageVariants: Skipping variant {res_key} ({res_max_dim}px) for {base_map_type} because target resolution is larger than source ({source_max_dim}px).") + continue # Skip to the next resolution + + # Calculate target dimensions for valid variants (equal or smaller than source) + if source_max_dim == res_max_dim: + # Use source dimensions if target is equal + target_w_res, target_h_res = source_w, source_h + logger.info(f"SaveImageVariants: Using source resolution ({source_w}x{source_h}) for {res_key} variant of {base_map_type} as target matches source.") + else: # Downscale (source_max_dim > res_max_dim) + # Downscale, maintaining aspect ratio + aspect_ratio = source_w / source_h + if source_w >= source_h: # Use >= to handle square images correctly + target_w_res = res_max_dim + target_h_res = max(1, int(res_max_dim / aspect_ratio)) # Ensure height is at least 1 + else: + target_h_res = res_max_dim + target_w_res = max(1, int(res_max_dim * aspect_ratio)) # Ensure width is at least 1 + logger.info(f"SaveImageVariants: Calculated downscale for {base_map_type} {res_key}: from ({source_w}x{source_h}) to ({target_w_res}x{target_h_res})") + + + # Resize source_image_data (only if necessary) + if (target_w_res, target_h_res) == (source_w, source_h): + # No resize needed if dimensions match + variant_data = source_image_data.copy() # Copy to avoid modifying original if needed later + logger.debug(f"SaveImageVariants: No resize needed for {base_map_type} {res_key}, using copy of source data.") + else: + # Perform resize only if dimensions differ (i.e., downscaling) + interpolation_method = cv2.INTER_AREA # Good for downscaling + try: + variant_data = ipu.resize_image(source_image_data, target_w_res, target_h_res, interpolation=interpolation_method) + if variant_data is None: # Check if resize failed + raise ValueError("ipu.resize_image returned None") + logger.debug(f"SaveImageVariants: Resized variant data shape for {base_map_type} {res_key}: {variant_data.shape}") + except Exception as e: + logger.error(f"SaveImageVariants: Error resizing image for {base_map_type} {res_key} variant: {e}") + continue # Skip this variant if resizing fails + + # Filename Construction + current_tokens = output_filename_pattern_tokens.copy() + current_tokens['maptype'] = base_map_type + current_tokens['resolution'] = res_key + + # Determine final extension for this variant, considering JPG threshold + final_variant_ext = current_output_ext + + # --- Start JPG Threshold Logging --- + logger.debug(f"SaveImageVariants: JPG Threshold Check for {base_map_type} {res_key}:") + logger.debug(f" - target_bit_depth: {target_bit_depth}") + logger.debug(f" - resolution_threshold_for_jpg: {resolution_threshold_for_jpg}") + logger.debug(f" - target_w_res: {target_w_res}, target_h_res: {target_h_res}") + logger.debug(f" - max(target_w_res, target_h_res): {max(target_w_res, target_h_res)}") + logger.debug(f" - current_output_ext: {current_output_ext}") + + cond_bit_depth = target_bit_depth == 8 + cond_threshold_not_none = resolution_threshold_for_jpg is not None + cond_res_exceeded = False + if cond_threshold_not_none: # Avoid comparison if threshold is None + cond_res_exceeded = max(target_w_res, target_h_res) > resolution_threshold_for_jpg + cond_is_png = current_output_ext == 'png' + + logger.debug(f" - Condition (target_bit_depth == 8): {cond_bit_depth}") + logger.debug(f" - Condition (resolution_threshold_for_jpg is not None): {cond_threshold_not_none}") + logger.debug(f" - Condition (max(res) > threshold): {cond_res_exceeded}") + logger.debug(f" - Condition (current_output_ext == 'png'): {cond_is_png}") + # --- End JPG Threshold Logging --- + + if cond_bit_depth and cond_threshold_not_none and cond_res_exceeded and cond_is_png: + final_variant_ext = 'jpg' + logger.info(f"SaveImageVariants: Overriding 8-bit PNG to JPG for {base_map_type} {res_key} due to resolution {max(target_w_res, target_h_res)}px > threshold {resolution_threshold_for_jpg}px.") + + current_tokens['ext'] = final_variant_ext + + try: + # Replace placeholders in the pattern + filename = output_filename_pattern + for token, value in current_tokens.items(): + # Ensure value is string for replacement, handle Path objects later + filename = filename.replace(f"[{token}]", str(value)) + + # Construct full output path + output_base_directory = current_tokens.get('output_base_directory') + if not isinstance(output_base_directory, Path): + logger.error(f"'output_base_directory' token is missing or not a Path object: {output_base_directory}. Cannot save file.") + continue # Skip this variant + + output_path = output_base_directory / filename + logger.info(f"SaveImageVariants: Constructed output path for {base_map_type} {res_key}: {output_path}") + + # Ensure parent directory exists + output_path.parent.mkdir(parents=True, exist_ok=True) + logger.debug(f"SaveImageVariants: Ensured directory exists for {base_map_type} {res_key}: {output_path.parent}") + + except Exception as e: + logger.error(f"SaveImageVariants: Error constructing filepath for {base_map_type} {res_key} variant: {e}") + continue # Skip this variant if path construction fails + + + # Prepare Save Parameters + save_params_cv2 = [] + if final_variant_ext == 'jpg': # Check against final_variant_ext + save_params_cv2.append(cv2.IMWRITE_JPEG_QUALITY) + save_params_cv2.append(jpg_quality) + logger.debug(f"SaveImageVariants: Using JPG quality: {jpg_quality} for {base_map_type} {res_key}") + elif final_variant_ext == 'png': # Check against final_variant_ext + save_params_cv2.append(cv2.IMWRITE_PNG_COMPRESSION) + save_params_cv2.append(png_compression_level) + logger.debug(f"SaveImageVariants: Using PNG compression level: {png_compression_level} for {base_map_type} {res_key}") + # Add other format specific parameters if needed (e.g., TIFF compression) + + + # Bit Depth Conversion is handled by ipu.save_image via output_dtype_target + image_data_for_save = variant_data # Use the resized variant data directly + + # Determine the target dtype for ipu.save_image + output_dtype_for_save: Optional[np.dtype] = None + if target_bit_depth == 8: + output_dtype_for_save = np.uint8 + elif target_bit_depth == 16: + output_dtype_for_save = np.uint16 + # Add other target bit depths like float16/float32 if necessary + # elif target_bit_depth == 32: # Assuming float32 for EXR etc. + # output_dtype_for_save = np.float32 + + + # Saving + try: + # ipu.save_image is expected to handle the actual cv2.imwrite call + logger.debug(f"SaveImageVariants: Attempting to save {base_map_type} {res_key} to {output_path} with params {save_params_cv2}, target_dtype: {output_dtype_for_save}") + success = ipu.save_image( + str(output_path), + image_data_for_save, + output_dtype_target=output_dtype_for_save, # Pass the target dtype + params=save_params_cv2 + ) + if success: + logger.info(f"SaveImageVariants: Successfully saved {base_map_type} {res_key} variant to {output_path}") + # Collect details for the returned list + saved_file_details.append({ + 'path': str(output_path), + 'resolution_key': res_key, + 'format': final_variant_ext, # Log the actual saved format + 'bit_depth': target_bit_depth, + 'dimensions': (target_w_res, target_h_res) + }) + else: + logger.error(f"SaveImageVariants: Failed to save {base_map_type} {res_key} variant to {output_path} (ipu.save_image returned False)") + + except Exception as e: + logger.error(f"SaveImageVariants: Error during ipu.save_image for {base_map_type} {res_key} variant to {output_path}: {e}", exc_info=True) + # Continue to next variant even if one fails + + + # Discard in-memory variant after saving (Python's garbage collection handles this) + del variant_data + del image_data_for_save + + + # 5. Return List of Saved File Details + logger.info(f"Finished saving variants for map type: {base_map_type}. Saved {len(saved_file_details)} variants.") + return saved_file_details + +# Optional Helper Functions (can be added here if needed) +# def _determine_target_bit_depth(...): ... +# def _determine_output_format(...): ... +# def _construct_variant_filepath(...): ... \ No newline at end of file diff --git a/processing_engine.py b/processing_engine.py new file mode 100644 index 0000000..441284d --- /dev/null +++ b/processing_engine.py @@ -0,0 +1,221 @@ +# processing_engine.py + +import os +import math +import shutil +import tempfile +import logging +from pathlib import Path +from typing import List, Dict, Tuple, Optional, Set +log = logging.getLogger(__name__) +# Attempt to import image processing libraries +try: + import cv2 + import numpy as np +except ImportError as e: + log.error(f"Failed to import cv2 or numpy in processing_engine.py: {e}", exc_info=True) + print("ERROR: Missing required image processing libraries. Please install opencv-python and numpy:") + print("pip install opencv-python numpy") + # Allow import to fail but log error; execution will likely fail later + cv2 = None + np = None + + +try: + from configuration import Configuration, ConfigurationError + from rule_structure import SourceRule, AssetRule, FileRule + from utils.path_utils import generate_path_from_pattern, sanitize_filename + from processing.utils import image_processing_utils as ipu # Corrected import +except ImportError as e: + # Temporarily print to console as log might not be initialized yet + print(f"ERROR during initial imports in processing_engine.py: {e}") + # log.error(f"Failed to import Configuration or rule_structure classes in processing_engine.py: {e}", exc_info=True) # Log will be used after init + print("ERROR: Cannot import Configuration or rule_structure classes.") + print("Ensure configuration.py and rule_structure.py are in the same directory or Python path.") + # Allow import to fail but log error; execution will likely fail later + Configuration = None + SourceRule = None + AssetRule = None + FileRule = None + + +# Initialize logger early +log = logging.getLogger(__name__) +# Basic config if logger hasn't been set up elsewhere (e.g., during testing) +if not log.hasHandlers(): + logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') + +# Use logger defined in main.py (or configure one here if run standalone) + +from processing.pipeline.orchestrator import PipelineOrchestrator +# from processing.pipeline.asset_context import AssetProcessingContext # AssetProcessingContext is used by the orchestrator +# Import stages that will be passed to the orchestrator (outer stages) +from processing.pipeline.stages.supplier_determination import SupplierDeterminationStage +from processing.pipeline.stages.asset_skip_logic import AssetSkipLogicStage +from processing.pipeline.stages.metadata_initialization import MetadataInitializationStage +from processing.pipeline.stages.file_rule_filter import FileRuleFilterStage +from processing.pipeline.stages.gloss_to_rough_conversion import GlossToRoughConversionStage +from processing.pipeline.stages.alpha_extraction_to_mask import AlphaExtractionToMaskStage +from processing.pipeline.stages.normal_map_green_channel import NormalMapGreenChannelStage +# Removed: from processing.pipeline.stages.individual_map_processing import IndividualMapProcessingStage +# Removed: from processing.pipeline.stages.map_merging import MapMergingStage +from processing.pipeline.stages.metadata_finalization_save import MetadataFinalizationAndSaveStage +from processing.pipeline.stages.output_organization import OutputOrganizationStage + +# --- Custom Exception --- +class ProcessingEngineError(Exception): + """Custom exception for errors during processing engine operations.""" + pass + +# Helper functions moved to processing.utils.image_processing_utils + +# --- Processing Engine Class --- +class ProcessingEngine: + """ + Handles the core processing pipeline for assets based on explicit rules + provided in a SourceRule object and static configuration. + It does not perform classification, prediction, or rule fallback internally. + """ + def __init__(self, config_obj: Configuration): + """ + Initializes the processing engine with static configuration. + + Args: + config_obj: The loaded Configuration object containing static settings. + """ + if cv2 is None or np is None or Configuration is None or SourceRule is None: + raise ProcessingEngineError("Essential libraries (OpenCV, NumPy) or classes (Configuration, SourceRule) are not available.") + + if not isinstance(config_obj, Configuration): + raise ProcessingEngineError("config_obj must be a valid Configuration object.") + + self.config_obj: Configuration = config_obj + self.temp_dir: Path | None = None # Path to the temporary working directory for a process run + self.loaded_data_cache: dict = {} # Cache for loaded/resized data within a single process call + + # --- Pipeline Orchestrator Setup --- + # Define pre-item and post-item processing stages + pre_item_stages = [ + SupplierDeterminationStage(), + AssetSkipLogicStage(), + MetadataInitializationStage(), + FileRuleFilterStage(), + GlossToRoughConversionStage(), # Assumed to run on context.files_to_process if needed by old logic + AlphaExtractionToMaskStage(), # Same assumption as above + NormalMapGreenChannelStage(), # Same assumption as above + # Note: The new RegularMapProcessorStage and MergedTaskProcessorStage handle their own transformations + # on the specific items they process. These global transformation stages might need review + # if they were intended to operate on a broader scope or if their logic is now fully + # encapsulated in the new item-specific processor stages. For now, keeping them as pre-stages. + ] + + post_item_stages = [ + OutputOrganizationStage(), # Must run after all items are saved to temp + MetadataFinalizationAndSaveStage(),# Must run after output organization to have final paths + ] + + try: + self.pipeline_orchestrator = PipelineOrchestrator( + config_obj=self.config_obj, + pre_item_stages=pre_item_stages, + post_item_stages=post_item_stages + ) + log.info("PipelineOrchestrator initialized successfully in ProcessingEngine with pre and post stages.") + except Exception as e: + log.error(f"Failed to initialize PipelineOrchestrator in ProcessingEngine: {e}", exc_info=True) + self.pipeline_orchestrator = None # Ensure it's None if init fails + + log.debug("ProcessingEngine initialized.") + + + def process( + self, + source_rule: SourceRule, + workspace_path: Path, + output_base_path: Path, + overwrite: bool = False, + incrementing_value: Optional[str] = None, + sha5_value: Optional[str] = None + ) -> Dict[str, List[str]]: + """ + Executes the processing pipeline for all assets defined in the SourceRule. + + Args: + source_rule: The SourceRule object containing explicit instructions for all assets and files. + workspace_path: The path to the directory containing the source files (e.g., extracted archive). + output_base_path: The base directory where processed output will be saved. + overwrite: If True, forces reprocessing even if output exists for an asset. + incrementing_value: Optional incrementing value for path tokens. + sha5_value: Optional SHA5 hash value for path tokens. + + Returns: + Dict[str, List[str]]: A dictionary summarizing the status of each asset: + {"processed": [asset_name1, ...], + "skipped": [asset_name2, ...], + "failed": [asset_name3, ...]} + """ + log.info(f"VERIFY: ProcessingEngine.process called with rule for input: {source_rule.input_path}") # DEBUG Verify + log.debug(f" VERIFY Rule Details: {source_rule}") # DEBUG Verify (Optional detailed log) + if not isinstance(source_rule, SourceRule): + raise ProcessingEngineError("process() requires a valid SourceRule object.") + if not isinstance(workspace_path, Path) or not workspace_path.is_dir(): + raise ProcessingEngineError(f"Invalid workspace path provided: {workspace_path}") + if not isinstance(output_base_path, Path): + raise ProcessingEngineError(f"Invalid output base path provided: {output_base_path}") + + log.info(f"ProcessingEngine starting process for {len(source_rule.assets)} asset(s) defined in SourceRule.") + overall_status = {"processed": [], "skipped": [], "failed": []} + self.loaded_data_cache = {} # Reset cache for this run + # Store incoming optional values for use in path generation + self.current_incrementing_value = incrementing_value + self.current_sha5_value = sha5_value + log.debug(f"Received incrementing_value: {self.current_incrementing_value}, sha5_value: {self.current_sha5_value}") + + # Use a temporary directory for intermediate files (like saved maps) + try: + self.temp_dir = Path(tempfile.mkdtemp(prefix=self.config_obj.temp_dir_prefix)) + log.debug(f"Created temporary workspace for engine: {self.temp_dir}") + # --- NEW PIPELINE ORCHESTRATOR LOGIC --- + if hasattr(self, 'pipeline_orchestrator') and self.pipeline_orchestrator: + log.info("Processing source rule using PipelineOrchestrator.") + overall_status = self.pipeline_orchestrator.process_source_rule( + source_rule=source_rule, + workspace_path=workspace_path, # This is the path to the source files (e.g. extracted archive) + output_base_path=output_base_path, + overwrite=overwrite, + incrementing_value=self.current_incrementing_value, + sha5_value=self.current_sha5_value + ) + else: + log.error(f"PipelineOrchestrator not available for SourceRule '{source_rule.input_path}'. Marking all {len(source_rule.assets)} assets as failed.") + for asset_rule in source_rule.assets: + overall_status["failed"].append(asset_rule.asset_name) + + log.info(f"ProcessingEngine finished. Summary: {overall_status}") + return overall_status + + except Exception as e: + log.exception(f"Processing engine failed unexpectedly: {e}") + # Ensure all assets not processed/skipped are marked as failed + processed_or_skipped = set(overall_status["processed"] + overall_status["skipped"]) + for asset_rule in source_rule.assets: + if asset_rule.asset_name not in processed_or_skipped: + overall_status["failed"].append(asset_rule.asset_name) + return overall_status # Return partial status if possible + finally: + self._cleanup_workspace() + + + def _cleanup_workspace(self): + """Removes the temporary workspace directory if it exists.""" + if self.temp_dir and self.temp_dir.exists(): + try: + log.debug(f"Cleaning up engine temporary workspace: {self.temp_dir}") + # Ignore errors during cleanup (e.g., permission errors on copied .git files) + shutil.rmtree(self.temp_dir, ignore_errors=True) + self.temp_dir = None + log.debug("Engine temporary workspace cleaned up successfully.") + except Exception as e: + log.error(f"Failed to remove engine temporary workspace {self.temp_dir}: {e}", exc_info=True) + self.loaded_data_cache = {} # Clear cache after cleanup + diff --git a/requirements.txt b/requirements.txt index f711dfe..90541f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ numpy openexr PySide6 py7zr -rarfile \ No newline at end of file +rarfile +requests \ No newline at end of file diff --git a/rule_structure.py b/rule_structure.py new file mode 100644 index 0000000..14b2313 --- /dev/null +++ b/rule_structure.py @@ -0,0 +1,57 @@ +import dataclasses +import json +from typing import List, Dict, Any, Tuple, Optional +@dataclasses.dataclass +class FileRule: + file_path: str = None + item_type: str = None # Base type determined by classification (e.g., MAP_COL, EXTRA) + item_type_override: str = None # Renamed from map_type_override + target_asset_name_override: str = None + resolution_override: Tuple[int, int] = None + channel_merge_instructions: Dict[str, Any] = dataclasses.field(default_factory=dict) + output_format_override: str = None + + def to_json(self) -> str: + return json.dumps(dataclasses.asdict(self), indent=4) + + @classmethod + def from_json(cls, json_string: str) -> 'FileRule': + data = json.loads(json_string) + return cls(**data) + +@dataclasses.dataclass +class AssetRule: + asset_name: str = None + asset_type: str = None # Predicted type + asset_type_override: str = None + common_metadata: Dict[str, Any] = dataclasses.field(default_factory=dict) + files: List[FileRule] = dataclasses.field(default_factory=list) + + def to_json(self) -> str: + return json.dumps(dataclasses.asdict(self), indent=4) + + @classmethod + def from_json(cls, json_string: str) -> 'AssetRule': + data = json.loads(json_string) + # Manually deserialize nested FileRule objects + data['files'] = [FileRule.from_json(json.dumps(file_data)) for file_data in data.get('files', [])] + return cls(**data) + +@dataclasses.dataclass +class SourceRule: + supplier_identifier: str = None # Predicted/Original identifier + supplier_override: str = None + high_level_sorting_parameters: Dict[str, Any] = dataclasses.field(default_factory=dict) + assets: List[AssetRule] = dataclasses.field(default_factory=list) + input_path: str = None + preset_name: str = None + + def to_json(self) -> str: + return json.dumps(dataclasses.asdict(self), indent=4) + + @classmethod + def from_json(cls, json_string: str) -> 'SourceRule': + data = json.loads(json_string) + # Manually deserialize nested AssetRule objects + data['assets'] = [AssetRule.from_json(json.dumps(asset_data)) for asset_data in data.get('assets', [])] + return cls(**data) \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..2e70fad --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# This file makes the 'tests' directory a Python package. \ No newline at end of file diff --git a/tests/processing/pipeline/__init__.py b/tests/processing/pipeline/__init__.py new file mode 100644 index 0000000..f178d82 --- /dev/null +++ b/tests/processing/pipeline/__init__.py @@ -0,0 +1 @@ +# This file makes Python treat the directory as a package. \ No newline at end of file diff --git a/tests/processing/pipeline/stages/__init__.py b/tests/processing/pipeline/stages/__init__.py new file mode 100644 index 0000000..f178d82 --- /dev/null +++ b/tests/processing/pipeline/stages/__init__.py @@ -0,0 +1 @@ +# This file makes Python treat the directory as a package. \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_alpha_extraction_to_mask.py b/tests/processing/pipeline/stages/test_alpha_extraction_to_mask.py new file mode 100644 index 0000000..198097b --- /dev/null +++ b/tests/processing/pipeline/stages/test_alpha_extraction_to_mask.py @@ -0,0 +1,273 @@ +import pytest +from unittest import mock +from pathlib import Path +import uuid +import numpy as np + +from processing.pipeline.stages.alpha_extraction_to_mask import AlphaExtractionToMaskStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule, FileRule +from configuration import Configuration, GeneralSettings +import processing.utils.image_processing_utils as ipu # Ensure ipu is available for mocking + +# Helper Functions +def create_mock_file_rule_for_alpha_test( + id_val: uuid.UUID = None, + map_type: str = "ALBEDO", + filename_pattern: str = "albedo.png", + item_type: str = "MAP_COL", + active: bool = True +) -> mock.MagicMock: + mock_fr = mock.MagicMock(spec=FileRule) + mock_fr.id = id_val if id_val else uuid.uuid4() + mock_fr.map_type = map_type + mock_fr.filename_pattern = filename_pattern + mock_fr.item_type = item_type + mock_fr.active = active + mock_fr.transform_settings = mock.MagicMock(spec=TransformSettings) + return mock_fr + +def create_alpha_extraction_mock_context( + initial_file_rules: list = None, + initial_processed_details: dict = None, + skip_asset_flag: bool = False, + asset_name: str = "AlphaAsset", + # extract_alpha_globally: bool = True # If stage checks this +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + + mock_source_rule = mock.MagicMock(spec=SourceRule) + + mock_gs = mock.MagicMock(spec=GeneralSettings) + # if your stage uses a global flag: + # mock_gs.extract_alpha_to_mask_globally = extract_alpha_globally + + mock_config = mock.MagicMock(spec=Configuration) + mock_config.general_settings = mock_gs + + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp_engine_dir"), + output_base_path=Path("/fake/output"), + effective_supplier="ValidSupplier", + asset_metadata={'asset_name': asset_name}, + processed_maps_details=initial_processed_details if initial_processed_details is not None else {}, + merged_maps_details={}, + files_to_process=list(initial_file_rules) if initial_file_rules else [], + loaded_data_cache={}, + config_obj=mock_config, + status_flags={'skip_asset': skip_asset_flag}, + incrementing_value=None, + sha5_value=None + ) + return context + +# Unit Tests +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image') +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image') +@mock.patch('logging.info') # Mock logging to avoid console output during tests +def test_asset_skipped(mock_log_info, mock_load_image, mock_save_image): + stage = AlphaExtractionToMaskStage() + context = create_alpha_extraction_mock_context(skip_asset_flag=True) + + updated_context = stage.execute(context) + + assert updated_context == context # Context should be unchanged + mock_load_image.assert_not_called() + mock_save_image.assert_not_called() + assert len(updated_context.files_to_process) == 0 + assert not updated_context.processed_maps_details + +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image') +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image') +@mock.patch('logging.info') +def test_existing_mask_map(mock_log_info, mock_load_image, mock_save_image): + stage = AlphaExtractionToMaskStage() + + existing_mask_rule = create_mock_file_rule_for_alpha_test(map_type="MASK", filename_pattern="mask.png") + context = create_alpha_extraction_mock_context(initial_file_rules=[existing_mask_rule]) + + updated_context = stage.execute(context) + + assert updated_context == context + mock_load_image.assert_not_called() + mock_save_image.assert_not_called() + assert len(updated_context.files_to_process) == 1 + assert updated_context.files_to_process[0].map_type == "MASK" + +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image') +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image') +@mock.patch('logging.info') +def test_alpha_extraction_success(mock_log_info, mock_load_image, mock_save_image): + stage = AlphaExtractionToMaskStage() + + albedo_rule_id = uuid.uuid4() + albedo_fr = create_mock_file_rule_for_alpha_test(id_val=albedo_rule_id, map_type="ALBEDO") + + initial_processed_details = { + albedo_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_albedo.png', 'status': 'Processed', 'map_type': 'ALBEDO', 'source_file_path': Path('/fake/source/albedo.png')} + } + context = create_alpha_extraction_mock_context( + initial_file_rules=[albedo_fr], + initial_processed_details=initial_processed_details + ) + + mock_rgba_data = np.zeros((10, 10, 4), dtype=np.uint8) + mock_rgba_data[:, :, 3] = 128 # Example alpha data + mock_load_image.side_effect = [mock_rgba_data, mock_rgba_data] + + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + assert mock_load_image.call_count == 2 + # First call to check for alpha, second to get data for saving + mock_load_image.assert_any_call(Path('/fake/temp_engine_dir/processed_albedo.png')) + + mock_save_image.assert_called_once() + saved_path_arg = mock_save_image.call_args[0][0] + saved_data_arg = mock_save_image.call_args[0][1] + + assert isinstance(saved_path_arg, Path) + assert "mask_from_alpha_" in saved_path_arg.name + assert np.array_equal(saved_data_arg, mock_rgba_data[:, :, 3]) + + assert len(updated_context.files_to_process) == 2 + new_mask_rule = None + for fr in updated_context.files_to_process: + if fr.map_type == "MASK": + new_mask_rule = fr + break + assert new_mask_rule is not None + assert new_mask_rule.item_type == "MAP_DER" # Derived map + + assert new_mask_rule.id.hex in updated_context.processed_maps_details + new_mask_detail = updated_context.processed_maps_details[new_mask_rule.id.hex] + assert new_mask_detail['map_type'] == "MASK" + assert "mask_from_alpha_" in new_mask_detail['temp_processed_file'] + assert "Generated from alpha of ALBEDO" in new_mask_detail['notes'] # Check for specific note + assert new_mask_detail['status'] == 'Processed' + +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image') +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image') +@mock.patch('logging.info') +def test_no_alpha_channel_in_source(mock_log_info, mock_load_image, mock_save_image): + stage = AlphaExtractionToMaskStage() + + albedo_rule_id = uuid.uuid4() + albedo_fr = create_mock_file_rule_for_alpha_test(id_val=albedo_rule_id, map_type="ALBEDO") + initial_processed_details = { + albedo_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_rgb_albedo.png', 'status': 'Processed', 'map_type': 'ALBEDO', 'source_file_path': Path('/fake/source/albedo_rgb.png')} + } + context = create_alpha_extraction_mock_context( + initial_file_rules=[albedo_fr], + initial_processed_details=initial_processed_details + ) + + mock_rgb_data = np.zeros((10, 10, 3), dtype=np.uint8) # RGB, no alpha + mock_load_image.return_value = mock_rgb_data # Only called once for check + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(Path('/fake/temp_engine_dir/processed_rgb_albedo.png')) + mock_save_image.assert_not_called() + assert len(updated_context.files_to_process) == 1 # No new MASK rule + assert albedo_fr.id.hex in updated_context.processed_maps_details + assert len(updated_context.processed_maps_details) == 1 + + +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image') +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image') +@mock.patch('logging.info') +def test_no_suitable_source_map_type(mock_log_info, mock_load_image, mock_save_image): + stage = AlphaExtractionToMaskStage() + + normal_rule_id = uuid.uuid4() + normal_fr = create_mock_file_rule_for_alpha_test(id_val=normal_rule_id, map_type="NORMAL") + initial_processed_details = { + normal_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_normal.png', 'status': 'Processed', 'map_type': 'NORMAL'} + } + context = create_alpha_extraction_mock_context( + initial_file_rules=[normal_fr], + initial_processed_details=initial_processed_details + ) + + updated_context = stage.execute(context) + + mock_load_image.assert_not_called() + mock_save_image.assert_not_called() + assert len(updated_context.files_to_process) == 1 + assert normal_fr.id.hex in updated_context.processed_maps_details + assert len(updated_context.processed_maps_details) == 1 + +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image') +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image') +@mock.patch('logging.warning') # Expect a warning log +def test_load_image_fails(mock_log_warning, mock_load_image, mock_save_image): + stage = AlphaExtractionToMaskStage() + + albedo_rule_id = uuid.uuid4() + albedo_fr = create_mock_file_rule_for_alpha_test(id_val=albedo_rule_id, map_type="ALBEDO") + initial_processed_details = { + albedo_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_albedo_load_fail.png', 'status': 'Processed', 'map_type': 'ALBEDO', 'source_file_path': Path('/fake/source/albedo_load_fail.png')} + } + context = create_alpha_extraction_mock_context( + initial_file_rules=[albedo_fr], + initial_processed_details=initial_processed_details + ) + + mock_load_image.return_value = None # Simulate load failure + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(Path('/fake/temp_engine_dir/processed_albedo_load_fail.png')) + mock_save_image.assert_not_called() + assert len(updated_context.files_to_process) == 1 + assert albedo_fr.id.hex in updated_context.processed_maps_details + assert len(updated_context.processed_maps_details) == 1 + mock_log_warning.assert_called_once() # Check that a warning was logged + +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image') +@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image') +@mock.patch('logging.error') # Expect an error log +def test_save_image_fails(mock_log_error, mock_load_image, mock_save_image): + stage = AlphaExtractionToMaskStage() + + albedo_rule_id = uuid.uuid4() + albedo_fr = create_mock_file_rule_for_alpha_test(id_val=albedo_rule_id, map_type="ALBEDO") + initial_processed_details = { + albedo_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_albedo_save_fail.png', 'status': 'Processed', 'map_type': 'ALBEDO', 'source_file_path': Path('/fake/source/albedo_save_fail.png')} + } + context = create_alpha_extraction_mock_context( + initial_file_rules=[albedo_fr], + initial_processed_details=initial_processed_details + ) + + mock_rgba_data = np.zeros((10, 10, 4), dtype=np.uint8) + mock_rgba_data[:, :, 3] = 128 + mock_load_image.side_effect = [mock_rgba_data, mock_rgba_data] # Load succeeds + + mock_save_image.return_value = False # Simulate save failure + + updated_context = stage.execute(context) + + assert mock_load_image.call_count == 2 + mock_save_image.assert_called_once() # Save was attempted + + assert len(updated_context.files_to_process) == 1 # No new MASK rule should be successfully added and detailed + + # Check that no new MASK details were added, or if they were, they reflect failure. + # The current stage logic returns context early, so no new rule or details should be present. + mask_rule_found = any(fr.map_type == "MASK" for fr in updated_context.files_to_process) + assert not mask_rule_found + + mask_details_found = any( + details['map_type'] == "MASK" + for fr_id, details in updated_context.processed_maps_details.items() + if fr_id != albedo_fr.id.hex # Exclude the original albedo + ) + assert not mask_details_found + mock_log_error.assert_called_once() # Check that an error was logged \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_asset_skip_logic.py b/tests/processing/pipeline/stages/test_asset_skip_logic.py new file mode 100644 index 0000000..388cc8c --- /dev/null +++ b/tests/processing/pipeline/stages/test_asset_skip_logic.py @@ -0,0 +1,213 @@ +import pytest +from unittest import mock +from pathlib import Path +from typing import Dict, Optional, Any + +from processing.pipeline.stages.asset_skip_logic import AssetSkipLogicStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule +from configuration import Configuration, GeneralSettings + +# Helper function to create a mock AssetProcessingContext +def create_skip_logic_mock_context( + effective_supplier: Optional[str] = "ValidSupplier", + asset_process_status: str = "PENDING", + overwrite_existing: bool = False, + asset_name: str = "TestAssetSkip" +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + mock_asset_rule.process_status = asset_process_status + mock_asset_rule.source_path = "fake/source" # Added for completeness + mock_asset_rule.output_path = "fake/output" # Added for completeness + mock_asset_rule.maps = [] # Added for completeness + mock_asset_rule.metadata = {} # Added for completeness + mock_asset_rule.material_name = None # Added for completeness + mock_asset_rule.notes = None # Added for completeness + mock_asset_rule.tags = [] # Added for completeness + mock_asset_rule.enabled = True # Added for completeness + + + mock_source_rule = mock.MagicMock(spec=SourceRule) + mock_source_rule.name = "TestSourceRule" # Added for completeness + mock_source_rule.path = "fake/source_rule_path" # Added for completeness + mock_source_rule.default_supplier = None # Added for completeness + mock_source_rule.assets = [mock_asset_rule] # Added for completeness + mock_source_rule.enabled = True # Added for completeness + + mock_general_settings = mock.MagicMock(spec=GeneralSettings) + mock_general_settings.overwrite_existing = overwrite_existing + + mock_config = mock.MagicMock(spec=Configuration) + mock_config.general_settings = mock_general_settings + mock_config.suppliers = {"ValidSupplier": mock.MagicMock()} + + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp"), + output_base_path=Path("/fake/output"), + effective_supplier=effective_supplier, + asset_metadata={}, + processed_maps_details={}, + merged_maps_details={}, + files_to_process=[], + loaded_data_cache={}, + config_obj=mock_config, + status_flags={}, + incrementing_value=None, + sha5_value=None # Corrected from sha5_value to sha256_value if that's the actual field + ) + # Ensure status_flags is initialized if AssetSkipLogicStage expects it + # context.status_flags = {} # Already done in constructor + return context +@mock.patch('logging.info') +def test_skip_due_to_missing_supplier(mock_log_info): + """ + Test that the asset is skipped if effective_supplier is None. + """ + stage = AssetSkipLogicStage() + context = create_skip_logic_mock_context(effective_supplier=None, asset_name="MissingSupplierAsset") + + updated_context = stage.execute(context) + + assert updated_context.status_flags.get('skip_asset') is True + assert updated_context.status_flags.get('skip_reason') == "Invalid or missing supplier" + mock_log_info.assert_any_call(f"Asset 'MissingSupplierAsset': Skipping due to missing or invalid supplier.") + +@mock.patch('logging.info') +def test_skip_due_to_process_status_skip(mock_log_info): + """ + Test that the asset is skipped if asset_rule.process_status is "SKIP". + """ + stage = AssetSkipLogicStage() + context = create_skip_logic_mock_context(asset_process_status="SKIP", asset_name="SkipStatusAsset") + + updated_context = stage.execute(context) + + assert updated_context.status_flags.get('skip_asset') is True + assert updated_context.status_flags.get('skip_reason') == "Process status set to SKIP" + mock_log_info.assert_any_call(f"Asset 'SkipStatusAsset': Skipping because process_status is 'SKIP'.") + +@mock.patch('logging.info') +def test_skip_due_to_processed_and_overwrite_disabled(mock_log_info): + """ + Test that the asset is skipped if asset_rule.process_status is "PROCESSED" + and overwrite_existing is False. + """ + stage = AssetSkipLogicStage() + context = create_skip_logic_mock_context( + asset_process_status="PROCESSED", + overwrite_existing=False, + asset_name="ProcessedNoOverwriteAsset" + ) + + updated_context = stage.execute(context) + + assert updated_context.status_flags.get('skip_asset') is True + assert updated_context.status_flags.get('skip_reason') == "Already processed, overwrite disabled" + mock_log_info.assert_any_call(f"Asset 'ProcessedNoOverwriteAsset': Skipping because already processed and overwrite is disabled.") + +@mock.patch('logging.info') +def test_no_skip_when_processed_and_overwrite_enabled(mock_log_info): + """ + Test that the asset is NOT skipped if asset_rule.process_status is "PROCESSED" + but overwrite_existing is True. + """ + stage = AssetSkipLogicStage() + context = create_skip_logic_mock_context( + asset_process_status="PROCESSED", + overwrite_existing=True, + effective_supplier="ValidSupplier", # Ensure supplier is valid + asset_name="ProcessedOverwriteAsset" + ) + + updated_context = stage.execute(context) + + assert updated_context.status_flags.get('skip_asset', False) is False # Default to False if key not present + # No specific skip_reason to check if not skipped + # Check that no skip log message was called for this specific reason + for call_args in mock_log_info.call_args_list: + assert "Skipping because already processed and overwrite is disabled" not in call_args[0][0] + assert "Skipping due to missing or invalid supplier" not in call_args[0][0] + assert "Skipping because process_status is 'SKIP'" not in call_args[0][0] + + +@mock.patch('logging.info') +def test_no_skip_when_process_status_pending(mock_log_info): + """ + Test that the asset is NOT skipped if asset_rule.process_status is "PENDING". + """ + stage = AssetSkipLogicStage() + context = create_skip_logic_mock_context( + asset_process_status="PENDING", + effective_supplier="ValidSupplier", # Ensure supplier is valid + asset_name="PendingAsset" + ) + + updated_context = stage.execute(context) + + assert updated_context.status_flags.get('skip_asset', False) is False + # Check that no skip log message was called + for call_args in mock_log_info.call_args_list: + assert "Skipping" not in call_args[0][0] + + +@mock.patch('logging.info') +def test_no_skip_when_process_status_failed_previously(mock_log_info): + """ + Test that the asset is NOT skipped if asset_rule.process_status is "FAILED_PREVIOUSLY". + """ + stage = AssetSkipLogicStage() + context = create_skip_logic_mock_context( + asset_process_status="FAILED_PREVIOUSLY", + effective_supplier="ValidSupplier", # Ensure supplier is valid + asset_name="FailedPreviouslyAsset" + ) + + updated_context = stage.execute(context) + + assert updated_context.status_flags.get('skip_asset', False) is False + # Check that no skip log message was called + for call_args in mock_log_info.call_args_list: + assert "Skipping" not in call_args[0][0] + +@mock.patch('logging.info') +def test_no_skip_when_process_status_other_valid_status(mock_log_info): + """ + Test that the asset is NOT skipped for other valid, non-skip process statuses. + """ + stage = AssetSkipLogicStage() + context = create_skip_logic_mock_context( + asset_process_status="READY_FOR_PROCESSING", # Example of another non-skip status + effective_supplier="ValidSupplier", + asset_name="ReadyAsset" + ) + updated_context = stage.execute(context) + assert updated_context.status_flags.get('skip_asset', False) is False + for call_args in mock_log_info.call_args_list: + assert "Skipping" not in call_args[0][0] + +@mock.patch('logging.info') +def test_skip_asset_flag_initialized_if_not_present(mock_log_info): + """ + Test that 'skip_asset' is initialized to False in status_flags if not skipped and not present. + """ + stage = AssetSkipLogicStage() + context = create_skip_logic_mock_context( + asset_process_status="PENDING", + effective_supplier="ValidSupplier", + asset_name="InitFlagAsset" + ) + # Ensure status_flags is empty before execute + context.status_flags = {} + + updated_context = stage.execute(context) + + # If not skipped, 'skip_asset' should be explicitly False. + assert updated_context.status_flags.get('skip_asset') is False + # No skip reason should be set + assert 'skip_reason' not in updated_context.status_flags + for call_args in mock_log_info.call_args_list: + assert "Skipping" not in call_args[0][0] \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_file_rule_filter.py b/tests/processing/pipeline/stages/test_file_rule_filter.py new file mode 100644 index 0000000..4a79308 --- /dev/null +++ b/tests/processing/pipeline/stages/test_file_rule_filter.py @@ -0,0 +1,330 @@ +import pytest +from unittest import mock +from pathlib import Path +import uuid +from typing import Optional # Added Optional for type hinting + +from processing.pipeline.stages.file_rule_filter import FileRuleFilterStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule, FileRule # FileRule is key here +from configuration import Configuration # Minimal config needed + +def create_mock_file_rule( + id_val: Optional[uuid.UUID] = None, + map_type: str = "Diffuse", + filename_pattern: str = "*.tif", + item_type: str = "MAP_COL", # e.g., MAP_COL, FILE_IGNORE + active: bool = True +) -> mock.MagicMock: # Return MagicMock to easily set other attributes if needed + mock_fr = mock.MagicMock(spec=FileRule) + mock_fr.id = id_val if id_val else uuid.uuid4() + mock_fr.map_type = map_type + mock_fr.filename_pattern = filename_pattern + mock_fr.item_type = item_type + mock_fr.active = active + return mock_fr + +def create_file_filter_mock_context( + file_rules_list: Optional[list] = None, # List of mock FileRule objects + skip_asset_flag: bool = False, + asset_name: str = "FileFilterAsset" +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + mock_asset_rule.file_rules = file_rules_list if file_rules_list is not None else [] + + mock_source_rule = mock.MagicMock(spec=SourceRule) + mock_config = mock.MagicMock(spec=Configuration) + + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp"), + output_base_path=Path("/fake/output"), + effective_supplier="ValidSupplier", # Assume valid for this stage + asset_metadata={'asset_name': asset_name}, # Assume metadata init happened + processed_maps_details={}, + merged_maps_details={}, + files_to_process=[], # Stage will populate this + loaded_data_cache={}, + config_obj=mock_config, + status_flags={'skip_asset': skip_asset_flag}, + incrementing_value=None, + sha5_value=None # Corrected from sha5_value to sha256_value based on AssetProcessingContext + ) + return context +# Test Cases for FileRuleFilterStage.execute() + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_asset_skipped(mock_log_debug, mock_log_info): + """ + Test case: Asset Skipped - status_flags['skip_asset'] is True. + Assert context.files_to_process remains empty. + """ + stage = FileRuleFilterStage() + context = create_file_filter_mock_context(skip_asset_flag=True) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 0 + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule filtering as 'skip_asset' is True.") + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_no_file_rules(mock_log_debug, mock_log_info): + """ + Test case: No File Rules - asset_rule.file_rules is empty. + Assert context.files_to_process is empty. + """ + stage = FileRuleFilterStage() + context = create_file_filter_mock_context(file_rules_list=[]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 0 + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': No file rules defined. Skipping file rule filtering.") + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_only_active_processable_rules(mock_log_debug, mock_log_info): + """ + Test case: Only Active, Processable Rules - All FileRules are active=True and item_type="MAP_COL". + Assert all are added to context.files_to_process. + """ + stage = FileRuleFilterStage() + fr1 = create_mock_file_rule(filename_pattern="diffuse.png", item_type="MAP_COL", active=True) + fr2 = create_mock_file_rule(filename_pattern="normal.png", item_type="MAP_COL", active=True) + context = create_file_filter_mock_context(file_rules_list=[fr1, fr2]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 2 + assert fr1 in updated_context.files_to_process + assert fr2 in updated_context.files_to_process + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 2 file rules queued for processing after filtering.") + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_inactive_rules(mock_log_debug, mock_log_info): + """ + Test case: Inactive Rules - Some FileRules have active=False. + Assert only active rules are added. + """ + stage = FileRuleFilterStage() + fr_active = create_mock_file_rule(filename_pattern="active.png", item_type="MAP_COL", active=True) + fr_inactive = create_mock_file_rule(filename_pattern="inactive.png", item_type="MAP_COL", active=False) + fr_another_active = create_mock_file_rule(filename_pattern="another_active.jpg", item_type="MAP_COL", active=True) + context = create_file_filter_mock_context(file_rules_list=[fr_active, fr_inactive, fr_another_active]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 2 + assert fr_active in updated_context.files_to_process + assert fr_another_active in updated_context.files_to_process + assert fr_inactive not in updated_context.files_to_process + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping inactive file rule: '{fr_inactive.filename_pattern}'") + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 2 file rules queued for processing after filtering.") + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_file_ignore_simple_match(mock_log_debug, mock_log_info): + """ + Test case: FILE_IGNORE Rule (Simple Match). + One FILE_IGNORE rule with filename_pattern="*_ignore.png". + One MAP_COL rule with filename_pattern="diffuse_ignore.png". + One MAP_COL rule with filename_pattern="normal_process.png". + Assert only "normal_process.png" rule is added. + """ + stage = FileRuleFilterStage() + fr_ignore = create_mock_file_rule(filename_pattern="*_ignore.png", item_type="FILE_IGNORE", active=True) + fr_ignored_map = create_mock_file_rule(filename_pattern="diffuse_ignore.png", item_type="MAP_COL", active=True) + fr_process_map = create_mock_file_rule(filename_pattern="normal_process.png", item_type="MAP_COL", active=True) + context = create_file_filter_mock_context(file_rules_list=[fr_ignore, fr_ignored_map, fr_process_map]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 1 + assert fr_process_map in updated_context.files_to_process + assert fr_ignored_map not in updated_context.files_to_process + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore.filename_pattern}'") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_ignored_map.filename_pattern}' due to matching ignore pattern.") + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 1 file rules queued for processing after filtering.") + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_file_ignore_glob_pattern(mock_log_debug, mock_log_info): + """ + Test case: FILE_IGNORE Rule (Glob Pattern). + One FILE_IGNORE rule with filename_pattern="*_ignore.*". + MAP_COL rules: "tex_ignore.tif", "tex_process.png". + Assert only "tex_process.png" rule is added. + """ + stage = FileRuleFilterStage() + fr_ignore_glob = create_mock_file_rule(filename_pattern="*_ignore.*", item_type="FILE_IGNORE", active=True) + fr_ignored_tif = create_mock_file_rule(filename_pattern="tex_ignore.tif", item_type="MAP_COL", active=True) + fr_process_png = create_mock_file_rule(filename_pattern="tex_process.png", item_type="MAP_COL", active=True) + context = create_file_filter_mock_context(file_rules_list=[fr_ignore_glob, fr_ignored_tif, fr_process_png]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 1 + assert fr_process_png in updated_context.files_to_process + assert fr_ignored_tif not in updated_context.files_to_process + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore_glob.filename_pattern}'") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_ignored_tif.filename_pattern}' due to matching ignore pattern.") + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 1 file rules queued for processing after filtering.") + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_multiple_file_ignore_rules(mock_log_debug, mock_log_info): + """ + Test case: Multiple FILE_IGNORE Rules. + Test with several ignore patterns and ensure they are all respected. + """ + stage = FileRuleFilterStage() + fr_ignore1 = create_mock_file_rule(filename_pattern="*.tmp", item_type="FILE_IGNORE", active=True) + fr_ignore2 = create_mock_file_rule(filename_pattern="backup_*", item_type="FILE_IGNORE", active=True) + fr_ignore3 = create_mock_file_rule(filename_pattern="*_old.png", item_type="FILE_IGNORE", active=True) + + fr_map_ignored1 = create_mock_file_rule(filename_pattern="data.tmp", item_type="MAP_COL", active=True) + fr_map_ignored2 = create_mock_file_rule(filename_pattern="backup_diffuse.jpg", item_type="MAP_COL", active=True) + fr_map_ignored3 = create_mock_file_rule(filename_pattern="normal_old.png", item_type="MAP_COL", active=True) + fr_map_process = create_mock_file_rule(filename_pattern="final_texture.tif", item_type="MAP_COL", active=True) + + context = create_file_filter_mock_context(file_rules_list=[ + fr_ignore1, fr_ignore2, fr_ignore3, + fr_map_ignored1, fr_map_ignored2, fr_map_ignored3, fr_map_process + ]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 1 + assert fr_map_process in updated_context.files_to_process + assert fr_map_ignored1 not in updated_context.files_to_process + assert fr_map_ignored2 not in updated_context.files_to_process + assert fr_map_ignored3 not in updated_context.files_to_process + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore1.filename_pattern}'") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore2.filename_pattern}'") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore3.filename_pattern}'") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_map_ignored1.filename_pattern}' due to matching ignore pattern.") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_map_ignored2.filename_pattern}' due to matching ignore pattern.") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_map_ignored3.filename_pattern}' due to matching ignore pattern.") + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 1 file rules queued for processing after filtering.") + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_file_ignore_rule_is_inactive(mock_log_debug, mock_log_info): + """ + Test case: FILE_IGNORE Rule is Inactive. + An ignore rule itself is active=False. Assert its pattern is NOT used for filtering. + """ + stage = FileRuleFilterStage() + fr_inactive_ignore = create_mock_file_rule(filename_pattern="*_ignore.tif", item_type="FILE_IGNORE", active=False) + fr_should_process1 = create_mock_file_rule(filename_pattern="diffuse_ignore.tif", item_type="MAP_COL", active=True) # Should be processed + fr_should_process2 = create_mock_file_rule(filename_pattern="normal_ok.png", item_type="MAP_COL", active=True) + context = create_file_filter_mock_context(file_rules_list=[fr_inactive_ignore, fr_should_process1, fr_should_process2]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 2 + assert fr_should_process1 in updated_context.files_to_process + assert fr_should_process2 in updated_context.files_to_process + # Ensure the inactive ignore rule's pattern was not registered + # We check this by ensuring no debug log for registering *that specific* pattern was made. + # A more robust way would be to check mock_log_debug.call_args_list, but this is simpler for now. + for call in mock_log_debug.call_args_list: + args, kwargs = call + if "Registering ignore pattern" in args[0] and fr_inactive_ignore.filename_pattern in args[0]: + pytest.fail(f"Inactive ignore pattern '{fr_inactive_ignore.filename_pattern}' was incorrectly registered.") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping inactive file rule: '{fr_inactive_ignore.filename_pattern}' (type: FILE_IGNORE)") + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 2 file rules queued for processing after filtering.") + + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_no_file_ignore_rules(mock_log_debug, mock_log_info): + """ + Test case: No FILE_IGNORE Rules. + All rules are MAP_COL or other processable types. + Assert all active, processable rules are included. + """ + stage = FileRuleFilterStage() + fr1 = create_mock_file_rule(filename_pattern="diffuse.png", item_type="MAP_COL", active=True) + fr2 = create_mock_file_rule(filename_pattern="normal.png", item_type="MAP_COL", active=True) + fr_other_type = create_mock_file_rule(filename_pattern="spec.tif", item_type="MAP_SPEC", active=True) # Assuming MAP_SPEC is processable + fr_inactive = create_mock_file_rule(filename_pattern="ao.jpg", item_type="MAP_AO", active=False) + + context = create_file_filter_mock_context(file_rules_list=[fr1, fr2, fr_other_type, fr_inactive]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 3 + assert fr1 in updated_context.files_to_process + assert fr2 in updated_context.files_to_process + assert fr_other_type in updated_context.files_to_process + assert fr_inactive not in updated_context.files_to_process + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping inactive file rule: '{fr_inactive.filename_pattern}'") + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 3 file rules queued for processing after filtering.") + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_item_type_not_processable(mock_log_debug, mock_log_info): + """ + Test case: Item type is not processable (e.g., not MAP_COL, MAP_AO etc., but something else like 'METADATA_ONLY'). + Assert such rules are not added to files_to_process, unless they are FILE_IGNORE. + """ + stage = FileRuleFilterStage() + fr_processable = create_mock_file_rule(filename_pattern="diffuse.png", item_type="MAP_COL", active=True) + fr_not_processable = create_mock_file_rule(filename_pattern="info.txt", item_type="METADATA_ONLY", active=True) + fr_ignore = create_mock_file_rule(filename_pattern="*.bak", item_type="FILE_IGNORE", active=True) + fr_ignored_by_bak = create_mock_file_rule(filename_pattern="diffuse.bak", item_type="MAP_COL", active=True) + + context = create_file_filter_mock_context(file_rules_list=[fr_processable, fr_not_processable, fr_ignore, fr_ignored_by_bak]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 1 + assert fr_processable in updated_context.files_to_process + assert fr_not_processable not in updated_context.files_to_process + assert fr_ignored_by_bak not in updated_context.files_to_process + + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore.filename_pattern}'") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_not_processable.filename_pattern}' as its item_type '{fr_not_processable.item_type}' is not processable.") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_ignored_by_bak.filename_pattern}' due to matching ignore pattern.") + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 1 file rules queued for processing after filtering.") + +# Example tests from instructions (can be adapted or used as a base) +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_basic_active_example(mock_log_debug, mock_log_info): # Renamed to avoid conflict + stage = FileRuleFilterStage() + fr1 = create_mock_file_rule(filename_pattern="diffuse.png", item_type="MAP_COL", active=True) + fr2 = create_mock_file_rule(filename_pattern="normal.png", item_type="MAP_COL", active=True) + context = create_file_filter_mock_context(file_rules_list=[fr1, fr2]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 2 + assert fr1 in updated_context.files_to_process + assert fr2 in updated_context.files_to_process + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 2 file rules queued for processing after filtering.") + +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_file_rule_filter_with_file_ignore_example(mock_log_debug, mock_log_info): # Renamed to avoid conflict + stage = FileRuleFilterStage() + fr_ignore = create_mock_file_rule(filename_pattern="*_ignore.tif", item_type="FILE_IGNORE", active=True) + fr_process = create_mock_file_rule(filename_pattern="diffuse_ok.tif", item_type="MAP_COL", active=True) + fr_skip = create_mock_file_rule(filename_pattern="normal_ignore.tif", item_type="MAP_COL", active=True) + context = create_file_filter_mock_context(file_rules_list=[fr_ignore, fr_process, fr_skip]) + + updated_context = stage.execute(context) + + assert len(updated_context.files_to_process) == 1 + assert fr_process in updated_context.files_to_process + assert fr_skip not in updated_context.files_to_process + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore.filename_pattern}'") + mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_skip.filename_pattern}' due to matching ignore pattern.") + mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 1 file rules queued for processing after filtering.") \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_gloss_to_rough_conversion.py b/tests/processing/pipeline/stages/test_gloss_to_rough_conversion.py new file mode 100644 index 0000000..934ad2c --- /dev/null +++ b/tests/processing/pipeline/stages/test_gloss_to_rough_conversion.py @@ -0,0 +1,486 @@ +import pytest +from unittest import mock +from pathlib import Path +import uuid +import numpy as np +from typing import Optional, List, Dict + +from processing.pipeline.stages.gloss_to_rough_conversion import GlossToRoughConversionStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule, FileRule +from configuration import Configuration, GeneralSettings +# No direct ipu import needed in test if we mock its usage by the stage + +def create_mock_file_rule_for_gloss_test( + id_val: Optional[uuid.UUID] = None, + map_type: str = "GLOSS", # Test with GLOSS and other types + filename_pattern: str = "gloss.png" +) -> mock.MagicMock: + mock_fr = mock.MagicMock(spec=FileRule) + mock_fr.id = id_val if id_val else uuid.uuid4() + mock_fr.map_type = map_type + mock_fr.filename_pattern = filename_pattern + mock_fr.item_type = "MAP_COL" + mock_fr.active = True + return mock_fr + +def create_gloss_conversion_mock_context( + initial_file_rules: Optional[List[FileRule]] = None, # Type hint corrected + initial_processed_details: Optional[Dict] = None, # Type hint corrected + skip_asset_flag: bool = False, + asset_name: str = "GlossAsset", + # Add a mock for general_settings if your stage checks a global flag + # convert_gloss_globally: bool = True +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + mock_asset_rule.file_rules = initial_file_rules if initial_file_rules is not None else [] + + mock_source_rule = mock.MagicMock(spec=SourceRule) + + mock_gs = mock.MagicMock(spec=GeneralSettings) + # if your stage uses a global flag: + # mock_gs.convert_gloss_to_rough_globally = convert_gloss_globally + + mock_config = mock.MagicMock(spec=Configuration) + mock_config.general_settings = mock_gs + + + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp_engine_dir"), # Important for new temp file paths + output_base_path=Path("/fake/output"), + effective_supplier="ValidSupplier", + asset_metadata={'asset_name': asset_name}, + processed_maps_details=initial_processed_details if initial_processed_details is not None else {}, + merged_maps_details={}, + files_to_process=list(initial_file_rules) if initial_file_rules else [], # Stage modifies this list + loaded_data_cache={}, + config_obj=mock_config, + status_flags={'skip_asset': skip_asset_flag}, + incrementing_value=None, # Added as per AssetProcessingContext definition + sha5_value=None # Added as per AssetProcessingContext definition + ) + return context + +# Unit tests will be added below +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image') +def test_asset_skipped(mock_load_image, mock_save_image): + """ + Test that if 'skip_asset' is True, no processing occurs. + """ + stage = GlossToRoughConversionStage() + + gloss_rule_id = uuid.uuid4() + gloss_fr = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS") + + initial_details = { + gloss_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_gloss_map.png', 'status': 'Processed', 'map_type': 'GLOSS'} + } + context = create_gloss_conversion_mock_context( + initial_file_rules=[gloss_fr], + initial_processed_details=initial_details, + skip_asset_flag=True # Asset is skipped + ) + + # Keep a copy of files_to_process and processed_maps_details to compare + original_files_to_process = list(context.files_to_process) + original_processed_maps_details = context.processed_maps_details.copy() + + updated_context = stage.execute(context) + + mock_load_image.assert_not_called() + mock_save_image.assert_not_called() + + assert updated_context.files_to_process == original_files_to_process, "files_to_process should not change if asset is skipped" + assert updated_context.processed_maps_details == original_processed_maps_details, "processed_maps_details should not change if asset is skipped" + assert updated_context.status_flags['skip_asset'] is True +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image') +def test_no_gloss_map_present(mock_load_image, mock_save_image): + """ + Test that if no GLOSS maps are in files_to_process, no conversion occurs. + """ + stage = GlossToRoughConversionStage() + + normal_rule_id = uuid.uuid4() + normal_fr = create_mock_file_rule_for_gloss_test(id_val=normal_rule_id, map_type="NORMAL", filename_pattern="normal.png") + albedo_fr = create_mock_file_rule_for_gloss_test(map_type="ALBEDO", filename_pattern="albedo.jpg") + + initial_details = { + normal_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_normal_map.png', 'status': 'Processed', 'map_type': 'NORMAL'} + } + context = create_gloss_conversion_mock_context( + initial_file_rules=[normal_fr, albedo_fr], + initial_processed_details=initial_details + ) + + original_files_to_process = list(context.files_to_process) + original_processed_maps_details = context.processed_maps_details.copy() + + updated_context = stage.execute(context) + + mock_load_image.assert_not_called() + mock_save_image.assert_not_called() + + assert updated_context.files_to_process == original_files_to_process, "files_to_process should not change if no GLOSS maps are present" + assert updated_context.processed_maps_details == original_processed_maps_details, "processed_maps_details should not change if no GLOSS maps are present" + + # Ensure map types of existing rules are unchanged + for fr_in_list in updated_context.files_to_process: + if fr_in_list.id == normal_fr.id: + assert fr_in_list.map_type == "NORMAL" + elif fr_in_list.id == albedo_fr.id: + assert fr_in_list.map_type == "ALBEDO" +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging') # Mock logging +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image') +def test_gloss_conversion_uint8_success(mock_load_image, mock_save_image, mock_logging): + """ + Test successful conversion of a GLOSS map (uint8 data) to ROUGHNESS. + """ + stage = GlossToRoughConversionStage() + + gloss_rule_id = uuid.uuid4() + # Use a distinct filename for the gloss map to ensure correct path construction + gloss_fr = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS", filename_pattern="my_gloss_map.png") + other_fr_id = uuid.uuid4() + other_fr = create_mock_file_rule_for_gloss_test(id_val=other_fr_id, map_type="NORMAL", filename_pattern="normal_map.png") + + initial_gloss_temp_path = Path("/fake/temp_engine_dir/processed_gloss_map.png") + initial_other_temp_path = Path("/fake/temp_engine_dir/processed_normal_map.png") + + initial_details = { + gloss_fr.id.hex: {'temp_processed_file': str(initial_gloss_temp_path), 'status': 'Processed', 'map_type': 'GLOSS'}, + other_fr.id.hex: {'temp_processed_file': str(initial_other_temp_path), 'status': 'Processed', 'map_type': 'NORMAL'} + } + context = create_gloss_conversion_mock_context( + initial_file_rules=[gloss_fr, other_fr], + initial_processed_details=initial_details + ) + + mock_loaded_gloss_data = np.array([10, 50, 250], dtype=np.uint8) + mock_load_image.return_value = mock_loaded_gloss_data + mock_save_image.return_value = True # Simulate successful save + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(initial_gloss_temp_path) + + # Check that save_image was called with inverted data and correct path + expected_inverted_data = 255 - mock_loaded_gloss_data + + # call_args[0] is a tuple of positional args, call_args[1] is a dict of kwargs + saved_path_arg = mock_save_image.call_args[0][0] + saved_data_arg = mock_save_image.call_args[0][1] + + assert np.array_equal(saved_data_arg, expected_inverted_data), "Image data passed to save_image is not correctly inverted." + assert "rough_from_gloss_" in saved_path_arg.name, "Saved file name should indicate conversion from gloss." + assert saved_path_arg.parent == Path("/fake/temp_engine_dir"), "Saved file should be in the engine temp directory." + # Ensure the new filename is based on the original gloss map's ID for uniqueness + assert gloss_fr.id.hex in saved_path_arg.name + + # Check context.files_to_process + assert len(updated_context.files_to_process) == 2, "Number of file rules in context should remain the same." + converted_rule_found = False + other_rule_untouched = False + for fr_in_list in updated_context.files_to_process: + if fr_in_list.id == gloss_fr.id: # Should be the same rule object, modified + assert fr_in_list.map_type == "ROUGHNESS", "GLOSS map_type should be changed to ROUGHNESS." + # Check if filename_pattern was updated (optional, depends on stage logic) + # For now, assume it might not be, as the primary identifier is map_type and ID + converted_rule_found = True + elif fr_in_list.id == other_fr.id: + assert fr_in_list.map_type == "NORMAL", "Other map_type should remain unchanged." + other_rule_untouched = True + assert converted_rule_found, "The converted GLOSS rule was not found or not updated correctly in files_to_process." + assert other_rule_untouched, "The non-GLOSS rule was modified unexpectedly." + + # Check context.processed_maps_details + assert len(updated_context.processed_maps_details) == 2, "Number of entries in processed_maps_details should remain the same." + + gloss_detail = updated_context.processed_maps_details[gloss_fr.id.hex] + assert "rough_from_gloss_" in gloss_detail['temp_processed_file'], "temp_processed_file for gloss map not updated." + assert Path(gloss_detail['temp_processed_file']).name == saved_path_arg.name, "Path in details should match saved path." + assert gloss_detail['original_map_type_before_conversion'] == "GLOSS", "original_map_type_before_conversion not set correctly." + assert "Converted from GLOSS to ROUGHNESS" in gloss_detail['notes'], "Conversion notes not added or incorrect." + assert gloss_detail['map_type'] == "ROUGHNESS", "map_type in details not updated to ROUGHNESS." + + + other_detail = updated_context.processed_maps_details[other_fr.id.hex] + assert other_detail['temp_processed_file'] == str(initial_other_temp_path), "Other map's temp_processed_file should be unchanged." + assert other_detail['map_type'] == "NORMAL", "Other map's map_type should be unchanged." + assert 'original_map_type_before_conversion' not in other_detail, "Other map should not have conversion history." + assert 'notes' not in other_detail or "Converted from GLOSS" not in other_detail['notes'], "Other map should not have conversion notes." + + mock_logging.info.assert_any_call(f"Successfully converted GLOSS map {gloss_fr.id.hex} to ROUGHNESS.") +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging') # Mock logging +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image') +def test_gloss_conversion_float_success(mock_load_image, mock_save_image, mock_logging): + """ + Test successful conversion of a GLOSS map (float data) to ROUGHNESS. + """ + stage = GlossToRoughConversionStage() + + gloss_rule_id = uuid.uuid4() + gloss_fr = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS", filename_pattern="gloss_float.hdr") # Example float format + + initial_gloss_temp_path = Path("/fake/temp_engine_dir/processed_gloss_float.hdr") + initial_details = { + gloss_fr.id.hex: {'temp_processed_file': str(initial_gloss_temp_path), 'status': 'Processed', 'map_type': 'GLOSS'} + } + context = create_gloss_conversion_mock_context( + initial_file_rules=[gloss_fr], + initial_processed_details=initial_details + ) + + mock_loaded_gloss_data = np.array([0.1, 0.5, 0.9], dtype=np.float32) + mock_load_image.return_value = mock_loaded_gloss_data + mock_save_image.return_value = True # Simulate successful save + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(initial_gloss_temp_path) + + expected_inverted_data = 1.0 - mock_loaded_gloss_data + + saved_path_arg = mock_save_image.call_args[0][0] + saved_data_arg = mock_save_image.call_args[0][1] + + assert np.allclose(saved_data_arg, expected_inverted_data), "Image data (float) passed to save_image is not correctly inverted." + assert "rough_from_gloss_" in saved_path_arg.name, "Saved file name should indicate conversion from gloss." + assert saved_path_arg.parent == Path("/fake/temp_engine_dir"), "Saved file should be in the engine temp directory." + assert gloss_fr.id.hex in saved_path_arg.name + + assert len(updated_context.files_to_process) == 1 + converted_rule = updated_context.files_to_process[0] + assert converted_rule.id == gloss_fr.id + assert converted_rule.map_type == "ROUGHNESS" + + gloss_detail = updated_context.processed_maps_details[gloss_fr.id.hex] + assert "rough_from_gloss_" in gloss_detail['temp_processed_file'] + assert Path(gloss_detail['temp_processed_file']).name == saved_path_arg.name + assert gloss_detail['original_map_type_before_conversion'] == "GLOSS" + assert "Converted from GLOSS to ROUGHNESS" in gloss_detail['notes'] + assert gloss_detail['map_type'] == "ROUGHNESS" + + mock_logging.info.assert_any_call(f"Successfully converted GLOSS map {gloss_fr.id.hex} to ROUGHNESS.") +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image') +def test_load_image_fails(mock_load_image, mock_save_image, mock_logging): + """ + Test behavior when ipu.load_image fails (returns None). + The original FileRule should be kept, and an error logged. + """ + stage = GlossToRoughConversionStage() + + gloss_rule_id = uuid.uuid4() + gloss_fr = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS", filename_pattern="gloss_fails_load.png") + + initial_gloss_temp_path = Path("/fake/temp_engine_dir/processed_gloss_fails_load.png") + initial_details = { + gloss_fr.id.hex: {'temp_processed_file': str(initial_gloss_temp_path), 'status': 'Processed', 'map_type': 'GLOSS'} + } + context = create_gloss_conversion_mock_context( + initial_file_rules=[gloss_fr], + initial_processed_details=initial_details + ) + + # Keep a copy for comparison + original_file_rule_map_type = gloss_fr.map_type + original_details_entry = context.processed_maps_details[gloss_fr.id.hex].copy() + + mock_load_image.return_value = None # Simulate load failure + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(initial_gloss_temp_path) + mock_save_image.assert_not_called() # Save should not be attempted + + # Check context.files_to_process: rule should be unchanged + assert len(updated_context.files_to_process) == 1 + processed_rule = updated_context.files_to_process[0] + assert processed_rule.id == gloss_fr.id + assert processed_rule.map_type == original_file_rule_map_type, "FileRule map_type should not change if load fails." + assert processed_rule.map_type == "GLOSS" # Explicitly check it's still GLOSS + + # Check context.processed_maps_details: details should be unchanged + current_details_entry = updated_context.processed_maps_details[gloss_fr.id.hex] + assert current_details_entry['temp_processed_file'] == str(initial_gloss_temp_path) + assert current_details_entry['map_type'] == "GLOSS" + assert 'original_map_type_before_conversion' not in current_details_entry + assert 'notes' not in current_details_entry or "Converted from GLOSS" not in current_details_entry['notes'] + + mock_logging.error.assert_called_once_with( + f"Failed to load image data for GLOSS map {gloss_fr.id.hex} from {initial_gloss_temp_path}. Skipping conversion for this map." + ) +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image') +def test_save_image_fails(mock_load_image, mock_save_image, mock_logging): + """ + Test behavior when ipu.save_image fails (returns False). + The original FileRule should be kept, and an error logged. + """ + stage = GlossToRoughConversionStage() + + gloss_rule_id = uuid.uuid4() + gloss_fr = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS", filename_pattern="gloss_fails_save.png") + + initial_gloss_temp_path = Path("/fake/temp_engine_dir/processed_gloss_fails_save.png") + initial_details = { + gloss_fr.id.hex: {'temp_processed_file': str(initial_gloss_temp_path), 'status': 'Processed', 'map_type': 'GLOSS'} + } + context = create_gloss_conversion_mock_context( + initial_file_rules=[gloss_fr], + initial_processed_details=initial_details + ) + + original_file_rule_map_type = gloss_fr.map_type + original_details_entry = context.processed_maps_details[gloss_fr.id.hex].copy() + + mock_loaded_gloss_data = np.array([10, 50, 250], dtype=np.uint8) + mock_load_image.return_value = mock_loaded_gloss_data + mock_save_image.return_value = False # Simulate save failure + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(initial_gloss_temp_path) + + # Check that save_image was called with correct data and path + expected_inverted_data = 255 - mock_loaded_gloss_data + # call_args[0] is a tuple of positional args + saved_path_arg = mock_save_image.call_args[0][0] + saved_data_arg = mock_save_image.call_args[0][1] + + assert np.array_equal(saved_data_arg, expected_inverted_data), "Image data passed to save_image is not correctly inverted even on failure." + assert "rough_from_gloss_" in saved_path_arg.name, "Attempted save file name should indicate conversion from gloss." + assert saved_path_arg.parent == Path("/fake/temp_engine_dir"), "Attempted save file should be in the engine temp directory." + + # Check context.files_to_process: rule should be unchanged + assert len(updated_context.files_to_process) == 1 + processed_rule = updated_context.files_to_process[0] + assert processed_rule.id == gloss_fr.id + assert processed_rule.map_type == original_file_rule_map_type, "FileRule map_type should not change if save fails." + assert processed_rule.map_type == "GLOSS" + + # Check context.processed_maps_details: details should be unchanged + current_details_entry = updated_context.processed_maps_details[gloss_fr.id.hex] + assert current_details_entry['temp_processed_file'] == str(initial_gloss_temp_path) + assert current_details_entry['map_type'] == "GLOSS" + assert 'original_map_type_before_conversion' not in current_details_entry + assert 'notes' not in current_details_entry or "Converted from GLOSS" not in current_details_entry['notes'] + + mock_logging.error.assert_called_once_with( + f"Failed to save inverted GLOSS map {gloss_fr.id.hex} to {saved_path_arg}. Retaining original GLOSS map." + ) +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image') +def test_gloss_map_in_files_to_process_but_not_in_details(mock_load_image, mock_save_image, mock_logging): + """ + Test behavior when a GLOSS FileRule is in files_to_process but its details + are missing from processed_maps_details. + The stage should log an error and skip this FileRule. + """ + stage = GlossToRoughConversionStage() + + gloss_rule_id = uuid.uuid4() + # This FileRule is in files_to_process + gloss_fr_in_list = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS", filename_pattern="orphan_gloss.png") + + # processed_maps_details is empty or does not contain gloss_fr_in_list.id.hex + initial_details = {} + + context = create_gloss_conversion_mock_context( + initial_file_rules=[gloss_fr_in_list], + initial_processed_details=initial_details + ) + + original_files_to_process = list(context.files_to_process) + original_processed_maps_details = context.processed_maps_details.copy() + + updated_context = stage.execute(context) + + mock_load_image.assert_not_called() # Load should not be attempted if details are missing + mock_save_image.assert_not_called() # Save should not be attempted + + # Check context.files_to_process: rule should be unchanged + assert len(updated_context.files_to_process) == 1 + processed_rule = updated_context.files_to_process[0] + assert processed_rule.id == gloss_fr_in_list.id + assert processed_rule.map_type == "GLOSS", "FileRule map_type should not change if its details are missing." + + # Check context.processed_maps_details: should remain unchanged + assert updated_context.processed_maps_details == original_processed_maps_details, "processed_maps_details should not change." + + mock_logging.error.assert_called_once_with( + f"GLOSS map {gloss_fr_in_list.id.hex} found in files_to_process but missing from processed_maps_details. Skipping conversion." + ) + +# Test for Case 8.2 (GLOSS map ID in processed_maps_details but no corresponding FileRule in files_to_process) +# This case is implicitly handled because the stage iterates files_to_process. +# If a FileRule isn't in files_to_process, its corresponding entry in processed_maps_details (if any) won't be acted upon. +# We can add a simple test to ensure no errors occur and non-relevant details are untouched. + +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image') +@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image') +def test_gloss_detail_exists_but_not_in_files_to_process(mock_load_image, mock_save_image, mock_logging): + """ + Test that if a GLOSS map detail exists in processed_maps_details but + no corresponding FileRule is in files_to_process, it's simply ignored + without error, and other valid conversions proceed. + """ + stage = GlossToRoughConversionStage() + + # This rule will be processed + convert_rule_id = uuid.uuid4() + convert_fr = create_mock_file_rule_for_gloss_test(id_val=convert_rule_id, map_type="GLOSS", filename_pattern="convert_me.png") + convert_initial_temp_path = Path("/fake/temp_engine_dir/processed_convert_me.png") + + # This rule's details exist, but the rule itself is not in files_to_process + orphan_detail_id = uuid.uuid4() + + initial_details = { + convert_fr.id.hex: {'temp_processed_file': str(convert_initial_temp_path), 'status': 'Processed', 'map_type': 'GLOSS'}, + orphan_detail_id.hex: {'temp_processed_file': '/fake/temp_engine_dir/orphan.png', 'status': 'Processed', 'map_type': 'GLOSS', 'notes': 'This is an orphan'} + } + + context = create_gloss_conversion_mock_context( + initial_file_rules=[convert_fr], # Only convert_fr is in files_to_process + initial_processed_details=initial_details + ) + + mock_loaded_data = np.array([100], dtype=np.uint8) + mock_load_image.return_value = mock_loaded_data + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + # Assert that load/save were called only for the rule in files_to_process + mock_load_image.assert_called_once_with(convert_initial_temp_path) + mock_save_image.assert_called_once() # Check it was called, details checked in other tests + + # Check that the orphan detail in processed_maps_details is untouched + assert orphan_detail_id.hex in updated_context.processed_maps_details + orphan_entry = updated_context.processed_maps_details[orphan_detail_id.hex] + assert orphan_entry['temp_processed_file'] == '/fake/temp_engine_dir/orphan.png' + assert orphan_entry['map_type'] == 'GLOSS' + assert orphan_entry['notes'] == 'This is an orphan' + assert 'original_map_type_before_conversion' not in orphan_entry + + # Check that the processed rule was indeed converted + assert convert_fr.id.hex in updated_context.processed_maps_details + converted_entry = updated_context.processed_maps_details[convert_fr.id.hex] + assert converted_entry['map_type'] == 'ROUGHNESS' + assert "rough_from_gloss_" in converted_entry['temp_processed_file'] + + # No errors should have been logged regarding the orphan detail + for call_args in mock_logging.error.call_args_list: + assert str(orphan_detail_id.hex) not in call_args[0][0], "Error logged for orphan detail" \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_individual_map_processing.py b/tests/processing/pipeline/stages/test_individual_map_processing.py new file mode 100644 index 0000000..8af2d19 --- /dev/null +++ b/tests/processing/pipeline/stages/test_individual_map_processing.py @@ -0,0 +1,555 @@ +import pytest +from unittest import mock +from pathlib import Path +import uuid +import numpy as np +from typing import Optional # Added for type hinting in helper functions + +from processing.pipeline.stages.individual_map_processing import IndividualMapProcessingStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule, FileRule # Key models +from configuration import Configuration, GeneralSettings +# cv2 might be imported by the stage for interpolation constants, ensure it's mockable if so. +# For now, assume ipu handles interpolation details. + +def create_mock_transform_settings( + target_width=0, target_height=0, resize_mode="FIT", + ensure_pot=False, allow_upscale=True, target_color_profile="RGB" # Add other fields as needed +) -> mock.MagicMock: + ts = mock.MagicMock(spec=TransformSettings) + ts.target_width = target_width + ts.target_height = target_height + ts.resize_mode = resize_mode + ts.ensure_pot = ensure_pot + ts.allow_upscale = allow_upscale + ts.target_color_profile = target_color_profile + # ts.resize_filter = "AREA" # if your stage uses this + return ts + +def create_mock_file_rule_for_individual_processing( + id_val: Optional[uuid.UUID] = None, + map_type: str = "ALBEDO", + filename_pattern: str = "albedo_*.png", # Pattern for glob + item_type: str = "MAP_COL", + active: bool = True, + transform_settings: Optional[mock.MagicMock] = None +) -> mock.MagicMock: + mock_fr = mock.MagicMock(spec=FileRule) + mock_fr.id = id_val if id_val else uuid.uuid4() + mock_fr.map_type = map_type + mock_fr.filename_pattern = filename_pattern + mock_fr.item_type = item_type + mock_fr.active = active + mock_fr.transform_settings = transform_settings if transform_settings else create_mock_transform_settings() + return mock_fr + +def create_individual_map_proc_mock_context( + initial_file_rules: Optional[list] = None, + asset_source_path_str: str = "/fake/asset_source", + skip_asset_flag: bool = False, + asset_name: str = "IndividualMapAsset" +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + mock_asset_rule.source_path = Path(asset_source_path_str) + # file_rules on AssetRule not directly used by stage, context.files_to_process is + + mock_source_rule = mock.MagicMock(spec=SourceRule) + mock_config = mock.MagicMock(spec=Configuration) + # mock_config.general_settings = mock.MagicMock(spec=GeneralSettings) # If needed + + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp_engine_dir"), + output_base_path=Path("/fake/output"), + effective_supplier="ValidSupplier", + asset_metadata={'asset_name': asset_name}, + processed_maps_details={}, # Stage populates this + merged_maps_details={}, + files_to_process=list(initial_file_rules) if initial_file_rules else [], + loaded_data_cache={}, + config_obj=mock_config, + status_flags={'skip_asset': skip_asset_flag}, + incrementing_value=None, + sha5_value=None # Corrected from sha5_value to sha_value if that's the actual param + ) + return context + +# Placeholder for tests to be added next +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu') +@mock.patch('logging.info') +def test_asset_skipped_if_flag_is_true(mock_log_info, mock_ipu): + stage = IndividualMapProcessingStage() + context = create_individual_map_proc_mock_context(skip_asset_flag=True) + + # Add a dummy file rule to ensure it's not processed + file_rule = create_mock_file_rule_for_individual_processing() + context.files_to_process = [file_rule] + + updated_context = stage.execute(context) + + mock_ipu.load_image.assert_not_called() + mock_ipu.save_image.assert_not_called() + assert not updated_context.processed_maps_details # No details should be added + # Check for a log message indicating skip, if applicable (depends on stage's logging) + # mock_log_info.assert_any_call("Skipping asset IndividualMapAsset due to status_flags['skip_asset'] = True") # Example + + +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu') +@mock.patch('logging.info') +def test_no_processing_if_no_map_col_rules(mock_log_info, mock_ipu): + stage = IndividualMapProcessingStage() + + # Create a file rule that is NOT of item_type MAP_COL + non_map_col_rule = create_mock_file_rule_for_individual_processing(item_type="METADATA") + context = create_individual_map_proc_mock_context(initial_file_rules=[non_map_col_rule]) + + updated_context = stage.execute(context) + + mock_ipu.load_image.assert_not_called() + mock_ipu.save_image.assert_not_called() + assert not updated_context.processed_maps_details + # mock_log_info.assert_any_call("No FileRules of item_type 'MAP_COL' to process for asset IndividualMapAsset.") # Example + + +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image') +@mock.patch('pathlib.Path.glob') # Mocking Path.glob used by the stage's _find_source_file +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_individual_map_processing_success_no_resize( + mock_log_error, mock_log_info, mock_path_glob, mock_load_image, + mock_calc_dims, mock_resize_image, mock_save_image +): + stage = IndividualMapProcessingStage() + + source_file_name = "albedo_source.png" + # The glob is called on context.asset_rule.source_path, so mock that Path object's glob + mock_asset_source_path = Path("/fake/asset_source") + mock_found_source_path = mock_asset_source_path / source_file_name + + # We need to mock the glob method of the Path instance + # that represents the asset's source directory. + # The stage does something like: Path(context.asset_rule.source_path).glob(...) + # So, we need to ensure that when Path() is called with that specific string, + # the resulting object's glob method is our mock. + # A more robust way is to mock Path itself to return a mock object + # whose glob method is also a mock. + + # Simpler approach for now: assume Path.glob is used as a static/class method call + # or that the instance it's called on is correctly patched by @mock.patch('pathlib.Path.glob') + # if the stage does `from pathlib import Path` and then `Path(path_str).glob(...)`. + # The prompt example uses @mock.patch('pathlib.Path.glob'), implying the stage might do this: + # for f_pattern in patterns: + # for found_file in Path(base_dir).glob(f_pattern): ... + # Let's refine the mock_path_glob setup. + # The stage's _find_source_file likely does: + # search_path = Path(self.context.asset_rule.source_path) + # found_files = list(search_path.glob(filename_pattern)) + + # To correctly mock this, we need to mock the `glob` method of the specific Path instance. + # Or, if `_find_source_file` instantiates `Path` like `Path(str(context.asset_rule.source_path)).glob(...)`, + # then patching `pathlib.Path.glob` might work if it's treated as a method that gets bound. + # Let's stick to the example's @mock.patch('pathlib.Path.glob') and assume it covers the usage. + mock_path_glob.return_value = [mock_found_source_path] # Glob finds one file + + ts = create_mock_transform_settings(target_width=100, target_height=100) + file_rule = create_mock_file_rule_for_individual_processing( + map_type="ALBEDO", filename_pattern="albedo_*.png", transform_settings=ts + ) + context = create_individual_map_proc_mock_context( + initial_file_rules=[file_rule], + asset_source_path_str=str(mock_asset_source_path) # Ensure context uses this path + ) + + mock_img_data = np.zeros((100, 100, 3), dtype=np.uint8) # Original dimensions + mock_load_image.return_value = mock_img_data + mock_calc_dims.return_value = (100, 100) # No resize needed + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + # Assert that Path(context.asset_rule.source_path).glob was called + # This requires a bit more intricate mocking if Path instances are created inside. + # For now, assert mock_path_glob was called with the pattern. + # The actual call in stage is `Path(context.asset_rule.source_path).glob(file_rule.filename_pattern)` + # So, `mock_path_glob` (if it patches `Path.glob` globally) should be called. + # We need to ensure the mock_path_glob is associated with the correct Path instance or that + # the global patch works as intended. + # A common pattern is: + # with mock.patch.object(Path, 'glob', return_value=[mock_found_source_path]) as specific_glob_mock: + # # execute code + # specific_glob_mock.assert_called_once_with(file_rule.filename_pattern) + # However, the decorator @mock.patch('pathlib.Path.glob') should work if the stage code is + # `from pathlib import Path; p = Path(...); p.glob(...)` + + # The stage's _find_source_file will instantiate a Path object from context.asset_rule.source_path + # and then call glob on it. + # So, @mock.patch('pathlib.Path.glob') is patching the method on the class. + # When an instance calls it, the mock is used. + mock_path_glob.assert_called_once_with(file_rule.filename_pattern) + + + mock_load_image.assert_called_once_with(mock_found_source_path) + # The actual call to calculate_target_dimensions is: + # ipu.calculate_target_dimensions(original_dims, ts.target_width, ts.target_height, ts.resize_mode, ts.ensure_pot, ts.allow_upscale) + mock_calc_dims.assert_called_once_with( + (100, 100), ts.target_width, ts.target_height, ts.resize_mode, ts.ensure_pot, ts.allow_upscale + ) + mock_resize_image.assert_not_called() # Crucial for this test case + mock_save_image.assert_called_once() + + # Check save path and data + saved_image_arg, saved_path_arg = mock_save_image.call_args[0] + assert np.array_equal(saved_image_arg, mock_img_data) # Ensure correct image data is passed to save + assert "processed_ALBEDO_" in saved_path_arg.name # Based on map_type + assert file_rule.id.hex in saved_path_arg.name # Ensure unique name with FileRule ID + assert saved_path_arg.parent == context.engine_temp_dir + + assert file_rule.id.hex in updated_context.processed_maps_details + details = updated_context.processed_maps_details[file_rule.id.hex] + assert details['status'] == 'Processed' + assert details['source_file'] == str(mock_found_source_path) + assert Path(details['temp_processed_file']) == saved_path_arg + assert details['original_dimensions'] == (100, 100) + assert details['processed_dimensions'] == (100, 100) + assert details['map_type'] == file_rule.map_type + mock_log_error.assert_not_called() + mock_log_info.assert_any_call(f"Successfully processed map {file_rule.map_type} (ID: {file_rule.id.hex}) for asset {context.asset_rule.name}. Output: {saved_path_arg}") + + +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image') +@mock.patch('pathlib.Path.glob') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_source_file_not_found( + mock_log_error, mock_log_info, mock_path_glob, mock_load_image, + mock_calc_dims, mock_resize_image, mock_save_image +): + stage = IndividualMapProcessingStage() + mock_asset_source_path = Path("/fake/asset_source") + + mock_path_glob.return_value = [] # Glob finds no files + + file_rule = create_mock_file_rule_for_individual_processing(filename_pattern="nonexistent_*.png") + context = create_individual_map_proc_mock_context( + initial_file_rules=[file_rule], + asset_source_path_str=str(mock_asset_source_path) + ) + + updated_context = stage.execute(context) + + mock_path_glob.assert_called_once_with(file_rule.filename_pattern) + mock_load_image.assert_not_called() + mock_calc_dims.assert_not_called() + mock_resize_image.assert_not_called() + mock_save_image.assert_not_called() + + assert file_rule.id.hex in updated_context.processed_maps_details + details = updated_context.processed_maps_details[file_rule.id.hex] + assert details['status'] == 'Source Not Found' + assert details['source_file'] is None + assert details['temp_processed_file'] is None + assert details['error_message'] is not None # Check an error message is present + mock_log_error.assert_called_once() + # Example: mock_log_error.assert_called_with(f"Could not find source file for rule {file_rule.id} (pattern: {file_rule.filename_pattern}) in {context.asset_rule.source_path}") + + +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image') +@mock.patch('pathlib.Path.glob') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_load_image_fails( + mock_log_error, mock_log_info, mock_path_glob, mock_load_image, + mock_calc_dims, mock_resize_image, mock_save_image +): + stage = IndividualMapProcessingStage() + source_file_name = "albedo_corrupt.png" + mock_asset_source_path = Path("/fake/asset_source") + mock_found_source_path = mock_asset_source_path / source_file_name + mock_path_glob.return_value = [mock_found_source_path] + + mock_load_image.return_value = None # Simulate load failure + + file_rule = create_mock_file_rule_for_individual_processing(filename_pattern="albedo_*.png") + context = create_individual_map_proc_mock_context( + initial_file_rules=[file_rule], + asset_source_path_str=str(mock_asset_source_path) + ) + + updated_context = stage.execute(context) + + mock_path_glob.assert_called_once_with(file_rule.filename_pattern) + mock_load_image.assert_called_once_with(mock_found_source_path) + mock_calc_dims.assert_not_called() + mock_resize_image.assert_not_called() + mock_save_image.assert_not_called() + + assert file_rule.id.hex in updated_context.processed_maps_details + details = updated_context.processed_maps_details[file_rule.id.hex] + assert details['status'] == 'Load Failed' + assert details['source_file'] == str(mock_found_source_path) + assert details['temp_processed_file'] is None + assert details['error_message'] is not None + mock_log_error.assert_called_once() + # Example: mock_log_error.assert_called_with(f"Failed to load image {mock_found_source_path} for rule {file_rule.id}") + + +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image') +@mock.patch('pathlib.Path.glob') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_resize_occurs_when_dimensions_differ( + mock_log_error, mock_log_info, mock_path_glob, mock_load_image, + mock_calc_dims, mock_resize_image, mock_save_image +): + stage = IndividualMapProcessingStage() + source_file_name = "albedo_resize.png" + mock_asset_source_path = Path("/fake/asset_source") + mock_found_source_path = mock_asset_source_path / source_file_name + mock_path_glob.return_value = [mock_found_source_path] + + original_dims = (100, 100) + target_dims = (50, 50) # Different dimensions + mock_img_data = np.zeros((*original_dims, 3), dtype=np.uint8) + mock_resized_img_data = np.zeros((*target_dims, 3), dtype=np.uint8) + + mock_load_image.return_value = mock_img_data + ts = create_mock_transform_settings(target_width=target_dims[0], target_height=target_dims[1]) + file_rule = create_mock_file_rule_for_individual_processing(transform_settings=ts) + context = create_individual_map_proc_mock_context( + initial_file_rules=[file_rule], + asset_source_path_str=str(mock_asset_source_path) + ) + + mock_calc_dims.return_value = target_dims # Simulate calc_dims returning new dimensions + mock_resize_image.return_value = mock_resized_img_data # Simulate resize returning new image data + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(mock_found_source_path) + mock_calc_dims.assert_called_once_with( + original_dims, ts.target_width, ts.target_height, ts.resize_mode, ts.ensure_pot, ts.allow_upscale + ) + # The actual call to resize_image is: + # ipu.resize_image(loaded_image, target_dims, ts.resize_filter) # Assuming resize_filter is used + # If resize_filter is not on TransformSettings or not used, adjust this. + # For now, let's assume it's ipu.resize_image(loaded_image, target_dims) or similar + # The stage code is: resized_image = ipu.resize_image(loaded_image, target_dims_calculated, file_rule.transform_settings.resize_filter) + # So we need to mock ts.resize_filter + ts.resize_filter = "LANCZOS4" # Example filter + mock_resize_image.assert_called_once_with(mock_img_data, target_dims, ts.resize_filter) + + saved_image_arg, saved_path_arg = mock_save_image.call_args[0] + assert np.array_equal(saved_image_arg, mock_resized_img_data) # Check resized data is saved + assert "processed_ALBEDO_" in saved_path_arg.name + assert saved_path_arg.parent == context.engine_temp_dir + + assert file_rule.id.hex in updated_context.processed_maps_details + details = updated_context.processed_maps_details[file_rule.id.hex] + assert details['status'] == 'Processed' + assert details['original_dimensions'] == original_dims + assert details['processed_dimensions'] == target_dims + mock_log_error.assert_not_called() + + +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image') +@mock.patch('pathlib.Path.glob') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_save_image_fails( + mock_log_error, mock_log_info, mock_path_glob, mock_load_image, + mock_calc_dims, mock_resize_image, mock_save_image +): + stage = IndividualMapProcessingStage() + source_file_name = "albedo_save_fail.png" + mock_asset_source_path = Path("/fake/asset_source") + mock_found_source_path = mock_asset_source_path / source_file_name + mock_path_glob.return_value = [mock_found_source_path] + + mock_img_data = np.zeros((100, 100, 3), dtype=np.uint8) + mock_load_image.return_value = mock_img_data + mock_calc_dims.return_value = (100, 100) # No resize + mock_save_image.return_value = False # Simulate save failure + + ts = create_mock_transform_settings() + file_rule = create_mock_file_rule_for_individual_processing(transform_settings=ts) + context = create_individual_map_proc_mock_context( + initial_file_rules=[file_rule], + asset_source_path_str=str(mock_asset_source_path) + ) + + updated_context = stage.execute(context) + + mock_save_image.assert_called_once() # Attempt to save should still happen + + assert file_rule.id.hex in updated_context.processed_maps_details + details = updated_context.processed_maps_details[file_rule.id.hex] + assert details['status'] == 'Save Failed' + assert details['source_file'] == str(mock_found_source_path) + assert details['temp_processed_file'] is not None # Path was generated + assert details['error_message'] is not None + mock_log_error.assert_called_once() + # Example: mock_log_error.assert_called_with(f"Failed to save processed image for rule {file_rule.id} to {details['temp_processed_file']}") + + +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.convert_bgr_to_rgb') +@mock.patch('pathlib.Path.glob') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_color_conversion_bgr_to_rgb( + mock_log_error, mock_log_info, mock_path_glob, mock_convert_bgr, mock_load_image, + mock_calc_dims, mock_resize_image, mock_save_image +): + stage = IndividualMapProcessingStage() + source_file_name = "albedo_bgr.png" + mock_asset_source_path = Path("/fake/asset_source") + mock_found_source_path = mock_asset_source_path / source_file_name + mock_path_glob.return_value = [mock_found_source_path] + + mock_bgr_img_data = np.zeros((100, 100, 3), dtype=np.uint8) # Loaded as BGR + mock_rgb_img_data = np.zeros((100, 100, 3), dtype=np.uint8) # After conversion + + mock_load_image.return_value = mock_bgr_img_data # Image is loaded (assume BGR by default from cv2) + mock_convert_bgr.return_value = mock_rgb_img_data # Mock the conversion + mock_calc_dims.return_value = (100, 100) # No resize + mock_save_image.return_value = True + + # Transform settings request RGB, and stage assumes load might be BGR + ts = create_mock_transform_settings(target_color_profile="RGB") + file_rule = create_mock_file_rule_for_individual_processing(transform_settings=ts) + context = create_individual_map_proc_mock_context( + initial_file_rules=[file_rule], + asset_source_path_str=str(mock_asset_source_path) + ) + # The stage code is: + # if file_rule.transform_settings.target_color_profile == "RGB" and loaded_image.shape[2] == 3: + # logger.info(f"Attempting to convert image from BGR to RGB for {file_rule_id_hex}") + # processed_image_data = ipu.convert_bgr_to_rgb(processed_image_data) + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(mock_found_source_path) + mock_convert_bgr.assert_called_once_with(mock_bgr_img_data) + mock_resize_image.assert_not_called() + + saved_image_arg, _ = mock_save_image.call_args[0] + assert np.array_equal(saved_image_arg, mock_rgb_img_data) # Ensure RGB data is saved + mock_log_error.assert_not_called() + mock_log_info.assert_any_call(f"Attempting to convert image from BGR to RGB for {file_rule.id.hex}") + + +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions') +@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image') +@mock.patch('pathlib.Path.glob') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_multiple_map_col_rules_processed( + mock_log_error, mock_log_info, mock_path_glob, mock_load_image, + mock_calc_dims, mock_resize_image, mock_save_image +): + stage = IndividualMapProcessingStage() + mock_asset_source_path = Path("/fake/asset_source") + + # Rule 1: Albedo + ts1 = create_mock_transform_settings(target_width=100, target_height=100) + file_rule1_id = uuid.uuid4() + file_rule1 = create_mock_file_rule_for_individual_processing( + id_val=file_rule1_id, map_type="ALBEDO", filename_pattern="albedo_*.png", transform_settings=ts1 + ) + source_file1 = mock_asset_source_path / "albedo_map.png" + img_data1 = np.zeros((100, 100, 3), dtype=np.uint8) + + # Rule 2: Roughness + ts2 = create_mock_transform_settings(target_width=50, target_height=50) # Resize + ts2.resize_filter = "AREA" + file_rule2_id = uuid.uuid4() + file_rule2 = create_mock_file_rule_for_individual_processing( + id_val=file_rule2_id, map_type="ROUGHNESS", filename_pattern="rough_*.png", transform_settings=ts2 + ) + source_file2 = mock_asset_source_path / "rough_map.png" + img_data2_orig = np.zeros((200, 200, 1), dtype=np.uint8) # Original, needs resize + img_data2_resized = np.zeros((50, 50, 1), dtype=np.uint8) # Resized + + context = create_individual_map_proc_mock_context( + initial_file_rules=[file_rule1, file_rule2], + asset_source_path_str=str(mock_asset_source_path) + ) + + # Mock behaviors for Path.glob, load_image, calc_dims, resize, save + # Path.glob will be called twice + mock_path_glob.side_effect = [ + [source_file1], # For albedo_*.png + [source_file2] # For rough_*.png + ] + mock_load_image.side_effect = [img_data1, img_data2_orig] + mock_calc_dims.side_effect = [ + (100, 100), # For rule1 (no change) + (50, 50) # For rule2 (change) + ] + mock_resize_image.return_value = img_data2_resized # Only called for rule2 + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + # Assertions for Rule 1 (Albedo) + assert mock_path_glob.call_args_list[0][0][0] == file_rule1.filename_pattern + assert mock_load_image.call_args_list[0][0][0] == source_file1 + assert mock_calc_dims.call_args_list[0][0] == ((100,100), ts1.target_width, ts1.target_height, ts1.resize_mode, ts1.ensure_pot, ts1.allow_upscale) + + # Assertions for Rule 2 (Roughness) + assert mock_path_glob.call_args_list[1][0][0] == file_rule2.filename_pattern + assert mock_load_image.call_args_list[1][0][0] == source_file2 + assert mock_calc_dims.call_args_list[1][0] == ((200,200), ts2.target_width, ts2.target_height, ts2.resize_mode, ts2.ensure_pot, ts2.allow_upscale) + mock_resize_image.assert_called_once_with(img_data2_orig, (50,50), ts2.resize_filter) + + assert mock_save_image.call_count == 2 + # Check saved image for rule 1 + saved_img1_arg, saved_path1_arg = mock_save_image.call_args_list[0][0] + assert np.array_equal(saved_img1_arg, img_data1) + assert "processed_ALBEDO_" in saved_path1_arg.name + assert file_rule1_id.hex in saved_path1_arg.name + + # Check saved image for rule 2 + saved_img2_arg, saved_path2_arg = mock_save_image.call_args_list[1][0] + assert np.array_equal(saved_img2_arg, img_data2_resized) + assert "processed_ROUGHNESS_" in saved_path2_arg.name + assert file_rule2_id.hex in saved_path2_arg.name + + # Check context details + assert file_rule1_id.hex in updated_context.processed_maps_details + details1 = updated_context.processed_maps_details[file_rule1_id.hex] + assert details1['status'] == 'Processed' + assert details1['original_dimensions'] == (100, 100) + assert details1['processed_dimensions'] == (100, 100) + + assert file_rule2_id.hex in updated_context.processed_maps_details + details2 = updated_context.processed_maps_details[file_rule2_id.hex] + assert details2['status'] == 'Processed' + assert details2['original_dimensions'] == (200, 200) # Original dims of img_data2_orig + assert details2['processed_dimensions'] == (50, 50) + + mock_log_error.assert_not_called() \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_map_merging.py b/tests/processing/pipeline/stages/test_map_merging.py new file mode 100644 index 0000000..f7c0c56 --- /dev/null +++ b/tests/processing/pipeline/stages/test_map_merging.py @@ -0,0 +1,538 @@ +import pytest +from unittest import mock +from pathlib import Path +import uuid +import numpy as np +from typing import Optional # Added Optional for type hinting + +from processing.pipeline.stages.map_merging import MapMergingStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule, FileRule +from configuration import Configuration + +# Mock Helper Functions +def create_mock_merge_input_channel( + file_rule_id: uuid.UUID, source_channel: int = 0, target_channel: int = 0, invert: bool = False +) -> mock.MagicMock: + mic = mock.MagicMock(spec=MergeInputChannel) + mic.file_rule_id = file_rule_id + mic.source_channel = source_channel + mic.target_channel = target_channel + mic.invert_source_channel = invert + mic.default_value_if_missing = 0 # Or some other default + return mic + +def create_mock_merge_settings( + input_maps: Optional[list] = None, # List of mock MergeInputChannel + output_channels: int = 3 +) -> mock.MagicMock: + ms = mock.MagicMock(spec=MergeSettings) + ms.input_maps = input_maps if input_maps is not None else [] + ms.output_channels = output_channels + return ms + +def create_mock_file_rule_for_merging( + id_val: Optional[uuid.UUID] = None, + map_type: str = "ORM", # Output map type + item_type: str = "MAP_MERGE", + merge_settings: Optional[mock.MagicMock] = None +) -> mock.MagicMock: + mock_fr = mock.MagicMock(spec=FileRule) + mock_fr.id = id_val if id_val else uuid.uuid4() + mock_fr.map_type = map_type + mock_fr.filename_pattern = f"{map_type.lower()}_merged.png" # Placeholder + mock_fr.item_type = item_type + mock_fr.active = True + mock_fr.merge_settings = merge_settings if merge_settings else create_mock_merge_settings() + return mock_fr + +def create_map_merging_mock_context( + initial_file_rules: Optional[list] = None, # Will contain the MAP_MERGE rule + initial_processed_details: Optional[dict] = None, # Pre-processed inputs for merge + skip_asset_flag: bool = False, + asset_name: str = "MergeAsset" +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + mock_source_rule = mock.MagicMock(spec=SourceRule) + mock_config = mock.MagicMock(spec=Configuration) + + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp_engine_dir"), + output_base_path=Path("/fake/output"), + effective_supplier="ValidSupplier", + asset_metadata={'asset_name': asset_name}, + processed_maps_details=initial_processed_details if initial_processed_details is not None else {}, + merged_maps_details={}, # Stage populates this + files_to_process=list(initial_file_rules) if initial_file_rules else [], + loaded_data_cache={}, + config_obj=mock_config, + status_flags={'skip_asset': skip_asset_flag}, + incrementing_value=None, + sha5_value=None # Corrected from sha5_value to sha_value based on AssetProcessingContext + ) + return context +def test_asset_skipped(): + stage = MapMergingStage() + context = create_map_merging_mock_context(skip_asset_flag=True) + + updated_context = stage.execute(context) + + assert updated_context == context # No changes expected + assert not updated_context.merged_maps_details # No maps should be merged + +def test_no_map_merge_rules(): + stage = MapMergingStage() + # Context with a non-MAP_MERGE rule + non_merge_rule = create_mock_file_rule_for_merging(item_type="TEXTURE_MAP", map_type="Diffuse") + context = create_map_merging_mock_context(initial_file_rules=[non_merge_rule]) + + updated_context = stage.execute(context) + + assert updated_context == context # No changes expected + assert not updated_context.merged_maps_details # No maps should be merged + +@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.resize_image') # If testing resize +@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_map_merging_rgb_success(mock_log_error, mock_log_info, mock_load_image, mock_resize_image, mock_save_image): + stage = MapMergingStage() + + # Input FileRules (mocked as already processed) + r_id, g_id, b_id = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + processed_details = { + r_id.hex: {'temp_processed_file': '/fake/red.png', 'status': 'Processed', 'map_type': 'RED_SRC'}, + g_id.hex: {'temp_processed_file': '/fake/green.png', 'status': 'Processed', 'map_type': 'GREEN_SRC'}, + b_id.hex: {'temp_processed_file': '/fake/blue.png', 'status': 'Processed', 'map_type': 'BLUE_SRC'} + } + # Mock loaded image data (grayscale for inputs) + mock_r_data = np.full((10, 10), 200, dtype=np.uint8) + mock_g_data = np.full((10, 10), 100, dtype=np.uint8) + mock_b_data = np.full((10, 10), 50, dtype=np.uint8) + mock_load_image.side_effect = [mock_r_data, mock_g_data, mock_b_data] + + # Merge Rule setup + merge_inputs = [ + create_mock_merge_input_channel(file_rule_id=r_id, source_channel=0, target_channel=0), # R to R + create_mock_merge_input_channel(file_rule_id=g_id, source_channel=0, target_channel=1), # G to G + create_mock_merge_input_channel(file_rule_id=b_id, source_channel=0, target_channel=2) # B to B + ] + merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=3) + merge_rule_id = uuid.uuid4() + merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="RGB_Combined", merge_settings=merge_settings) + + context = create_map_merging_mock_context( + initial_file_rules=[merge_rule], + initial_processed_details=processed_details + ) + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + assert mock_load_image.call_count == 3 + mock_resize_image.assert_not_called() # Assuming all inputs are same size for this test + mock_save_image.assert_called_once() + + # Check that the correct filename was passed to save_image + # The filename is constructed as: f"{context.asset_rule.name}_merged_{merge_rule.map_type}{Path(first_input_path).suffix}" + # In this case, first_input_path is '/fake/red.png', so suffix is '.png' + # Asset name is "MergeAsset" + expected_filename_part = f"{context.asset_rule.name}_merged_{merge_rule.map_type}.png" + saved_path_arg = mock_save_image.call_args[0][0] + assert expected_filename_part in str(saved_path_arg) + + + saved_data = mock_save_image.call_args[0][1] + assert saved_data.shape == (10, 10, 3) + assert np.all(saved_data[:,:,0] == 200) # Red channel + assert np.all(saved_data[:,:,1] == 100) # Green channel + assert np.all(saved_data[:,:,2] == 50) # Blue channel + + assert merge_rule.id.hex in updated_context.merged_maps_details + details = updated_context.merged_maps_details[merge_rule.id.hex] + assert details['status'] == 'Processed' + # The temp_merged_file path will be under engine_temp_dir / asset_name / filename + assert f"{context.engine_temp_dir / context.asset_rule.name / expected_filename_part}" == details['temp_merged_file'] + mock_log_error.assert_not_called() + mock_log_info.assert_any_call(f"Successfully merged map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'.") + +# Unit tests will be added below this line +@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.resize_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_map_merging_channel_inversion(mock_log_error, mock_log_info, mock_load_image, mock_resize_image, mock_save_image): + stage = MapMergingStage() + + # Input FileRule + input_id = uuid.uuid4() + processed_details = { + input_id.hex: {'temp_processed_file': '/fake/source.png', 'status': 'Processed', 'map_type': 'SOURCE_MAP'} + } + # Mock loaded image data (single channel for simplicity, to be inverted) + mock_source_data = np.array([[0, 100], [155, 255]], dtype=np.uint8) + mock_load_image.return_value = mock_source_data + + # Merge Rule setup: one input, inverted, to one output channel + merge_inputs = [ + create_mock_merge_input_channel(file_rule_id=input_id, source_channel=0, target_channel=0, invert=True) + ] + merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1) + merge_rule_id = uuid.uuid4() + merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="Inverted_Gray", merge_settings=merge_settings) + + context = create_map_merging_mock_context( + initial_file_rules=[merge_rule], + initial_processed_details=processed_details + ) + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(Path('/fake/source.png')) + mock_resize_image.assert_not_called() + mock_save_image.assert_called_once() + + saved_data = mock_save_image.call_args[0][1] + assert saved_data.shape == (2, 2) # Grayscale output + + # Expected inverted data: 255-original + expected_inverted_data = np.array([[255, 155], [100, 0]], dtype=np.uint8) + assert np.all(saved_data == expected_inverted_data) + + assert merge_rule.id.hex in updated_context.merged_maps_details + details = updated_context.merged_maps_details[merge_rule.id.hex] + assert details['status'] == 'Processed' + assert "merged_Inverted_Gray" in details['temp_merged_file'] + mock_log_error.assert_not_called() + mock_log_info.assert_any_call(f"Successfully merged map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'.") +@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image') +@mock.patch('logging.error') +def test_map_merging_input_map_missing(mock_log_error, mock_load_image, mock_save_image): + stage = MapMergingStage() + + # Input FileRule ID that will be missing from processed_details + missing_input_id = uuid.uuid4() + + # Merge Rule setup + merge_inputs = [ + create_mock_merge_input_channel(file_rule_id=missing_input_id, source_channel=0, target_channel=0) + ] + merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1) + merge_rule_id = uuid.uuid4() + merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="TestMissing", merge_settings=merge_settings) + + # processed_details is empty, so missing_input_id will not be found + context = create_map_merging_mock_context( + initial_file_rules=[merge_rule], + initial_processed_details={} + ) + + updated_context = stage.execute(context) + + mock_load_image.assert_not_called() + mock_save_image.assert_not_called() + + assert merge_rule.id.hex in updated_context.merged_maps_details + details = updated_context.merged_maps_details[merge_rule.id.hex] + assert details['status'] == 'Failed' + assert 'error_message' in details + assert f"Input map FileRule ID {missing_input_id.hex} not found in processed_maps_details or not successfully processed" in details['error_message'] + + mock_log_error.assert_called_once() + assert f"Failed to merge map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'" in mock_log_error.call_args[0][0] + assert f"Input map FileRule ID {missing_input_id.hex} not found in processed_maps_details or not successfully processed" in mock_log_error.call_args[0][0] + +@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image') +@mock.patch('logging.error') +def test_map_merging_input_map_status_not_processed(mock_log_error, mock_load_image, mock_save_image): + stage = MapMergingStage() + + input_id = uuid.uuid4() + processed_details = { + # Status is 'Failed', not 'Processed' + input_id.hex: {'temp_processed_file': '/fake/source.png', 'status': 'Failed', 'map_type': 'SOURCE_MAP'} + } + + merge_inputs = [ + create_mock_merge_input_channel(file_rule_id=input_id, source_channel=0, target_channel=0) + ] + merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1) + merge_rule_id = uuid.uuid4() + merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="TestNotProcessed", merge_settings=merge_settings) + + context = create_map_merging_mock_context( + initial_file_rules=[merge_rule], + initial_processed_details=processed_details + ) + + updated_context = stage.execute(context) + + mock_load_image.assert_not_called() + mock_save_image.assert_not_called() + + assert merge_rule.id.hex in updated_context.merged_maps_details + details = updated_context.merged_maps_details[merge_rule.id.hex] + assert details['status'] == 'Failed' + assert 'error_message' in details + assert f"Input map FileRule ID {input_id.hex} not found in processed_maps_details or not successfully processed" in details['error_message'] + + mock_log_error.assert_called_once() + assert f"Failed to merge map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'" in mock_log_error.call_args[0][0] + assert f"Input map FileRule ID {input_id.hex} not found in processed_maps_details or not successfully processed" in mock_log_error.call_args[0][0] +@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image') +@mock.patch('logging.error') +def test_map_merging_load_image_fails(mock_log_error, mock_load_image, mock_save_image): + stage = MapMergingStage() + + input_id = uuid.uuid4() + processed_details = { + input_id.hex: {'temp_processed_file': '/fake/source.png', 'status': 'Processed', 'map_type': 'SOURCE_MAP'} + } + + # Configure mock_load_image to raise an exception + mock_load_image.side_effect = Exception("Failed to load image") + + merge_inputs = [ + create_mock_merge_input_channel(file_rule_id=input_id, source_channel=0, target_channel=0) + ] + merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1) + merge_rule_id = uuid.uuid4() + merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="TestLoadFail", merge_settings=merge_settings) + + context = create_map_merging_mock_context( + initial_file_rules=[merge_rule], + initial_processed_details=processed_details + ) + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(Path('/fake/source.png')) + mock_save_image.assert_not_called() + + assert merge_rule.id.hex in updated_context.merged_maps_details + details = updated_context.merged_maps_details[merge_rule.id.hex] + assert details['status'] == 'Failed' + assert 'error_message' in details + assert "Failed to load image for merge input" in details['error_message'] + assert str(Path('/fake/source.png')) in details['error_message'] + + mock_log_error.assert_called_once() + assert f"Failed to merge map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'" in mock_log_error.call_args[0][0] + assert "Failed to load image for merge input" in mock_log_error.call_args[0][0] +@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image') +@mock.patch('logging.error') +def test_map_merging_save_image_fails(mock_log_error, mock_load_image, mock_save_image): + stage = MapMergingStage() + + input_id = uuid.uuid4() + processed_details = { + input_id.hex: {'temp_processed_file': '/fake/source.png', 'status': 'Processed', 'map_type': 'SOURCE_MAP'} + } + mock_source_data = np.full((10, 10), 128, dtype=np.uint8) + mock_load_image.return_value = mock_source_data + + # Configure mock_save_image to return False (indicating failure) + mock_save_image.return_value = False + + merge_inputs = [ + create_mock_merge_input_channel(file_rule_id=input_id, source_channel=0, target_channel=0) + ] + merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1) + merge_rule_id = uuid.uuid4() + merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="TestSaveFail", merge_settings=merge_settings) + + context = create_map_merging_mock_context( + initial_file_rules=[merge_rule], + initial_processed_details=processed_details + ) + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(Path('/fake/source.png')) + mock_save_image.assert_called_once() # save_image is called, but returns False + + assert merge_rule.id.hex in updated_context.merged_maps_details + details = updated_context.merged_maps_details[merge_rule.id.hex] + assert details['status'] == 'Failed' + assert 'error_message' in details + assert "Failed to save merged map" in details['error_message'] + + mock_log_error.assert_called_once() + assert f"Failed to merge map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'" in mock_log_error.call_args[0][0] + assert "Failed to save merged map" in mock_log_error.call_args[0][0] +@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.resize_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_map_merging_dimension_mismatch_handling(mock_log_error, mock_log_info, mock_load_image, mock_resize_image, mock_save_image): + stage = MapMergingStage() + + # Input FileRules + id1, id2 = uuid.uuid4(), uuid.uuid4() + processed_details = { + id1.hex: {'temp_processed_file': '/fake/img1.png', 'status': 'Processed', 'map_type': 'IMG1_SRC'}, + id2.hex: {'temp_processed_file': '/fake/img2.png', 'status': 'Processed', 'map_type': 'IMG2_SRC'} + } + + # Mock loaded image data with different dimensions + mock_img1_data = np.full((10, 10), 100, dtype=np.uint8) # 10x10 + mock_img2_data_original = np.full((5, 5), 200, dtype=np.uint8) # 5x5, will be resized + + mock_load_image.side_effect = [mock_img1_data, mock_img2_data_original] + + # Mock resize_image to return an image of the target dimensions + # For simplicity, it just creates a new array of the target size filled with a value. + mock_img2_data_resized = np.full((10, 10), 210, dtype=np.uint8) # Resized to 10x10 + mock_resize_image.return_value = mock_img2_data_resized + + # Merge Rule setup: two inputs, one output channel (e.g., averaging them) + # Target channel 0 for both, the stage should handle combining them if they map to the same target. + # However, the current stage logic for multiple inputs to the same target channel is to take the last one. + # Let's make them target different channels for a clearer test of resize. + merge_inputs = [ + create_mock_merge_input_channel(file_rule_id=id1, source_channel=0, target_channel=0), + create_mock_merge_input_channel(file_rule_id=id2, source_channel=0, target_channel=1) + ] + merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=2) # Outputting 2 channels + merge_rule_id = uuid.uuid4() + merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="ResizedMerge", merge_settings=merge_settings) + + context = create_map_merging_mock_context( + initial_file_rules=[merge_rule], + initial_processed_details=processed_details + ) + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + assert mock_load_image.call_count == 2 + mock_load_image.assert_any_call(Path('/fake/img1.png')) + mock_load_image.assert_any_call(Path('/fake/img2.png')) + + # Assert resize_image was called for the second image to match the first's dimensions + mock_resize_image.assert_called_once() + # The first argument to resize_image is the image data, second is target_shape tuple (height, width) + # np.array_equal is needed for comparing numpy arrays in mock calls + assert np.array_equal(mock_resize_image.call_args[0][0], mock_img2_data_original) + assert mock_resize_image.call_args[0][1] == (10, 10) + + mock_save_image.assert_called_once() + + saved_data = mock_save_image.call_args[0][1] + assert saved_data.shape == (10, 10, 2) # 2 output channels + assert np.all(saved_data[:,:,0] == mock_img1_data) # First channel from img1 + assert np.all(saved_data[:,:,1] == mock_img2_data_resized) # Second channel from resized img2 + + assert merge_rule.id.hex in updated_context.merged_maps_details + details = updated_context.merged_maps_details[merge_rule.id.hex] + assert details['status'] == 'Processed' + assert "merged_ResizedMerge" in details['temp_merged_file'] + mock_log_error.assert_not_called() + mock_log_info.assert_any_call(f"Resized input map from {Path('/fake/img2.png')} from {mock_img2_data_original.shape} to {(10,10)} to match first loaded map.") + mock_log_info.assert_any_call(f"Successfully merged map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'.") +@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.resize_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_map_merging_to_grayscale_output(mock_log_error, mock_log_info, mock_load_image, mock_resize_image, mock_save_image): + stage = MapMergingStage() + + # Input FileRule (e.g., an RGB image) + input_id = uuid.uuid4() + processed_details = { + input_id.hex: {'temp_processed_file': '/fake/rgb_source.png', 'status': 'Processed', 'map_type': 'RGB_SRC'} + } + # Mock loaded image data (3 channels) + mock_rgb_data = np.full((10, 10, 3), [50, 100, 150], dtype=np.uint8) + mock_load_image.return_value = mock_rgb_data + + # Merge Rule setup: take the Green channel (source_channel=1) from input and map it to the single output channel (target_channel=0) + merge_inputs = [ + create_mock_merge_input_channel(file_rule_id=input_id, source_channel=1, target_channel=0) # G to Grayscale + ] + # output_channels = 1 for grayscale + merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1) + merge_rule_id = uuid.uuid4() + merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="GrayscaleFromGreen", merge_settings=merge_settings) + + context = create_map_merging_mock_context( + initial_file_rules=[merge_rule], + initial_processed_details=processed_details + ) + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(Path('/fake/rgb_source.png')) + mock_resize_image.assert_not_called() + mock_save_image.assert_called_once() + + saved_data = mock_save_image.call_args[0][1] + assert saved_data.shape == (10, 10) # Grayscale output (2D) + assert np.all(saved_data == 100) # Green channel's value + + assert merge_rule.id.hex in updated_context.merged_maps_details + details = updated_context.merged_maps_details[merge_rule.id.hex] + assert details['status'] == 'Processed' + assert "merged_GrayscaleFromGreen" in details['temp_merged_file'] + mock_log_error.assert_not_called() + mock_log_info.assert_any_call(f"Successfully merged map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'.") + +@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image') +@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image') +@mock.patch('logging.error') +def test_map_merging_default_value_if_missing_channel(mock_log_error, mock_load_image, mock_save_image): + stage = MapMergingStage() + + input_id = uuid.uuid4() + processed_details = { + # Input is a grayscale image (1 channel) + input_id.hex: {'temp_processed_file': '/fake/gray_source.png', 'status': 'Processed', 'map_type': 'GRAY_SRC'} + } + mock_gray_data = np.full((10, 10), 50, dtype=np.uint8) + mock_load_image.return_value = mock_gray_data + + # Merge Rule: try to read source_channel 1 (which doesn't exist in grayscale) + # and use default_value_if_missing for target_channel 0. + # Also, read source_channel 0 (which exists) for target_channel 1. + mic1 = create_mock_merge_input_channel(file_rule_id=input_id, source_channel=1, target_channel=0) + mic1.default_value_if_missing = 128 # Set a specific default value + mic2 = create_mock_merge_input_channel(file_rule_id=input_id, source_channel=0, target_channel=1) + + merge_settings = create_mock_merge_settings(input_maps=[mic1, mic2], output_channels=2) + merge_rule_id = uuid.uuid4() + merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="DefaultValueTest", merge_settings=merge_settings) + + context = create_map_merging_mock_context( + initial_file_rules=[merge_rule], + initial_processed_details=processed_details + ) + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(Path('/fake/gray_source.png')) + mock_save_image.assert_called_once() + + saved_data = mock_save_image.call_args[0][1] + assert saved_data.shape == (10, 10, 2) + assert np.all(saved_data[:,:,0] == 128) # Default value for missing source channel 1 + assert np.all(saved_data[:,:,1] == 50) # Value from existing source channel 0 + + assert merge_rule.id.hex in updated_context.merged_maps_details + details = updated_context.merged_maps_details[merge_rule.id.hex] + assert details['status'] == 'Processed' + mock_log_error.assert_not_called() \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_metadata_finalization_save.py b/tests/processing/pipeline/stages/test_metadata_finalization_save.py new file mode 100644 index 0000000..68741ce --- /dev/null +++ b/tests/processing/pipeline/stages/test_metadata_finalization_save.py @@ -0,0 +1,359 @@ +import pytest +from unittest import mock +from pathlib import Path +import datetime +import json # For comparing dumped content +import uuid +from typing import Optional, Dict, Any + +from processing.pipeline.stages.metadata_finalization_save import MetadataFinalizationAndSaveStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule +from configuration import Configuration, GeneralSettings # Added GeneralSettings as it's in the helper + + +def create_metadata_save_mock_context( + status_flags: Optional[Dict[str, Any]] = None, + initial_asset_metadata: Optional[Dict[str, Any]] = None, + processed_details: Optional[Dict[str, Any]] = None, + merged_details: Optional[Dict[str, Any]] = None, + asset_name: str = "MetaSaveAsset", + output_path_pattern_val: str = "{asset_name}/metadata/{filename}", + # ... other common context fields ... +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + mock_asset_rule.output_path_pattern = output_path_pattern_val + mock_asset_rule.id = uuid.uuid4() # Needed for generate_path_from_pattern if it uses it + + mock_source_rule = mock.MagicMock(spec=SourceRule) + mock_source_rule.name = "MetaSaveSource" + + mock_config = mock.MagicMock(spec=Configuration) + # mock_config.general_settings = mock.MagicMock(spec=GeneralSettings) # If needed + + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp_engine_dir"), + output_base_path=Path("/fake/output_base"), # For generate_path + effective_supplier="ValidSupplier", + asset_metadata=initial_asset_metadata if initial_asset_metadata is not None else {}, + processed_maps_details=processed_details if processed_details is not None else {}, + merged_maps_details=merged_details if merged_details is not None else {}, + files_to_process=[], + loaded_data_cache={}, + config_obj=mock_config, + status_flags=status_flags if status_flags is not None else {}, + incrementing_value="001", # Example for path generation + sha5_value="abc" # Example for path generation + ) + return context +@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump') +@mock.patch('builtins.open', new_callable=mock.mock_open) +@mock.patch('pathlib.Path.mkdir') +@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern') +@mock.patch('datetime.datetime') +def test_asset_skipped_before_metadata_init( + mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump +): + """ + Tests that if an asset is marked for skipping and has no initial metadata, + the stage returns early without attempting to save metadata. + """ + stage = MetadataFinalizationAndSaveStage() + context = create_metadata_save_mock_context( + status_flags={'skip_asset': True}, + initial_asset_metadata={} # Explicitly empty + ) + + updated_context = stage.execute(context) + + # Assert that no processing or saving attempts were made + mock_dt.now.assert_not_called() # Should not even try to set end time if no metadata + mock_gen_path.assert_not_called() + mock_mkdir.assert_not_called() + mock_file_open.assert_not_called() + mock_json_dump.assert_not_called() + + assert updated_context.asset_metadata == {} # Metadata remains empty + assert 'metadata_file_path' not in updated_context.asset_metadata + assert updated_context.status_flags.get('metadata_save_error') is None +@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump') +@mock.patch('builtins.open', new_callable=mock.mock_open) +@mock.patch('pathlib.Path.mkdir') +@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern') +@mock.patch('datetime.datetime') +def test_asset_skipped_after_metadata_init( + mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump +): + """ + Tests that if an asset is marked for skipping but has initial metadata, + the status is updated to 'Skipped' and metadata is saved. + """ + stage = MetadataFinalizationAndSaveStage() + + fixed_now = datetime.datetime(2023, 1, 1, 12, 0, 0) + mock_dt.now.return_value = fixed_now + + fake_metadata_path_str = "/fake/output_base/SkippedAsset/metadata/SkippedAsset_metadata.json" + mock_gen_path.return_value = fake_metadata_path_str + + initial_meta = {'asset_name': "SkippedAsset", 'status': "Pending"} + + context = create_metadata_save_mock_context( + asset_name="SkippedAsset", + status_flags={'skip_asset': True}, + initial_asset_metadata=initial_meta + ) + + updated_context = stage.execute(context) + + mock_dt.now.assert_called_once() + mock_gen_path.assert_called_once_with( + context.asset_rule.output_path_pattern, + context.asset_rule, + context.source_rule, + context.output_base_path, + context.asset_metadata, # Original metadata passed for path gen + context.incrementing_value, + context.sha5_value, + filename_override=f"{context.asset_rule.name}_metadata.json" + ) + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) + mock_file_open.assert_called_once_with(Path(fake_metadata_path_str), 'w') + mock_json_dump.assert_called_once() + + dumped_data = mock_json_dump.call_args[0][0] + assert dumped_data['status'] == "Skipped" + assert dumped_data['processing_end_time'] == fixed_now.isoformat() + assert 'processed_map_details' not in dumped_data # Should not be present if skipped early + assert 'merged_map_details' not in dumped_data # Should not be present if skipped early + + assert updated_context.asset_metadata['status'] == "Skipped" + assert updated_context.asset_metadata['processing_end_time'] == fixed_now.isoformat() + assert updated_context.asset_metadata['metadata_file_path'] == fake_metadata_path_str + assert updated_context.status_flags.get('metadata_save_error') is None +@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump') +@mock.patch('builtins.open', new_callable=mock.mock_open) # Mocks open() +@mock.patch('pathlib.Path.mkdir') +@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern') +@mock.patch('datetime.datetime') +def test_metadata_save_success(mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump): + """ + Tests successful metadata finalization and saving, including serialization of Path objects. + """ + stage = MetadataFinalizationAndSaveStage() + + fixed_now = datetime.datetime(2023, 1, 1, 12, 30, 0) + mock_dt.now.return_value = fixed_now + + fake_metadata_path_str = "/fake/output_base/MetaSaveAsset/metadata/MetaSaveAsset_metadata.json" + mock_gen_path.return_value = fake_metadata_path_str + + initial_meta = {'asset_name': "MetaSaveAsset", 'status': "Pending", 'processing_start_time': "2023-01-01T12:00:00"} + # Example of a Path object that needs serialization + proc_details = {'map1': {'temp_processed_file': Path('/fake/temp_engine_dir/map1.png'), 'final_file_path': Path('/fake/output_base/MetaSaveAsset/map1.png')}} + merged_details = {'merged_map_A': {'output_path': Path('/fake/output_base/MetaSaveAsset/merged_A.png')}} + + context = create_metadata_save_mock_context( + initial_asset_metadata=initial_meta, + processed_details=proc_details, + merged_details=merged_details, + status_flags={} # No errors, no skip + ) + + updated_context = stage.execute(context) + + mock_dt.now.assert_called_once() + mock_gen_path.assert_called_once_with( + context.asset_rule.output_path_pattern, + context.asset_rule, + context.source_rule, + context.output_base_path, + context.asset_metadata, # The metadata *before* adding end_time, status etc. + context.incrementing_value, + context.sha5_value, + filename_override=f"{context.asset_rule.name}_metadata.json" + ) + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) # Checks parent dir of fake_metadata_path_str + mock_file_open.assert_called_once_with(Path(fake_metadata_path_str), 'w') + mock_json_dump.assert_called_once() + + # Check what was passed to json.dump + dumped_data = mock_json_dump.call_args[0][0] + assert dumped_data['status'] == "Processed" + assert dumped_data['processing_end_time'] == fixed_now.isoformat() + assert 'processing_start_time' in dumped_data # Ensure existing fields are preserved + + # Verify processed_map_details and Path serialization + assert 'processed_map_details' in dumped_data + assert dumped_data['processed_map_details']['map1']['temp_processed_file'] == '/fake/temp_engine_dir/map1.png' + assert dumped_data['processed_map_details']['map1']['final_file_path'] == '/fake/output_base/MetaSaveAsset/map1.png' + + # Verify merged_map_details and Path serialization + assert 'merged_map_details' in dumped_data + assert dumped_data['merged_map_details']['merged_map_A']['output_path'] == '/fake/output_base/MetaSaveAsset/merged_A.png' + + assert updated_context.asset_metadata['metadata_file_path'] == fake_metadata_path_str + assert updated_context.asset_metadata['status'] == "Processed" + assert updated_context.status_flags.get('metadata_save_error') is None +@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump') +@mock.patch('builtins.open', new_callable=mock.mock_open) +@mock.patch('pathlib.Path.mkdir') +@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern') +@mock.patch('datetime.datetime') +def test_processing_failed_due_to_previous_error( + mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump +): + """ + Tests that if a previous stage set an error flag, the status is 'Failed' + and metadata (including any existing details) is saved. + """ + stage = MetadataFinalizationAndSaveStage() + + fixed_now = datetime.datetime(2023, 1, 1, 12, 45, 0) + mock_dt.now.return_value = fixed_now + + fake_metadata_path_str = "/fake/output_base/FailedAsset/metadata/FailedAsset_metadata.json" + mock_gen_path.return_value = fake_metadata_path_str + + initial_meta = {'asset_name': "FailedAsset", 'status': "Processing"} + # Simulate some details might exist even if a later stage failed + proc_details = {'map1_partial': {'temp_processed_file': Path('/fake/temp_engine_dir/map1_partial.png')}} + + context = create_metadata_save_mock_context( + asset_name="FailedAsset", + initial_asset_metadata=initial_meta, + processed_details=proc_details, + merged_details={}, # No merged details if processing failed before that + status_flags={'file_processing_error': True, 'error_message': "Something went wrong"} + ) + + updated_context = stage.execute(context) + + mock_dt.now.assert_called_once() + mock_gen_path.assert_called_once() # Path generation should still occur + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) + mock_file_open.assert_called_once_with(Path(fake_metadata_path_str), 'w') + mock_json_dump.assert_called_once() + + dumped_data = mock_json_dump.call_args[0][0] + assert dumped_data['status'] == "Failed" + assert dumped_data['processing_end_time'] == fixed_now.isoformat() + assert 'error_message' in dumped_data # Assuming error messages from status_flags are copied + assert dumped_data['error_message'] == "Something went wrong" + + # Check that existing details are included + assert 'processed_map_details' in dumped_data + assert dumped_data['processed_map_details']['map1_partial']['temp_processed_file'] == '/fake/temp_engine_dir/map1_partial.png' + assert 'merged_map_details' in dumped_data # Should be present, even if empty + assert dumped_data['merged_map_details'] == {} + + assert updated_context.asset_metadata['status'] == "Failed" + assert updated_context.asset_metadata['metadata_file_path'] == fake_metadata_path_str + assert updated_context.status_flags.get('metadata_save_error') is None + # Ensure the original error flag is preserved + assert updated_context.status_flags['file_processing_error'] is True +@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump') +@mock.patch('builtins.open', new_callable=mock.mock_open) +@mock.patch('pathlib.Path.mkdir') +@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern') +@mock.patch('datetime.datetime') +@mock.patch('logging.error') # To check if error is logged +def test_generate_path_fails( + mock_log_error, mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump +): + """ + Tests behavior when generate_path_from_pattern raises an exception. + Ensures status is updated, error flag is set, and no save is attempted. + """ + stage = MetadataFinalizationAndSaveStage() + + fixed_now = datetime.datetime(2023, 1, 1, 12, 50, 0) + mock_dt.now.return_value = fixed_now + + mock_gen_path.side_effect = Exception("Simulated path generation error") + + initial_meta = {'asset_name': "PathFailAsset", 'status': "Processing"} + context = create_metadata_save_mock_context( + asset_name="PathFailAsset", + initial_asset_metadata=initial_meta, + status_flags={} + ) + + updated_context = stage.execute(context) + + mock_dt.now.assert_called_once() # Time is set before path generation + mock_gen_path.assert_called_once() # generate_path_from_pattern is called + + # File operations should NOT be called if path generation fails + mock_mkdir.assert_not_called() + mock_file_open.assert_not_called() + mock_json_dump.assert_not_called() + + mock_log_error.assert_called_once() # Check that an error was logged + # Example: check if the log message contains relevant info, if needed + # assert "Failed to generate metadata path" in mock_log_error.call_args[0][0] + + assert updated_context.asset_metadata['status'] == "Failed" # Or a more specific error status + assert 'processing_end_time' in updated_context.asset_metadata # End time should still be set + assert updated_context.asset_metadata['processing_end_time'] == fixed_now.isoformat() + assert 'metadata_file_path' not in updated_context.asset_metadata # Path should not be set + + assert updated_context.status_flags.get('metadata_save_error') is True + assert 'error_message' in updated_context.asset_metadata # Check if error message is populated + assert "Simulated path generation error" in updated_context.asset_metadata['error_message'] +@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump') +@mock.patch('builtins.open', new_callable=mock.mock_open) +@mock.patch('pathlib.Path.mkdir') +@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern') +@mock.patch('datetime.datetime') +@mock.patch('logging.error') # To check if error is logged +def test_json_dump_fails( + mock_log_error, mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump +): + """ + Tests behavior when json.dump raises an exception during saving. + Ensures status is updated, error flag is set, and error is logged. + """ + stage = MetadataFinalizationAndSaveStage() + + fixed_now = datetime.datetime(2023, 1, 1, 12, 55, 0) + mock_dt.now.return_value = fixed_now + + fake_metadata_path_str = "/fake/output_base/JsonDumpFailAsset/metadata/JsonDumpFailAsset_metadata.json" + mock_gen_path.return_value = fake_metadata_path_str + + mock_json_dump.side_effect = IOError("Simulated JSON dump error") # Or TypeError for non-serializable + + initial_meta = {'asset_name': "JsonDumpFailAsset", 'status': "Processing"} + context = create_metadata_save_mock_context( + asset_name="JsonDumpFailAsset", + initial_asset_metadata=initial_meta, + status_flags={} + ) + + updated_context = stage.execute(context) + + mock_dt.now.assert_called_once() + mock_gen_path.assert_called_once() + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) + mock_file_open.assert_called_once_with(Path(fake_metadata_path_str), 'w') + mock_json_dump.assert_called_once() # json.dump was attempted + + mock_log_error.assert_called_once() + # assert "Failed to save metadata JSON" in mock_log_error.call_args[0][0] + + assert updated_context.asset_metadata['status'] == "Failed" # Or specific "Metadata Save Failed" + assert 'processing_end_time' in updated_context.asset_metadata + assert updated_context.asset_metadata['processing_end_time'] == fixed_now.isoformat() + # metadata_file_path might be set if path generation succeeded, even if dump failed. + # Depending on desired behavior, this could be asserted or not. + # For now, let's assume it's set if path generation was successful. + assert updated_context.asset_metadata['metadata_file_path'] == fake_metadata_path_str + + assert updated_context.status_flags.get('metadata_save_error') is True + assert 'error_message' in updated_context.asset_metadata + assert "Simulated JSON dump error" in updated_context.asset_metadata['error_message'] \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_metadata_initialization.py b/tests/processing/pipeline/stages/test_metadata_initialization.py new file mode 100644 index 0000000..5b358fd --- /dev/null +++ b/tests/processing/pipeline/stages/test_metadata_initialization.py @@ -0,0 +1,169 @@ +import pytest +from unittest import mock +from pathlib import Path +import datetime +import uuid +from typing import Optional + +from processing.pipeline.stages.metadata_initialization import MetadataInitializationStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule +from configuration import Configuration, GeneralSettings + +# Helper function to create a mock AssetProcessingContext +def create_metadata_init_mock_context( + skip_asset_flag: bool = False, + asset_name: str = "MetaAsset", + asset_id: uuid.UUID = None, # Allow None to default to uuid.uuid4() + source_path_str: str = "source/meta_asset", + output_pattern: str = "{asset_name}/{map_type}", + tags: list = None, + custom_fields: dict = None, + source_rule_name: str = "MetaSource", + source_rule_id: uuid.UUID = None, # Allow None to default to uuid.uuid4() + eff_supplier: Optional[str] = "SupplierMeta", + app_version_str: str = "1.0.0-test", + inc_val: Optional[str] = None, + sha_val: Optional[str] = None +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + mock_asset_rule.id = asset_id if asset_id is not None else uuid.uuid4() + mock_asset_rule.source_path = Path(source_path_str) + mock_asset_rule.output_path_pattern = output_pattern + mock_asset_rule.tags = tags if tags is not None else ["tag1", "test_tag"] + mock_asset_rule.custom_fields = custom_fields if custom_fields is not None else {"custom_key": "custom_value"} + + mock_source_rule = mock.MagicMock(spec=SourceRule) + mock_source_rule.name = source_rule_name + mock_source_rule.id = source_rule_id if source_rule_id is not None else uuid.uuid4() + + mock_general_settings = mock.MagicMock(spec=GeneralSettings) + mock_general_settings.app_version = app_version_str + + mock_config = mock.MagicMock(spec=Configuration) + mock_config.general_settings = mock_general_settings + + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp"), + output_base_path=Path("/fake/output"), + effective_supplier=eff_supplier, + asset_metadata={}, + processed_maps_details={}, + merged_maps_details={}, + files_to_process=[], + loaded_data_cache={}, + config_obj=mock_config, + status_flags={'skip_asset': skip_asset_flag}, + incrementing_value=inc_val, + sha5_value=sha_val + ) + return context + +@mock.patch('processing.pipeline.stages.metadata_initialization.datetime') +def test_metadata_initialization_not_skipped(mock_datetime_module): + stage = MetadataInitializationStage() + + fixed_now = datetime.datetime(2023, 10, 26, 12, 0, 0, tzinfo=datetime.timezone.utc) + mock_datetime_module.datetime.now.return_value = fixed_now + + asset_id_val = uuid.uuid4() + source_id_val = uuid.uuid4() + + context = create_metadata_init_mock_context( + skip_asset_flag=False, + asset_id=asset_id_val, + source_rule_id=source_id_val, + inc_val="001", + sha_val="abcde" + ) + + updated_context = stage.execute(context) + + assert isinstance(updated_context.asset_metadata, dict) + assert isinstance(updated_context.processed_maps_details, dict) + assert isinstance(updated_context.merged_maps_details, dict) + + md = updated_context.asset_metadata + assert md['asset_name'] == "MetaAsset" + assert md['asset_id'] == str(asset_id_val) + assert md['source_rule_name'] == "MetaSource" + assert md['source_rule_id'] == str(source_id_val) + assert md['source_path'] == "source/meta_asset" + assert md['effective_supplier'] == "SupplierMeta" + assert md['output_path_pattern'] == "{asset_name}/{map_type}" + assert md['processing_start_time'] == fixed_now.isoformat() + assert md['status'] == "Pending" + assert md['version'] == "1.0.0-test" + assert md['tags'] == ["tag1", "test_tag"] + assert md['custom_fields'] == {"custom_key": "custom_value"} + assert md['incrementing_value'] == "001" + assert md['sha5_value'] == "abcde" + +@mock.patch('processing.pipeline.stages.metadata_initialization.datetime') +def test_metadata_initialization_not_skipped_none_inc_sha(mock_datetime_module): + stage = MetadataInitializationStage() + + fixed_now = datetime.datetime(2023, 10, 26, 12, 0, 0, tzinfo=datetime.timezone.utc) + mock_datetime_module.datetime.now.return_value = fixed_now + + context = create_metadata_init_mock_context( + skip_asset_flag=False, + inc_val=None, + sha_val=None + ) + + updated_context = stage.execute(context) + + md = updated_context.asset_metadata + assert 'incrementing_value' not in md # Or assert md['incrementing_value'] is None, depending on desired behavior + assert 'sha5_value' not in md # Or assert md['sha5_value'] is None + +def test_metadata_initialization_skipped(): + stage = MetadataInitializationStage() + context = create_metadata_init_mock_context(skip_asset_flag=True) + + # Make copies of initial state to ensure they are not modified + initial_asset_metadata = dict(context.asset_metadata) + initial_processed_maps = dict(context.processed_maps_details) + initial_merged_maps = dict(context.merged_maps_details) + + updated_context = stage.execute(context) + + assert updated_context.asset_metadata == initial_asset_metadata + assert updated_context.processed_maps_details == initial_processed_maps + assert updated_context.merged_maps_details == initial_merged_maps + assert not updated_context.asset_metadata # Explicitly check it's empty as per initial setup + assert not updated_context.processed_maps_details + assert not updated_context.merged_maps_details + +@mock.patch('processing.pipeline.stages.metadata_initialization.datetime') +def test_tags_and_custom_fields_are_copies(mock_datetime_module): + stage = MetadataInitializationStage() + fixed_now = datetime.datetime(2023, 10, 26, 12, 0, 0, tzinfo=datetime.timezone.utc) + mock_datetime_module.datetime.now.return_value = fixed_now + + original_tags = ["original_tag"] + original_custom_fields = {"original_key": "original_value"} + + context = create_metadata_init_mock_context( + skip_asset_flag=False, + tags=original_tags, + custom_fields=original_custom_fields + ) + + # Modify originals after context creation but before stage execution + original_tags.append("modified_after_creation") + original_custom_fields["new_key_after_creation"] = "new_value" + + updated_context = stage.execute(context) + + md = updated_context.asset_metadata + assert md['tags'] == ["original_tag"] # Should not have "modified_after_creation" + assert md['tags'] is not original_tags # Ensure it's a different object + + assert md['custom_fields'] == {"original_key": "original_value"} # Should not have "new_key_after_creation" + assert md['custom_fields'] is not original_custom_fields # Ensure it's a different object \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_normal_map_green_channel.py b/tests/processing/pipeline/stages/test_normal_map_green_channel.py new file mode 100644 index 0000000..3120655 --- /dev/null +++ b/tests/processing/pipeline/stages/test_normal_map_green_channel.py @@ -0,0 +1,323 @@ +import pytest +from unittest import mock +from pathlib import Path +import uuid +import numpy as np +import logging # Added for mocking logger + +from processing.pipeline.stages.normal_map_green_channel import NormalMapGreenChannelStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule, FileRule +from configuration import Configuration, GeneralSettings + +# Helper functions +def create_mock_file_rule_for_normal_test( + id_val: uuid.UUID = None, # Corrected type hint from Optional[uuid.UUID] + map_type: str = "NORMAL", + filename_pattern: str = "normal.png" +) -> mock.MagicMock: + mock_fr = mock.MagicMock(spec=FileRule) + mock_fr.id = id_val if id_val else uuid.uuid4() + mock_fr.map_type = map_type + mock_fr.filename_pattern = filename_pattern + mock_fr.item_type = "MAP_COL" # As per example, though not directly used by stage + mock_fr.active = True # As per example + return mock_fr + +def create_normal_map_mock_context( + initial_file_rules: list = None, # Corrected type hint + initial_processed_details: dict = None, # Corrected type hint + invert_green_globally: bool = False, + skip_asset_flag: bool = False, + asset_name: str = "NormalMapAsset" +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + + mock_source_rule = mock.MagicMock(spec=SourceRule) + + mock_gs = mock.MagicMock(spec=GeneralSettings) + mock_gs.invert_normal_map_green_channel_globally = invert_green_globally + + mock_config = mock.MagicMock(spec=Configuration) + mock_config.general_settings = mock_gs + + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp_engine_dir"), + output_base_path=Path("/fake/output"), + effective_supplier="ValidSupplier", + asset_metadata={'asset_name': asset_name}, + processed_maps_details=initial_processed_details if initial_processed_details is not None else {}, + merged_maps_details={}, + files_to_process=list(initial_file_rules) if initial_file_rules else [], + loaded_data_cache={}, + config_obj=mock_config, + status_flags={'skip_asset': skip_asset_flag}, + incrementing_value=None, # Added as per AssetProcessingContext constructor + sha5_value=None # Added as per AssetProcessingContext constructor + ) + return context + +# Unit tests will be added below +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image') +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image') +def test_asset_skipped(mock_load_image, mock_save_image): + stage = NormalMapGreenChannelStage() + normal_fr = create_mock_file_rule_for_normal_test(map_type="NORMAL") + initial_details = { + normal_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_normal.png', 'status': 'Processed', 'map_type': 'NORMAL', 'notes': ''} + } + context = create_normal_map_mock_context( + initial_file_rules=[normal_fr], + initial_processed_details=initial_details, + invert_green_globally=True, + skip_asset_flag=True # Asset is skipped + ) + original_details = context.processed_maps_details.copy() + + updated_context = stage.execute(context) + + mock_load_image.assert_not_called() + mock_save_image.assert_not_called() + assert updated_context.processed_maps_details == original_details + assert normal_fr in updated_context.files_to_process # Ensure rule is still there + +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image') +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image') +def test_no_normal_map_present(mock_load_image, mock_save_image): + stage = NormalMapGreenChannelStage() + # Create a non-normal map rule + diffuse_fr = create_mock_file_rule_for_normal_test(map_type="DIFFUSE", filename_pattern="diffuse.png") + initial_details = { + diffuse_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_diffuse.png', 'status': 'Processed', 'map_type': 'DIFFUSE', 'notes': ''} + } + context = create_normal_map_mock_context( + initial_file_rules=[diffuse_fr], + initial_processed_details=initial_details, + invert_green_globally=True # Inversion enabled, but no normal map + ) + original_details = context.processed_maps_details.copy() + + updated_context = stage.execute(context) + + mock_load_image.assert_not_called() + mock_save_image.assert_not_called() + assert updated_context.processed_maps_details == original_details + assert diffuse_fr in updated_context.files_to_process + +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image') +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image') +def test_normal_map_present_inversion_disabled(mock_load_image, mock_save_image): + stage = NormalMapGreenChannelStage() + normal_rule_id = uuid.uuid4() + normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL") + initial_details = { + normal_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_normal.png', 'status': 'Processed', 'map_type': 'NORMAL', 'notes': 'Initial note'} + } + context = create_normal_map_mock_context( + initial_file_rules=[normal_fr], + initial_processed_details=initial_details, + invert_green_globally=False # Inversion disabled + ) + original_details_entry = context.processed_maps_details[normal_fr.id.hex].copy() + + updated_context = stage.execute(context) + + mock_load_image.assert_not_called() + mock_save_image.assert_not_called() + assert updated_context.processed_maps_details[normal_fr.id.hex] == original_details_entry + assert normal_fr in updated_context.files_to_process + +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image') +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image') +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_normal_map_inversion_uint8_success(mock_log_debug, mock_log_info, mock_load_image, mock_save_image): + stage = NormalMapGreenChannelStage() + + normal_rule_id = uuid.uuid4() + normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL") + + initial_temp_path = Path('/fake/temp_engine_dir/processed_normal.png') + initial_details = { + normal_fr.id.hex: {'temp_processed_file': str(initial_temp_path), 'status': 'Processed', 'map_type': 'NORMAL', 'notes': 'Initial note'} + } + context = create_normal_map_mock_context( + initial_file_rules=[normal_fr], + initial_processed_details=initial_details, + invert_green_globally=True # Enable inversion + ) + + # R=10, G=50, B=100 + mock_loaded_normal_data = np.array([[[10, 50, 100]]], dtype=np.uint8) + mock_load_image.return_value = mock_loaded_normal_data + mock_save_image.return_value = True # Simulate successful save + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(initial_temp_path) + + # Check that save_image was called with green channel inverted + assert mock_save_image.call_count == 1 + saved_path_arg, saved_data_arg = mock_save_image.call_args[0] + + assert saved_data_arg[0,0,0] == 10 # R unchanged + assert saved_data_arg[0,0,1] == 255 - 50 # G inverted + assert saved_data_arg[0,0,2] == 100 # B unchanged + + assert isinstance(saved_path_arg, Path) + assert "normal_g_inv_" in saved_path_arg.name + assert saved_path_arg.parent == initial_temp_path.parent # Should be in same temp dir + + normal_detail = updated_context.processed_maps_details[normal_fr.id.hex] + assert "normal_g_inv_" in normal_detail['temp_processed_file'] + assert Path(normal_detail['temp_processed_file']).name == saved_path_arg.name + assert "Green channel inverted" in normal_detail['notes'] + assert "Initial note" in normal_detail['notes'] # Check existing notes preserved + + assert normal_fr in updated_context.files_to_process + +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image') +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image') +@mock.patch('logging.info') +@mock.patch('logging.debug') +def test_normal_map_inversion_float_success(mock_log_debug, mock_log_info, mock_load_image, mock_save_image): + stage = NormalMapGreenChannelStage() + normal_rule_id = uuid.uuid4() + normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL") + initial_temp_path = Path('/fake/temp_engine_dir/processed_normal_float.png') + initial_details = { + normal_fr.id.hex: {'temp_processed_file': str(initial_temp_path), 'status': 'Processed', 'map_type': 'NORMAL', 'notes': 'Float image'} + } + context = create_normal_map_mock_context( + initial_file_rules=[normal_fr], + initial_processed_details=initial_details, + invert_green_globally=True + ) + + # R=0.1, G=0.25, B=0.75 + mock_loaded_normal_data = np.array([[[0.1, 0.25, 0.75]]], dtype=np.float32) + mock_load_image.return_value = mock_loaded_normal_data + mock_save_image.return_value = True + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(initial_temp_path) + + assert mock_save_image.call_count == 1 + saved_path_arg, saved_data_arg = mock_save_image.call_args[0] + + assert np.isclose(saved_data_arg[0,0,0], 0.1) # R unchanged + assert np.isclose(saved_data_arg[0,0,1], 1.0 - 0.25) # G inverted + assert np.isclose(saved_data_arg[0,0,2], 0.75) # B unchanged + + assert "normal_g_inv_" in saved_path_arg.name + normal_detail = updated_context.processed_maps_details[normal_fr.id.hex] + assert "normal_g_inv_" in normal_detail['temp_processed_file'] + assert "Green channel inverted" in normal_detail['notes'] + assert "Float image" in normal_detail['notes'] + assert normal_fr in updated_context.files_to_process + +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image') +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image') +@mock.patch('logging.error') +def test_load_image_fails(mock_log_error, mock_load_image, mock_save_image): + stage = NormalMapGreenChannelStage() + normal_rule_id = uuid.uuid4() + normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL") + initial_temp_path_str = '/fake/temp_engine_dir/processed_normal_load_fail.png' + initial_details = { + normal_fr.id.hex: {'temp_processed_file': initial_temp_path_str, 'status': 'Processed', 'map_type': 'NORMAL', 'notes': 'Load fail test'} + } + context = create_normal_map_mock_context( + initial_file_rules=[normal_fr], + initial_processed_details=initial_details, + invert_green_globally=True + ) + original_details_entry = context.processed_maps_details[normal_fr.id.hex].copy() + + mock_load_image.return_value = None # Simulate load failure + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(Path(initial_temp_path_str)) + mock_save_image.assert_not_called() + mock_log_error.assert_called_once() + assert f"Failed to load image {Path(initial_temp_path_str)} for green channel inversion." in mock_log_error.call_args[0][0] + + # Details should be unchanged + assert updated_context.processed_maps_details[normal_fr.id.hex] == original_details_entry + assert normal_fr in updated_context.files_to_process + +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image') +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image') +@mock.patch('logging.error') +def test_save_image_fails(mock_log_error, mock_load_image, mock_save_image): + stage = NormalMapGreenChannelStage() + normal_rule_id = uuid.uuid4() + normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL") + initial_temp_path = Path('/fake/temp_engine_dir/processed_normal_save_fail.png') + initial_details = { + normal_fr.id.hex: {'temp_processed_file': str(initial_temp_path), 'status': 'Processed', 'map_type': 'NORMAL', 'notes': 'Save fail test'} + } + context = create_normal_map_mock_context( + initial_file_rules=[normal_fr], + initial_processed_details=initial_details, + invert_green_globally=True + ) + original_details_entry = context.processed_maps_details[normal_fr.id.hex].copy() + + mock_loaded_normal_data = np.array([[[10, 50, 100]]], dtype=np.uint8) + mock_load_image.return_value = mock_loaded_normal_data + mock_save_image.return_value = False # Simulate save failure + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(initial_temp_path) + mock_save_image.assert_called_once() # Save is attempted + + saved_path_arg = mock_save_image.call_args[0][0] # Get the path it tried to save to + mock_log_error.assert_called_once() + assert f"Failed to save green channel inverted image to {saved_path_arg}." in mock_log_error.call_args[0][0] + + # Details should be unchanged + assert updated_context.processed_maps_details[normal_fr.id.hex] == original_details_entry + assert normal_fr in updated_context.files_to_process + +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image') +@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image') +@mock.patch('logging.error') +@pytest.mark.parametrize("unsuitable_data, description", [ + (np.array([[1, 2], [3, 4]], dtype=np.uint8), "2D array"), # 2D array + (np.array([[[1, 2]]], dtype=np.uint8), "2-channel image") # Image with less than 3 channels +]) +def test_image_not_suitable_for_inversion(mock_log_error, mock_load_image, mock_save_image, unsuitable_data, description): + stage = NormalMapGreenChannelStage() + normal_rule_id = uuid.uuid4() + normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL") + initial_temp_path_str = f'/fake/temp_engine_dir/unsuitable_{description.replace(" ", "_")}.png' + initial_details = { + normal_fr.id.hex: {'temp_processed_file': initial_temp_path_str, 'status': 'Processed', 'map_type': 'NORMAL', 'notes': f'Unsuitable: {description}'} + } + context = create_normal_map_mock_context( + initial_file_rules=[normal_fr], + initial_processed_details=initial_details, + invert_green_globally=True + ) + original_details_entry = context.processed_maps_details[normal_fr.id.hex].copy() + + mock_load_image.return_value = unsuitable_data + + updated_context = stage.execute(context) + + mock_load_image.assert_called_once_with(Path(initial_temp_path_str)) + mock_save_image.assert_not_called() # Save should not be attempted + mock_log_error.assert_called_once() + assert f"Image at {Path(initial_temp_path_str)} is not suitable for green channel inversion (e.g., not RGB/RGBA)." in mock_log_error.call_args[0][0] + + # Details should be unchanged + assert updated_context.processed_maps_details[normal_fr.id.hex] == original_details_entry + assert normal_fr in updated_context.files_to_process \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_output_organization.py b/tests/processing/pipeline/stages/test_output_organization.py new file mode 100644 index 0000000..ccf6c08 --- /dev/null +++ b/tests/processing/pipeline/stages/test_output_organization.py @@ -0,0 +1,417 @@ +import pytest +from unittest import mock +from pathlib import Path +import shutil # To check if shutil.copy2 is called +import uuid +from typing import Optional # Added for type hinting in helper + +from processing.pipeline.stages.output_organization import OutputOrganizationStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule, FileRule # For context setup +from configuration import Configuration, GeneralSettings + +def create_output_org_mock_context( + status_flags: Optional[dict] = None, + asset_metadata_status: str = "Processed", # Default to processed for testing copy + processed_map_details: Optional[dict] = None, + merged_map_details: Optional[dict] = None, + overwrite_setting: bool = False, + asset_name: str = "OutputOrgAsset", + output_path_pattern_val: str = "{asset_name}/{map_type}/{filename}" +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + mock_asset_rule.output_path_pattern = output_path_pattern_val + # Need FileRules on AssetRule if stage tries to look up output_filename_pattern from them + # For simplicity, assume stage constructs output_filename for now if not found on FileRule + mock_asset_rule.file_rules = [] # Or mock FileRules if stage uses them for output_filename_pattern + + mock_source_rule = mock.MagicMock(spec=SourceRule) + mock_source_rule.name = "OutputOrgSource" + + mock_gs = mock.MagicMock(spec=GeneralSettings) + mock_gs.overwrite_existing = overwrite_setting + + mock_config = mock.MagicMock(spec=Configuration) + mock_config.general_settings = mock_gs + + # Ensure asset_metadata has a status + initial_asset_metadata = {'asset_name': asset_name, 'status': asset_metadata_status} + + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp_engine_dir"), + output_base_path=Path("/fake/output_final"), + effective_supplier="ValidSupplier", + asset_metadata=initial_asset_metadata, + processed_maps_details=processed_map_details if processed_map_details is not None else {}, + merged_maps_details=merged_map_details if merged_map_details is not None else {}, + files_to_process=[], # Not directly used by this stage, but good to have + loaded_data_cache={}, + config_obj=mock_config, + status_flags=status_flags if status_flags is not None else {}, + incrementing_value="001", + sha5_value="xyz" # Corrected from sha5_value to sha256_value if that's the actual param, or ensure it's a valid param. Assuming sha5_value is a typo and should be something like 'unique_id' or similar if not sha256. For now, keeping as sha5_value as per instructions. + ) + return context +@mock.patch('shutil.copy2') +@mock.patch('logging.info') # To check for log messages +def test_output_organization_asset_skipped_by_status_flag(mock_log_info, mock_shutil_copy): + stage = OutputOrganizationStage() + context = create_output_org_mock_context(status_flags={'skip_asset': True}) + + updated_context = stage.execute(context) + + mock_shutil_copy.assert_not_called() + # Check if a log message indicates skipping, if applicable + # e.g., mock_log_info.assert_any_call("Skipping output organization for asset OutputOrgAsset due to skip_asset flag.") + assert 'final_output_files' not in updated_context.asset_metadata # Or assert it's empty + assert updated_context.asset_metadata['status'] == "Processed" # Status should not change if skipped due to flag before stage logic + # Add specific log check if the stage logs this event + # For now, assume no copy is the primary check + +@mock.patch('shutil.copy2') +@mock.patch('logging.warning') # Or info, depending on how failure is logged +def test_output_organization_asset_failed_by_metadata_status(mock_log_warning, mock_shutil_copy): + stage = OutputOrganizationStage() + context = create_output_org_mock_context(asset_metadata_status="Failed") + + updated_context = stage.execute(context) + + mock_shutil_copy.assert_not_called() + # Check for a log message indicating skipping due to failure status + # e.g., mock_log_warning.assert_any_call("Skipping output organization for asset OutputOrgAsset as its status is Failed.") + assert 'final_output_files' not in updated_context.asset_metadata # Or assert it's empty + assert updated_context.asset_metadata['status'] == "Failed" # Status remains Failed + +@mock.patch('shutil.copy2') +@mock.patch('pathlib.Path.mkdir') +@mock.patch('pathlib.Path.exists') +@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_output_organization_success_no_overwrite( + mock_log_error, mock_log_info, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy +): + stage = OutputOrganizationStage() + + proc_id_1 = uuid.uuid4().hex + merged_id_1 = uuid.uuid4().hex + + processed_details = { + proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc1.png', 'map_type': 'Diffuse', 'output_filename': 'OutputOrgAsset_Diffuse.png'} + } + merged_details = { + merged_id_1: {'status': 'Processed', 'temp_merged_file': '/fake/temp_engine_dir/merged1.png', 'map_type': 'ORM', 'output_filename': 'OutputOrgAsset_ORM.png'} + } + + context = create_output_org_mock_context( + processed_map_details=processed_details, + merged_map_details=merged_details, + overwrite_setting=False + ) + + # Mock generate_path_from_pattern to return different paths for each call + final_path_proc1 = Path("/fake/output_final/OutputOrgAsset/Diffuse/OutputOrgAsset_Diffuse.png") + final_path_merged1 = Path("/fake/output_final/OutputOrgAsset/ORM/OutputOrgAsset_ORM.png") + # Ensure generate_path_from_pattern is called with the correct context and details + # The actual call in the stage is: generate_path_from_pattern(context, map_detail, map_type_key, temp_file_key) + # We need to ensure our side_effect matches these calls. + + def gen_path_side_effect(ctx, detail, map_type_key, temp_file_key, output_filename_key): + if detail['temp_processed_file'] == '/fake/temp_engine_dir/proc1.png': + return final_path_proc1 + elif detail['temp_merged_file'] == '/fake/temp_engine_dir/merged1.png': + return final_path_merged1 + raise ValueError("Unexpected call to generate_path_from_pattern") + + mock_gen_path.side_effect = gen_path_side_effect + + mock_path_exists.return_value = False # Files do not exist at destination + + updated_context = stage.execute(context) + + assert mock_shutil_copy.call_count == 2 + mock_shutil_copy.assert_any_call(Path(processed_details[proc_id_1]['temp_processed_file']), final_path_proc1) + mock_shutil_copy.assert_any_call(Path(merged_details[merged_id_1]['temp_merged_file']), final_path_merged1) + + # Check mkdir calls + # It should be called for each unique parent directory + expected_mkdir_calls = [ + mock.call(Path("/fake/output_final/OutputOrgAsset/Diffuse"), parents=True, exist_ok=True), + mock.call(Path("/fake/output_final/OutputOrgAsset/ORM"), parents=True, exist_ok=True) + ] + mock_mkdir.assert_has_calls(expected_mkdir_calls, any_order=True) + # Ensure mkdir was called for the parent of each file + assert mock_mkdir.call_count >= 1 # Could be 1 or 2 if paths share a base that's created once + + assert len(updated_context.asset_metadata['final_output_files']) == 2 + assert str(final_path_proc1) in updated_context.asset_metadata['final_output_files'] + assert str(final_path_merged1) in updated_context.asset_metadata['final_output_files'] + + assert updated_context.processed_maps_details[proc_id_1]['final_output_path'] == str(final_path_proc1) + assert updated_context.merged_maps_details[merged_id_1]['final_output_path'] == str(final_path_merged1) + mock_log_error.assert_not_called() + # Check for specific info logs if necessary + # mock_log_info.assert_any_call(f"Copying {processed_details[proc_id_1]['temp_processed_file']} to {final_path_proc1}") + # mock_log_info.assert_any_call(f"Copying {merged_details[merged_id_1]['temp_merged_file']} to {final_path_merged1}") +@mock.patch('shutil.copy2') +@mock.patch('pathlib.Path.mkdir') # Still might be called if other files are processed +@mock.patch('pathlib.Path.exists') +@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern') +@mock.patch('logging.info') +def test_output_organization_overwrite_disabled_file_exists( + mock_log_info, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy +): + stage = OutputOrganizationStage() + proc_id_1 = uuid.uuid4().hex + processed_details = { + proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_exists.png', 'map_type': 'Diffuse', 'output_filename': 'OutputOrgAsset_Diffuse_Exists.png'} + } + context = create_output_org_mock_context( + processed_map_details=processed_details, + overwrite_setting=False + ) + + final_path_proc1 = Path("/fake/output_final/OutputOrgAsset/Diffuse/OutputOrgAsset_Diffuse_Exists.png") + mock_gen_path.return_value = final_path_proc1 # Only one file + mock_path_exists.return_value = True # File exists at destination + + updated_context = stage.execute(context) + + mock_shutil_copy.assert_not_called() + mock_log_info.assert_any_call( + f"Skipping copy for {final_path_proc1} as it already exists and overwrite is disabled." + ) + # final_output_files should still be populated if the file exists and is considered "organized" + assert str(final_path_proc1) in updated_context.asset_metadata['final_output_files'] + assert updated_context.processed_maps_details[proc_id_1]['final_output_path'] == str(final_path_proc1) + + +@mock.patch('shutil.copy2') +@mock.patch('pathlib.Path.mkdir') +@mock.patch('pathlib.Path.exists') +@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern') +@mock.patch('logging.info') +@mock.patch('logging.error') +def test_output_organization_overwrite_enabled_file_exists( + mock_log_error, mock_log_info, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy +): + stage = OutputOrganizationStage() + proc_id_1 = uuid.uuid4().hex + processed_details = { + proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_overwrite.png', 'map_type': 'Diffuse', 'output_filename': 'OutputOrgAsset_Diffuse_Overwrite.png'} + } + context = create_output_org_mock_context( + processed_map_details=processed_details, + overwrite_setting=True # Overwrite is enabled + ) + + final_path_proc1 = Path("/fake/output_final/OutputOrgAsset/Diffuse/OutputOrgAsset_Diffuse_Overwrite.png") + mock_gen_path.return_value = final_path_proc1 + mock_path_exists.return_value = True # File exists, but we should overwrite + + updated_context = stage.execute(context) + + mock_shutil_copy.assert_called_once_with(Path(processed_details[proc_id_1]['temp_processed_file']), final_path_proc1) + mock_mkdir.assert_called_once_with(final_path_proc1.parent, parents=True, exist_ok=True) + assert str(final_path_proc1) in updated_context.asset_metadata['final_output_files'] + assert updated_context.processed_maps_details[proc_id_1]['final_output_path'] == str(final_path_proc1) + mock_log_error.assert_not_called() + # Optionally check for a log message indicating overwrite, if implemented + # mock_log_info.assert_any_call(f"Overwriting existing file {final_path_proc1}...") + + +@mock.patch('shutil.copy2') +@mock.patch('pathlib.Path.mkdir') +@mock.patch('pathlib.Path.exists') +@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern') +@mock.patch('logging.error') +def test_output_organization_only_processed_maps( + mock_log_error, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy +): + stage = OutputOrganizationStage() + proc_id_1 = uuid.uuid4().hex + processed_details = { + proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_only.png', 'map_type': 'Albedo', 'output_filename': 'OutputOrgAsset_Albedo.png'} + } + context = create_output_org_mock_context( + processed_map_details=processed_details, + merged_map_details={}, # No merged maps + overwrite_setting=False + ) + + final_path_proc1 = Path("/fake/output_final/OutputOrgAsset/Albedo/OutputOrgAsset_Albedo.png") + mock_gen_path.return_value = final_path_proc1 + mock_path_exists.return_value = False + + updated_context = stage.execute(context) + + mock_shutil_copy.assert_called_once_with(Path(processed_details[proc_id_1]['temp_processed_file']), final_path_proc1) + mock_mkdir.assert_called_once_with(final_path_proc1.parent, parents=True, exist_ok=True) + assert len(updated_context.asset_metadata['final_output_files']) == 1 + assert str(final_path_proc1) in updated_context.asset_metadata['final_output_files'] + assert updated_context.processed_maps_details[proc_id_1]['final_output_path'] == str(final_path_proc1) + assert not updated_context.merged_maps_details # Should remain empty + mock_log_error.assert_not_called() + +@mock.patch('shutil.copy2') +@mock.patch('pathlib.Path.mkdir') +@mock.patch('pathlib.Path.exists') +@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern') +@mock.patch('logging.error') +def test_output_organization_only_merged_maps( + mock_log_error, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy +): + stage = OutputOrganizationStage() + merged_id_1 = uuid.uuid4().hex + merged_details = { + merged_id_1: {'status': 'Processed', 'temp_merged_file': '/fake/temp_engine_dir/merged_only.png', 'map_type': 'Metallic', 'output_filename': 'OutputOrgAsset_Metallic.png'} + } + context = create_output_org_mock_context( + processed_map_details={}, # No processed maps + merged_map_details=merged_details, + overwrite_setting=False + ) + + final_path_merged1 = Path("/fake/output_final/OutputOrgAsset/Metallic/OutputOrgAsset_Metallic.png") + mock_gen_path.return_value = final_path_merged1 + mock_path_exists.return_value = False + + updated_context = stage.execute(context) + + mock_shutil_copy.assert_called_once_with(Path(merged_details[merged_id_1]['temp_merged_file']), final_path_merged1) + mock_mkdir.assert_called_once_with(final_path_merged1.parent, parents=True, exist_ok=True) + assert len(updated_context.asset_metadata['final_output_files']) == 1 + assert str(final_path_merged1) in updated_context.asset_metadata['final_output_files'] + assert updated_context.merged_maps_details[merged_id_1]['final_output_path'] == str(final_path_merged1) + assert not updated_context.processed_maps_details # Should remain empty + mock_log_error.assert_not_called() + +@mock.patch('shutil.copy2') +@mock.patch('pathlib.Path.mkdir') +@mock.patch('pathlib.Path.exists') +@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern') +@mock.patch('logging.warning') # Expect a warning for skipped map +@mock.patch('logging.error') +def test_output_organization_map_status_not_processed( + mock_log_error, mock_log_warning, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy +): + stage = OutputOrganizationStage() + + proc_id_1_failed = uuid.uuid4().hex + proc_id_2_ok = uuid.uuid4().hex + + processed_details = { + proc_id_1_failed: {'status': 'Failed', 'temp_processed_file': '/fake/temp_engine_dir/proc_failed.png', 'map_type': 'Diffuse', 'output_filename': 'OutputOrgAsset_Diffuse_Failed.png'}, + proc_id_2_ok: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_ok.png', 'map_type': 'Normal', 'output_filename': 'OutputOrgAsset_Normal_OK.png'} + } + context = create_output_org_mock_context( + processed_map_details=processed_details, + overwrite_setting=False + ) + + final_path_proc_ok = Path("/fake/output_final/OutputOrgAsset/Normal/OutputOrgAsset_Normal_OK.png") + # generate_path_from_pattern should only be called for the 'Processed' map + mock_gen_path.return_value = final_path_proc_ok + mock_path_exists.return_value = False + + updated_context = stage.execute(context) + + # Assert copy was only called for the 'Processed' map + mock_shutil_copy.assert_called_once_with(Path(processed_details[proc_id_2_ok]['temp_processed_file']), final_path_proc_ok) + mock_mkdir.assert_called_once_with(final_path_proc_ok.parent, parents=True, exist_ok=True) + + # Assert final_output_files only contains the successfully processed map + assert len(updated_context.asset_metadata['final_output_files']) == 1 + assert str(final_path_proc_ok) in updated_context.asset_metadata['final_output_files'] + + # Assert final_output_path is set for the processed map + assert updated_context.processed_maps_details[proc_id_2_ok]['final_output_path'] == str(final_path_proc_ok) + # Assert final_output_path is NOT set for the failed map + assert 'final_output_path' not in updated_context.processed_maps_details[proc_id_1_failed] + + mock_log_warning.assert_any_call( + f"Skipping output organization for map with ID {proc_id_1_failed} (type: Diffuse) as its status is 'Failed'." + ) + mock_log_error.assert_not_called() +@mock.patch('shutil.copy2') +@mock.patch('pathlib.Path.mkdir') +@mock.patch('pathlib.Path.exists') +@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern') +@mock.patch('logging.error') +def test_output_organization_generate_path_fails( + mock_log_error, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy +): + stage = OutputOrganizationStage() + proc_id_1 = uuid.uuid4().hex + processed_details = { + proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_path_fail.png', 'map_type': 'Roughness', 'output_filename': 'OutputOrgAsset_Roughness_PathFail.png'} + } + context = create_output_org_mock_context( + processed_map_details=processed_details, + overwrite_setting=False + ) + + mock_gen_path.side_effect = Exception("Simulated path generation error") + mock_path_exists.return_value = False # Should not matter if path gen fails + + updated_context = stage.execute(context) + + mock_shutil_copy.assert_not_called() # No copy if path generation fails + mock_mkdir.assert_not_called() # No mkdir if path generation fails + + assert not updated_context.asset_metadata.get('final_output_files') # No files should be listed + assert 'final_output_path' not in updated_context.processed_maps_details[proc_id_1] + + assert updated_context.status_flags.get('output_organization_error') is True + assert updated_context.asset_metadata['status'] == "Error" # Or "Failed" depending on desired behavior + + mock_log_error.assert_any_call( + f"Error generating output path for map ID {proc_id_1} (type: Roughness): Simulated path generation error" + ) + +@mock.patch('shutil.copy2') +@mock.patch('pathlib.Path.mkdir') +@mock.patch('pathlib.Path.exists') +@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern') +@mock.patch('logging.error') +def test_output_organization_shutil_copy_fails( + mock_log_error, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy +): + stage = OutputOrganizationStage() + proc_id_1 = uuid.uuid4().hex + processed_details = { + proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_copy_fail.png', 'map_type': 'AO', 'output_filename': 'OutputOrgAsset_AO_CopyFail.png'} + } + context = create_output_org_mock_context( + processed_map_details=processed_details, + overwrite_setting=False + ) + + final_path_proc1 = Path("/fake/output_final/OutputOrgAsset/AO/OutputOrgAsset_AO_CopyFail.png") + mock_gen_path.return_value = final_path_proc1 + mock_path_exists.return_value = False + mock_shutil_copy.side_effect = shutil.Error("Simulated copy error") # Can also be IOError, OSError + + updated_context = stage.execute(context) + + mock_mkdir.assert_called_once_with(final_path_proc1.parent, parents=True, exist_ok=True) # mkdir would be called before copy + mock_shutil_copy.assert_called_once_with(Path(processed_details[proc_id_1]['temp_processed_file']), final_path_proc1) + + # Even if copy fails, the path might be added to final_output_files before the error is caught, + # or the design might be to not add it. Let's assume it's not added on error. + # Check the stage's actual behavior for this. + # If the intention is to record the *attempted* path, this assertion might change. + # For now, assume failure means it's not a "final" output. + assert not updated_context.asset_metadata.get('final_output_files') + assert 'final_output_path' not in updated_context.processed_maps_details[proc_id_1] # Or it might contain the path but status is error + + assert updated_context.status_flags.get('output_organization_error') is True + assert updated_context.asset_metadata['status'] == "Error" # Or "Failed" + + mock_log_error.assert_any_call( + f"Error copying file {processed_details[proc_id_1]['temp_processed_file']} to {final_path_proc1}: Simulated copy error" + ) \ No newline at end of file diff --git a/tests/processing/pipeline/stages/test_supplier_determination.py b/tests/processing/pipeline/stages/test_supplier_determination.py new file mode 100644 index 0000000..a1613b1 --- /dev/null +++ b/tests/processing/pipeline/stages/test_supplier_determination.py @@ -0,0 +1,213 @@ +import pytest +from unittest import mock +from pathlib import Path +from typing import Dict, List, Optional, Any + +# Assuming pytest is run from project root, adjust if necessary +from processing.pipeline.stages.supplier_determination import SupplierDeterminationStage +from processing.pipeline.asset_context import AssetProcessingContext +from rule_structure import AssetRule, SourceRule, FileRule # For constructing mock context +from configuration import Configuration, GeneralSettings, Supplier # For mock config + +# Example helper (can be a pytest fixture too) +def create_mock_context( + asset_rule_supplier_override: Optional[str] = None, + source_rule_supplier: Optional[str] = None, + config_suppliers: Optional[Dict[str, Any]] = None, # Mocked Supplier objects or dicts + asset_name: str = "TestAsset" +) -> AssetProcessingContext: + mock_asset_rule = mock.MagicMock(spec=AssetRule) + mock_asset_rule.name = asset_name + mock_asset_rule.supplier_override = asset_rule_supplier_override + # ... other AssetRule fields if needed by the stage ... + + mock_source_rule = mock.MagicMock(spec=SourceRule) + mock_source_rule.supplier = source_rule_supplier + # ... other SourceRule fields ... + + mock_config = mock.MagicMock(spec=Configuration) + mock_config.suppliers = config_suppliers if config_suppliers is not None else {} + + # Basic AssetProcessingContext fields + context = AssetProcessingContext( + source_rule=mock_source_rule, + asset_rule=mock_asset_rule, + workspace_path=Path("/fake/workspace"), + engine_temp_dir=Path("/fake/temp"), + output_base_path=Path("/fake/output"), + effective_supplier=None, + asset_metadata={}, + processed_maps_details={}, + merged_maps_details={}, + files_to_process=[], + loaded_data_cache={}, + config_obj=mock_config, + status_flags={}, + incrementing_value=None, + sha5_value=None # Corrected from sha5_value to sha256_value if that's the actual field name + ) + return context + +@pytest.fixture +def supplier_stage(): + return SupplierDeterminationStage() + +@mock.patch('logging.error') +@mock.patch('logging.info') +def test_supplier_from_asset_rule_override_valid(mock_log_info, mock_log_error, supplier_stage): + mock_suppliers_config = {"SupplierA": mock.MagicMock(spec=Supplier)} + context = create_mock_context( + asset_rule_supplier_override="SupplierA", + config_suppliers=mock_suppliers_config + ) + + updated_context = supplier_stage.execute(context) + + assert updated_context.effective_supplier == "SupplierA" + assert not updated_context.status_flags.get('supplier_error') + mock_log_info.assert_any_call("Effective supplier for asset 'TestAsset' set to 'SupplierA' from asset rule override.") + mock_log_error.assert_not_called() + +@mock.patch('logging.error') +@mock.patch('logging.info') +def test_supplier_from_source_rule_fallback_valid(mock_log_info, mock_log_error, supplier_stage): + mock_suppliers_config = {"SupplierB": mock.MagicMock(spec=Supplier)} + context = create_mock_context( + asset_rule_supplier_override=None, + source_rule_supplier="SupplierB", + config_suppliers=mock_suppliers_config + ) + + updated_context = supplier_stage.execute(context) + + assert updated_context.effective_supplier == "SupplierB" + assert not updated_context.status_flags.get('supplier_error') + mock_log_info.assert_any_call("Effective supplier for asset 'TestAsset' set to 'SupplierB' from source rule.") + mock_log_error.assert_not_called() + +@mock.patch('logging.error') +@mock.patch('logging.warning') # supplier_determination uses logging.warning for invalid suppliers +def test_asset_rule_override_invalid_supplier(mock_log_warning, mock_log_error, supplier_stage): + context = create_mock_context( + asset_rule_supplier_override="InvalidSupplier", + config_suppliers={"SupplierA": mock.MagicMock(spec=Supplier)} # "InvalidSupplier" not in config + ) + + updated_context = supplier_stage.execute(context) + + assert updated_context.effective_supplier is None + assert updated_context.status_flags.get('supplier_error') is True + mock_log_warning.assert_any_call( + "Asset 'TestAsset' has supplier_override 'InvalidSupplier' which is not defined in global suppliers. No supplier set." + ) + mock_log_error.assert_not_called() + + +@mock.patch('logging.error') +@mock.patch('logging.warning') +def test_source_rule_fallback_invalid_supplier(mock_log_warning, mock_log_error, supplier_stage): + context = create_mock_context( + asset_rule_supplier_override=None, + source_rule_supplier="InvalidSupplierB", + config_suppliers={"SupplierA": mock.MagicMock(spec=Supplier)} # "InvalidSupplierB" not in config + ) + + updated_context = supplier_stage.execute(context) + + assert updated_context.effective_supplier is None + assert updated_context.status_flags.get('supplier_error') is True + mock_log_warning.assert_any_call( + "Asset 'TestAsset' has source rule supplier 'InvalidSupplierB' which is not defined in global suppliers. No supplier set." + ) + mock_log_error.assert_not_called() + +@mock.patch('logging.error') +@mock.patch('logging.warning') +def test_no_supplier_defined(mock_log_warning, mock_log_error, supplier_stage): + context = create_mock_context( + asset_rule_supplier_override=None, + source_rule_supplier=None, + config_suppliers={"SupplierA": mock.MagicMock(spec=Supplier)} + ) + + updated_context = supplier_stage.execute(context) + + assert updated_context.effective_supplier is None + assert updated_context.status_flags.get('supplier_error') is True + mock_log_warning.assert_any_call( + "No supplier could be determined for asset 'TestAsset'. " + "AssetRule override is None and SourceRule supplier is None or empty." + ) + mock_log_error.assert_not_called() + +@mock.patch('logging.error') +@mock.patch('logging.warning') +def test_empty_config_suppliers_with_asset_override(mock_log_warning, mock_log_error, supplier_stage): + context = create_mock_context( + asset_rule_supplier_override="SupplierX", + config_suppliers={} # Empty global supplier config + ) + + updated_context = supplier_stage.execute(context) + + assert updated_context.effective_supplier is None + assert updated_context.status_flags.get('supplier_error') is True + mock_log_warning.assert_any_call( + "Asset 'TestAsset' has supplier_override 'SupplierX' which is not defined in global suppliers. No supplier set." + ) + mock_log_error.assert_not_called() + +@mock.patch('logging.error') +@mock.patch('logging.warning') +def test_empty_config_suppliers_with_source_rule(mock_log_warning, mock_log_error, supplier_stage): + context = create_mock_context( + source_rule_supplier="SupplierY", + config_suppliers={} # Empty global supplier config + ) + + updated_context = supplier_stage.execute(context) + + assert updated_context.effective_supplier is None + assert updated_context.status_flags.get('supplier_error') is True + mock_log_warning.assert_any_call( + "Asset 'TestAsset' has source rule supplier 'SupplierY' which is not defined in global suppliers. No supplier set." + ) + mock_log_error.assert_not_called() + +@mock.patch('logging.error') +@mock.patch('logging.info') +def test_asset_rule_override_empty_string(mock_log_info, mock_log_error, supplier_stage): + # This scenario should fall back to source_rule.supplier if asset_rule.supplier_override is "" + mock_suppliers_config = {"SupplierB": mock.MagicMock(spec=Supplier)} + context = create_mock_context( + asset_rule_supplier_override="", # Empty string override + source_rule_supplier="SupplierB", + config_suppliers=mock_suppliers_config + ) + + updated_context = supplier_stage.execute(context) + + assert updated_context.effective_supplier == "SupplierB" # Falls back to SourceRule + assert not updated_context.status_flags.get('supplier_error') + mock_log_info.assert_any_call("Effective supplier for asset 'TestAsset' set to 'SupplierB' from source rule.") + mock_log_error.assert_not_called() + +@mock.patch('logging.error') +@mock.patch('logging.warning') +def test_source_rule_supplier_empty_string(mock_log_warning, mock_log_error, supplier_stage): + # This scenario should result in an error if asset_rule.supplier_override is None and source_rule.supplier is "" + context = create_mock_context( + asset_rule_supplier_override=None, + source_rule_supplier="", # Empty string source supplier + config_suppliers={"SupplierA": mock.MagicMock(spec=Supplier)} + ) + + updated_context = supplier_stage.execute(context) + + assert updated_context.effective_supplier is None + assert updated_context.status_flags.get('supplier_error') is True + mock_log_warning.assert_any_call( + "No supplier could be determined for asset 'TestAsset'. " + "AssetRule override is None and SourceRule supplier is None or empty." + ) + mock_log_error.assert_not_called() \ No newline at end of file diff --git a/tests/processing/pipeline/test_orchestrator.py b/tests/processing/pipeline/test_orchestrator.py new file mode 100644 index 0000000..3f52908 --- /dev/null +++ b/tests/processing/pipeline/test_orchestrator.py @@ -0,0 +1,383 @@ +import pytest +from unittest import mock +from pathlib import Path +import uuid +import shutil # For checking rmtree +import tempfile # For mocking mkdtemp + +from processing.pipeline.orchestrator import PipelineOrchestrator +from processing.pipeline.asset_context import AssetProcessingContext +from processing.pipeline.stages.base_stage import ProcessingStage # For mocking stages +from rule_structure import SourceRule, AssetRule, FileRule +from configuration import Configuration, GeneralSettings + +# Mock Stage that modifies context +class MockPassThroughStage(ProcessingStage): + def __init__(self, stage_name="mock_stage"): + self.stage_name = stage_name + self.execute_call_count = 0 + self.contexts_called_with = [] # To store contexts for verification + + def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: + self.execute_call_count += 1 + self.contexts_called_with.append(context) + # Optionally, modify context for testing + context.asset_metadata[f'{self.stage_name}_executed'] = True + if self.stage_name == "skipper_stage": # Example conditional logic + context.status_flags['skip_asset'] = True + context.status_flags['skip_reason'] = "Skipped by skipper_stage" + elif self.stage_name == "error_stage": # Example error-raising stage + raise ValueError("Simulated error in error_stage") + + # Simulate status update based on stage execution + if not context.status_flags.get('skip_asset') and not context.status_flags.get('asset_failed'): + context.asset_metadata['status'] = "Processed" # Default to processed if not skipped/failed + return context + +def create_orchestrator_test_config() -> mock.MagicMock: + mock_config = mock.MagicMock(spec=Configuration) + mock_config.general_settings = mock.MagicMock(spec=GeneralSettings) + mock_config.general_settings.temp_dir_override = None # Default, can be overridden in tests + # Add other config details if orchestrator or stages depend on them directly + return mock_config + +def create_orchestrator_test_asset_rule(name: str, num_file_rules: int = 1) -> mock.MagicMock: + asset_rule = mock.MagicMock(spec=AssetRule) + asset_rule.name = name + asset_rule.id = uuid.uuid4() + asset_rule.source_path = Path(f"/fake/source/{name}") # Using Path object + asset_rule.file_rules = [mock.MagicMock(spec=FileRule) for _ in range(num_file_rules)] + asset_rule.enabled = True + asset_rule.map_types = {} # Initialize as dict + asset_rule.material_name_scheme = "{asset_name}" + asset_rule.texture_name_scheme = "{asset_name}_{map_type}" + asset_rule.output_path_scheme = "{source_name}/{asset_name}" + # ... other necessary AssetRule fields ... + return asset_rule + +def create_orchestrator_test_source_rule(name: str, num_assets: int = 1, asset_names: list = None) -> mock.MagicMock: + source_rule = mock.MagicMock(spec=SourceRule) + source_rule.name = name + source_rule.id = uuid.uuid4() + if asset_names: + source_rule.assets = [create_orchestrator_test_asset_rule(an) for an in asset_names] + else: + source_rule.assets = [create_orchestrator_test_asset_rule(f"Asset_{i+1}_in_{name}") for i in range(num_assets)] + source_rule.enabled = True + source_rule.source_path = Path(f"/fake/source_root/{name}") # Using Path object + # ... other necessary SourceRule fields ... + return source_rule + +# --- Test Cases for PipelineOrchestrator.process_source_rule() --- + +@mock.patch('shutil.rmtree') +@mock.patch('tempfile.mkdtemp') +def test_orchestrator_basic_flow_mock_stages(mock_mkdtemp, mock_rmtree): + mock_mkdtemp.return_value = "/fake/engine_temp_dir_path" # Path for mkdtemp + + config = create_orchestrator_test_config() + stage1 = MockPassThroughStage("stage1") + stage2 = MockPassThroughStage("stage2") + orchestrator = PipelineOrchestrator(config_obj=config, stages=[stage1, stage2]) + + source_rule = create_orchestrator_test_source_rule("MySourceRule", num_assets=2) + asset1_name = source_rule.assets[0].name + asset2_name = source_rule.assets[1].name + + # Mock asset_metadata to be updated by stages for status check + # The MockPassThroughStage already sets a 'status' = "Processed" if not skipped/failed + # and adds '{stage_name}_executed' = True to asset_metadata. + + results = orchestrator.process_source_rule( + source_rule, Path("/ws"), Path("/out"), False, "inc_val_123", "sha_val_abc" + ) + + assert stage1.execute_call_count == 2 # Called for each asset + assert stage2.execute_call_count == 2 # Called for each asset + + assert asset1_name in results['processed'] + assert asset2_name in results['processed'] + assert not results['skipped'] + assert not results['failed'] + + # Verify context modifications by stages + for i in range(2): # For each asset + # Stage 1 context checks + s1_context_asset = stage1.contexts_called_with[i] + assert s1_context_asset.asset_metadata.get('stage1_executed') is True + assert s1_context_asset.asset_metadata.get('stage2_executed') is None # Stage 2 not yet run for this asset + + # Stage 2 context checks + s2_context_asset = stage2.contexts_called_with[i] + assert s2_context_asset.asset_metadata.get('stage1_executed') is True # From stage 1 + assert s2_context_asset.asset_metadata.get('stage2_executed') is True + assert s2_context_asset.asset_metadata.get('status') == "Processed" + + mock_mkdtemp.assert_called_once() + # The orchestrator creates a subdirectory within the mkdtemp path + expected_temp_path = Path(mock_mkdtemp.return_value) / source_rule.id.hex + mock_rmtree.assert_called_once_with(expected_temp_path, ignore_errors=True) + +@mock.patch('shutil.rmtree') +@mock.patch('tempfile.mkdtemp') +def test_orchestrator_asset_skipping_by_stage(mock_mkdtemp, mock_rmtree): + mock_mkdtemp.return_value = "/fake/engine_temp_dir_path_skip" + + config = create_orchestrator_test_config() + skipper_stage = MockPassThroughStage("skipper_stage") # This stage will set skip_asset = True + stage_after_skip = MockPassThroughStage("stage_after_skip") + + orchestrator = PipelineOrchestrator(config_obj=config, stages=[skipper_stage, stage_after_skip]) + + source_rule = create_orchestrator_test_source_rule("SkipSourceRule", num_assets=1) + asset_to_skip_name = source_rule.assets[0].name + + results = orchestrator.process_source_rule( + source_rule, Path("/ws_skip"), Path("/out_skip"), False, "inc_skip", "sha_skip" + ) + + assert skipper_stage.execute_call_count == 1 # Called for the asset + assert stage_after_skip.execute_call_count == 0 # Not called because asset was skipped + + assert asset_to_skip_name in results['skipped'] + assert not results['processed'] + assert not results['failed'] + + # Verify skip reason in context if needed (MockPassThroughStage stores contexts) + skipped_context = skipper_stage.contexts_called_with[0] + assert skipped_context.status_flags['skip_asset'] is True + assert skipped_context.status_flags['skip_reason'] == "Skipped by skipper_stage" + + mock_mkdtemp.assert_called_once() + expected_temp_path = Path(mock_mkdtemp.return_value) / source_rule.id.hex + mock_rmtree.assert_called_once_with(expected_temp_path, ignore_errors=True) + +@mock.patch('shutil.rmtree') +@mock.patch('tempfile.mkdtemp') +def test_orchestrator_no_assets_in_source_rule(mock_mkdtemp, mock_rmtree): + mock_mkdtemp.return_value = "/fake/engine_temp_dir_no_assets" + + config = create_orchestrator_test_config() + stage1 = MockPassThroughStage("stage1_no_assets") + orchestrator = PipelineOrchestrator(config_obj=config, stages=[stage1]) + + source_rule = create_orchestrator_test_source_rule("NoAssetSourceRule", num_assets=0) + + results = orchestrator.process_source_rule( + source_rule, Path("/ws_no_assets"), Path("/out_no_assets"), False, "inc_no", "sha_no" + ) + + assert stage1.execute_call_count == 0 + assert not results['processed'] + assert not results['skipped'] + assert not results['failed'] + + # mkdtemp should still be called for the source rule processing, even if no assets + mock_mkdtemp.assert_called_once() + expected_temp_path = Path(mock_mkdtemp.return_value) / source_rule.id.hex + mock_rmtree.assert_called_once_with(expected_temp_path, ignore_errors=True) + + +@mock.patch('shutil.rmtree') +@mock.patch('tempfile.mkdtemp') +def test_orchestrator_error_during_stage_execution(mock_mkdtemp, mock_rmtree): + mock_mkdtemp.return_value = "/fake/engine_temp_dir_error" + + config = create_orchestrator_test_config() + error_stage = MockPassThroughStage("error_stage") # This stage will raise an error + stage_after_error = MockPassThroughStage("stage_after_error") + + orchestrator = PipelineOrchestrator(config_obj=config, stages=[error_stage, stage_after_error]) + + # Test with two assets, one fails, one processes (if orchestrator continues) + # The current orchestrator's process_asset is per asset, so an error in one + # should not stop processing of other assets in the same source_rule. + source_rule = create_orchestrator_test_source_rule("ErrorSourceRule", asset_names=["AssetFails", "AssetSucceeds"]) + asset_fails_name = source_rule.assets[0].name + asset_succeeds_name = source_rule.assets[1].name + + # Make only the first asset's processing trigger the error + original_execute = error_stage.execute + def error_execute_side_effect(context: AssetProcessingContext): + if context.asset_rule.name == asset_fails_name: + # The MockPassThroughStage is already configured to raise ValueError for "error_stage" + # but we need to ensure it's only for the first asset. + # We can achieve this by modifying the stage_name temporarily or by checking asset_rule.name + # For simplicity, let's assume the mock stage's error logic is fine, + # and we just need to check the outcome. + # The error_stage will raise ValueError("Simulated error in error_stage") + # The orchestrator's _process_single_asset catches generic Exception. + return original_execute(context) # This will call the erroring logic + else: + # For the second asset, make it pass through without error + context.asset_metadata[f'{error_stage.stage_name}_executed'] = True + context.asset_metadata['status'] = "Processed" + return context + + error_stage.execute = mock.MagicMock(side_effect=error_execute_side_effect) + # stage_after_error should still be called for the successful asset + + results = orchestrator.process_source_rule( + source_rule, Path("/ws_error"), Path("/out_error"), False, "inc_err", "sha_err" + ) + + assert error_stage.execute.call_count == 2 # Called for both assets + # stage_after_error is only called for the asset that didn't fail in error_stage + assert stage_after_error.execute_call_count == 1 + + assert asset_fails_name in results['failed'] + assert asset_succeeds_name in results['processed'] + assert not results['skipped'] + + # Verify the context of the failed asset + failed_context = None + for ctx in error_stage.contexts_called_with: + if ctx.asset_rule.name == asset_fails_name: + failed_context = ctx + break + assert failed_context is not None + assert failed_context.status_flags['asset_failed'] is True + assert "Simulated error in error_stage" in failed_context.status_flags['failure_reason'] + + # Verify the context of the successful asset after stage_after_error + successful_context_after_s2 = None + for ctx in stage_after_error.contexts_called_with: + if ctx.asset_rule.name == asset_succeeds_name: + successful_context_after_s2 = ctx + break + assert successful_context_after_s2 is not None + assert successful_context_after_s2.asset_metadata.get('error_stage_executed') is True # from the non-erroring path + assert successful_context_after_s2.asset_metadata.get('stage_after_error_executed') is True + assert successful_context_after_s2.asset_metadata.get('status') == "Processed" + + + mock_mkdtemp.assert_called_once() + expected_temp_path = Path(mock_mkdtemp.return_value) / source_rule.id.hex + mock_rmtree.assert_called_once_with(expected_temp_path, ignore_errors=True) + + +@mock.patch('shutil.rmtree') +@mock.patch('tempfile.mkdtemp') +def test_orchestrator_asset_processing_context_initialization(mock_mkdtemp, mock_rmtree): + mock_engine_temp_dir = "/fake/engine_temp_dir_context_init" + mock_mkdtemp.return_value = mock_engine_temp_dir + + config = create_orchestrator_test_config() + mock_stage = MockPassThroughStage("context_check_stage") + orchestrator = PipelineOrchestrator(config_obj=config, stages=[mock_stage]) + + source_rule = create_orchestrator_test_source_rule("ContextSourceRule", num_assets=1) + asset_rule = source_rule.assets[0] + + workspace_path = Path("/ws_context") + output_base_path = Path("/out_context") + incrementing_value = "inc_context_123" + sha5_value = "sha_context_abc" + + orchestrator.process_source_rule( + source_rule, workspace_path, output_base_path, False, incrementing_value, sha5_value + ) + + assert mock_stage.execute_call_count == 1 + + # Retrieve the context passed to the mock stage + captured_context = mock_stage.contexts_called_with[0] + + assert captured_context.source_rule == source_rule + assert captured_context.asset_rule == asset_rule + assert captured_context.workspace_path == workspace_path + + # engine_temp_dir for the asset is a sub-directory of the source_rule's temp dir + # which itself is a sub-directory of the main engine_temp_dir from mkdtemp + expected_source_rule_temp_dir = Path(mock_engine_temp_dir) / source_rule.id.hex + expected_asset_temp_dir = expected_source_rule_temp_dir / asset_rule.id.hex + assert captured_context.engine_temp_dir == expected_asset_temp_dir + + assert captured_context.output_base_path == output_base_path + assert captured_context.config_obj == config + assert captured_context.incrementing_value == incrementing_value + assert captured_context.sha5_value == sha5_value + + # Check initial state of other context fields + assert captured_context.asset_metadata == {} # Should be empty initially for an asset + assert captured_context.status_flags == {} # Should be empty initially + assert captured_context.shared_data == {} # Should be empty initially + assert captured_context.current_files == [] # Should be empty initially + + mock_mkdtemp.assert_called_once() + mock_rmtree.assert_called_once_with(expected_source_rule_temp_dir, ignore_errors=True) + +@mock.patch('shutil.rmtree') +@mock.patch('tempfile.mkdtemp') +def test_orchestrator_temp_dir_override_from_config(mock_mkdtemp, mock_rmtree): + # This test verifies that if config.general_settings.temp_dir_override is set, + # mkdtemp is NOT called, and the override path is used and cleaned up. + + config = create_orchestrator_test_config() + override_temp_path_str = "/override/temp/path" + config.general_settings.temp_dir_override = override_temp_path_str + + stage1 = MockPassThroughStage("stage_temp_override") + orchestrator = PipelineOrchestrator(config_obj=config, stages=[stage1]) + + source_rule = create_orchestrator_test_source_rule("TempOverrideRule", num_assets=1) + asset_rule = source_rule.assets[0] + + results = orchestrator.process_source_rule( + source_rule, Path("/ws_override"), Path("/out_override"), False, "inc_override", "sha_override" + ) + + assert stage1.execute_call_count == 1 + assert asset_rule.name in results['processed'] + + mock_mkdtemp.assert_not_called() # mkdtemp should not be called due to override + + # The orchestrator should create its source-rule specific subdir within the override + expected_source_rule_temp_dir_in_override = Path(override_temp_path_str) / source_rule.id.hex + + # Verify the context passed to the stage uses the overridden path structure + captured_context = stage1.contexts_called_with[0] + expected_asset_temp_dir_in_override = expected_source_rule_temp_dir_in_override / asset_rule.id.hex + assert captured_context.engine_temp_dir == expected_asset_temp_dir_in_override + + # rmtree should be called on the source_rule's directory within the override path + mock_rmtree.assert_called_once_with(expected_source_rule_temp_dir_in_override, ignore_errors=True) + +@mock.patch('shutil.rmtree') +@mock.patch('tempfile.mkdtemp') +def test_orchestrator_disabled_asset_rule_is_skipped(mock_mkdtemp, mock_rmtree): + mock_mkdtemp.return_value = "/fake/engine_temp_dir_disabled_asset" + + config = create_orchestrator_test_config() + stage1 = MockPassThroughStage("stage_disabled_check") + orchestrator = PipelineOrchestrator(config_obj=config, stages=[stage1]) + + source_rule = create_orchestrator_test_source_rule("DisabledAssetSourceRule", asset_names=["EnabledAsset", "DisabledAsset"]) + enabled_asset = source_rule.assets[0] + disabled_asset = source_rule.assets[1] + disabled_asset.enabled = False # Disable this asset rule + + results = orchestrator.process_source_rule( + source_rule, Path("/ws_disabled"), Path("/out_disabled"), False, "inc_dis", "sha_dis" + ) + + assert stage1.execute_call_count == 1 # Only called for the enabled asset + + assert enabled_asset.name in results['processed'] + assert disabled_asset.name in results['skipped'] + assert not results['failed'] + + # Verify context for the processed asset + assert stage1.contexts_called_with[0].asset_rule.name == enabled_asset.name + + # Verify skip reason for the disabled asset (this is set by the orchestrator itself) + # The orchestrator's _process_single_asset checks asset_rule.enabled + # We need to inspect the results dictionary for the skip reason if it's stored there, + # or infer it. The current structure of `results` doesn't store detailed skip reasons directly, + # but the test ensures it's in the 'skipped' list. + # For a more detailed check, one might need to adjust how results are reported or mock deeper. + # For now, confirming it's in 'skipped' and stage1 wasn't called for it is sufficient. + + mock_mkdtemp.assert_called_once() + expected_temp_path = Path(mock_mkdtemp.return_value) / source_rule.id.hex + mock_rmtree.assert_called_once_with(expected_temp_path, ignore_errors=True) \ No newline at end of file diff --git a/tests/processing/utils/test_image_processing_utils.py b/tests/processing/utils/test_image_processing_utils.py new file mode 100644 index 0000000..e128b3f --- /dev/null +++ b/tests/processing/utils/test_image_processing_utils.py @@ -0,0 +1,504 @@ +import pytest +from unittest import mock +import numpy as np +from pathlib import Path +import sys + +# Attempt to import the module under test +# This assumes that the 'tests' directory is at the same level as the 'processing' directory, +# and pytest handles the PYTHONPATH correctly. +try: + from processing.utils import image_processing_utils as ipu + import cv2 # Import cv2 here if it's used for constants like cv2.COLOR_BGR2RGB +except ImportError: + # Fallback for environments where PYTHONPATH might not be set up as expected by pytest initially + # This adds the project root to sys.path to find the 'processing' module + # Adjust the number of Path.parent calls if your test structure is deeper or shallower + project_root = Path(__file__).parent.parent.parent.parent + sys.path.insert(0, str(project_root)) + from processing.utils import image_processing_utils as ipu + import cv2 # Import cv2 here as well + +# If cv2 is imported directly in image_processing_utils, you might need to mock it globally for some tests +# For example, at the top of the test file: +# sys.modules['cv2'] = mock.MagicMock() # Basic global mock if needed +# We will use more targeted mocks with @mock.patch where cv2 is used. + +# --- Tests for Mathematical Helpers --- + +def test_is_power_of_two(): + assert ipu.is_power_of_two(1) is True + assert ipu.is_power_of_two(2) is True + assert ipu.is_power_of_two(4) is True + assert ipu.is_power_of_two(16) is True + assert ipu.is_power_of_two(1024) is True + assert ipu.is_power_of_two(0) is False + assert ipu.is_power_of_two(-2) is False + assert ipu.is_power_of_two(3) is False + assert ipu.is_power_of_two(100) is False + +def test_get_nearest_pot(): + assert ipu.get_nearest_pot(1) == 1 + assert ipu.get_nearest_pot(2) == 2 + # Based on current implementation: + # For 3: lower=2, upper=4. (3-2)=1, (4-3)=1. Else branch returns upper_pot. So 4. + assert ipu.get_nearest_pot(3) == 4 + assert ipu.get_nearest_pot(50) == 64 # (50-32)=18, (64-50)=14 -> upper + assert ipu.get_nearest_pot(100) == 128 # (100-64)=36, (128-100)=28 -> upper + assert ipu.get_nearest_pot(256) == 256 + assert ipu.get_nearest_pot(0) == 1 + assert ipu.get_nearest_pot(-10) == 1 + # For 700: value.bit_length() = 10. lower_pot = 1<<(10-1) = 512. upper_pot = 1<<10 = 1024. + # (700-512) = 188. (1024-700) = 324. (188 < 324) is True. Returns lower_pot. So 512. + assert ipu.get_nearest_pot(700) == 512 + assert ipu.get_nearest_pot(6) == 8 # (6-4)=2, (8-6)=2. Returns upper. + assert ipu.get_nearest_pot(5) == 4 # (5-4)=1, (8-5)=3. Returns lower. + + +@pytest.mark.parametrize( + "orig_w, orig_h, target_w, target_h, resize_mode, ensure_pot, allow_upscale, target_max_dim, expected_w, expected_h", + [ + # FIT mode + (1000, 800, 500, None, "fit", False, False, None, 500, 400), # Fit width + (1000, 800, None, 400, "fit", False, False, None, 500, 400), # Fit height + (1000, 800, 500, 500, "fit", False, False, None, 500, 400), # Fit to box (width constrained) + (800, 1000, 500, 500, "fit", False, False, None, 400, 500), # Fit to box (height constrained) + (100, 80, 200, None, "fit", False, False, None, 100, 80), # Fit width, no upscale + (100, 80, 200, None, "fit", False, True, None, 200, 160), # Fit width, allow upscale + (100, 80, 128, None, "fit", True, False, None, 128, 64), # Re-evaluated + (100, 80, 128, None, "fit", True, True, None, 128, 128), # Fit width, ensure_pot, allow upscale (128, 102 -> pot 128, 128) + + # STRETCH mode + (1000, 800, 500, 400, "stretch", False, False, None, 500, 400), + (100, 80, 200, 160, "stretch", False, True, None, 200, 160), # Stretch, allow upscale + (100, 80, 200, 160, "stretch", False, False, None, 100, 80), # Stretch, no upscale + (100, 80, 128, 128, "stretch", True, True, None, 128, 128), # Stretch, ensure_pot, allow upscale + (100, 80, 70, 70, "stretch", True, False, None, 64, 64), # Stretch, ensure_pot, no upscale (70,70 -> pot 64,64) + + # MAX_DIM_POT mode + (1000, 800, None, None, "max_dim_pot", True, False, 512, 512, 512), + (800, 1000, None, None, "max_dim_pot", True, False, 512, 512, 512), + (1920, 1080, None, None, "max_dim_pot", True, False, 1024, 1024, 512), + (100, 100, None, None, "max_dim_pot", True, False, 60, 64, 64), + # Edge cases for calculate_target_dimensions + (0, 0, 512, 512, "fit", False, False, None, 512, 512), + (10, 10, 512, 512, "fit", True, False, None, 8, 8), + (100, 100, 150, 150, "fit", True, False, None, 128, 128), + ] +) +def test_calculate_target_dimensions(orig_w, orig_h, target_w, target_h, resize_mode, ensure_pot, allow_upscale, target_max_dim, expected_w, expected_h): + if resize_mode == "max_dim_pot" and target_max_dim is None: + with pytest.raises(ValueError, match="target_max_dim_for_pot_mode must be provided"): + ipu.calculate_target_dimensions(orig_w, orig_h, target_width=target_w, target_height=target_h, + resize_mode=resize_mode, ensure_pot=ensure_pot, allow_upscale=allow_upscale, + target_max_dim_for_pot_mode=target_max_dim) + elif (resize_mode == "fit" and target_w is None and target_h is None) or \ + (resize_mode == "stretch" and (target_w is None or target_h is None)): + with pytest.raises(ValueError): + ipu.calculate_target_dimensions(orig_w, orig_h, target_width=target_w, target_height=target_h, + resize_mode=resize_mode, ensure_pot=ensure_pot, allow_upscale=allow_upscale, + target_max_dim_for_pot_mode=target_max_dim) + else: + actual_w, actual_h = ipu.calculate_target_dimensions( + orig_w, orig_h, target_width=target_w, target_height=target_h, + resize_mode=resize_mode, ensure_pot=ensure_pot, allow_upscale=allow_upscale, + target_max_dim_for_pot_mode=target_max_dim + ) + assert (actual_w, actual_h) == (expected_w, expected_h), \ + f"Input: ({orig_w},{orig_h}), T=({target_w},{target_h}), M={resize_mode}, POT={ensure_pot}, UPSC={allow_upscale}, TMAX={target_max_dim}" + + +def test_calculate_target_dimensions_invalid_mode(): + with pytest.raises(ValueError, match="Unsupported resize_mode"): + ipu.calculate_target_dimensions(100, 100, 50, 50, resize_mode="invalid_mode") + +@pytest.mark.parametrize( + "ow, oh, rw, rh, expected_str", + [ + (100, 100, 100, 100, "EVEN"), + (100, 100, 200, 200, "EVEN"), + (200, 200, 100, 100, "EVEN"), + (100, 100, 150, 100, "X15Y1"), + (100, 100, 50, 100, "X05Y1"), + (100, 100, 100, 150, "X1Y15"), + (100, 100, 100, 50, "X1Y05"), + (100, 50, 150, 75, "EVEN"), + (100, 50, 150, 50, "X15Y1"), + (100, 50, 100, 75, "X1Y15"), + (100, 50, 120, 60, "EVEN"), + (100, 50, 133, 66, "EVEN"), + (100, 100, 133, 100, "X133Y1"), + (100, 100, 100, 133, "X1Y133"), + (100, 100, 133, 133, "EVEN"), + (100, 100, 67, 100, "X067Y1"), + (100, 100, 100, 67, "X1Y067"), + (100, 100, 67, 67, "EVEN"), + (1920, 1080, 1024, 576, "EVEN"), + (1920, 1080, 1024, 512, "X112Y1"), + (0, 100, 50, 50, "InvalidInput"), + (100, 0, 50, 50, "InvalidInput"), + (100, 100, 0, 50, "InvalidResize"), + (100, 100, 50, 0, "InvalidResize"), + ] +) +def test_normalize_aspect_ratio_change(ow, oh, rw, rh, expected_str): + assert ipu.normalize_aspect_ratio_change(ow, oh, rw, rh) == expected_str + +# --- Tests for Image Manipulation --- + +@mock.patch('cv2.imread') +def test_load_image_success_str_path(mock_cv2_imread): + mock_img_data = np.array([[[1, 2, 3]]], dtype=np.uint8) + mock_cv2_imread.return_value = mock_img_data + + result = ipu.load_image("dummy/path.png") + + mock_cv2_imread.assert_called_once_with("dummy/path.png", cv2.IMREAD_UNCHANGED) + assert np.array_equal(result, mock_img_data) + +@mock.patch('cv2.imread') +def test_load_image_success_path_obj(mock_cv2_imread): + mock_img_data = np.array([[[1, 2, 3]]], dtype=np.uint8) + mock_cv2_imread.return_value = mock_img_data + dummy_path = Path("dummy/path.png") + + result = ipu.load_image(dummy_path) + + mock_cv2_imread.assert_called_once_with(str(dummy_path), cv2.IMREAD_UNCHANGED) + assert np.array_equal(result, mock_img_data) + +@mock.patch('cv2.imread') +def test_load_image_failure(mock_cv2_imread): + mock_cv2_imread.return_value = None + + result = ipu.load_image("dummy/path.png") + + mock_cv2_imread.assert_called_once_with("dummy/path.png", cv2.IMREAD_UNCHANGED) + assert result is None + +@mock.patch('cv2.imread', side_effect=Exception("CV2 Read Error")) +def test_load_image_exception(mock_cv2_imread): + result = ipu.load_image("dummy/path.png") + mock_cv2_imread.assert_called_once_with("dummy/path.png", cv2.IMREAD_UNCHANGED) + assert result is None + + +@mock.patch('cv2.cvtColor') +def test_convert_bgr_to_rgb_3_channel(mock_cv2_cvtcolor): + bgr_image = np.random.randint(0, 255, (10, 10, 3), dtype=np.uint8) + rgb_image_mock = np.random.randint(0, 255, (10, 10, 3), dtype=np.uint8) + mock_cv2_cvtcolor.return_value = rgb_image_mock + + result = ipu.convert_bgr_to_rgb(bgr_image) + + mock_cv2_cvtcolor.assert_called_once_with(bgr_image, cv2.COLOR_BGR2RGB) + assert np.array_equal(result, rgb_image_mock) + +@mock.patch('cv2.cvtColor') +def test_convert_bgr_to_rgb_4_channel_bgra(mock_cv2_cvtcolor): + bgra_image = np.random.randint(0, 255, (10, 10, 4), dtype=np.uint8) + rgb_image_mock = np.random.randint(0, 255, (10, 10, 3), dtype=np.uint8) # cvtColor BGRA2RGB drops alpha + mock_cv2_cvtcolor.return_value = rgb_image_mock # Mocking the output of BGRA2RGB + + result = ipu.convert_bgr_to_rgb(bgra_image) + + mock_cv2_cvtcolor.assert_called_once_with(bgra_image, cv2.COLOR_BGRA2RGB) + assert np.array_equal(result, rgb_image_mock) + + +def test_convert_bgr_to_rgb_none_input(): + assert ipu.convert_bgr_to_rgb(None) is None + +def test_convert_bgr_to_rgb_grayscale_input(): + gray_image = np.random.randint(0, 255, (10, 10), dtype=np.uint8) + result = ipu.convert_bgr_to_rgb(gray_image) + assert np.array_equal(result, gray_image) # Should return as is + +@mock.patch('cv2.cvtColor') +def test_convert_rgb_to_bgr_3_channel(mock_cv2_cvtcolor): + rgb_image = np.random.randint(0, 255, (10, 10, 3), dtype=np.uint8) + bgr_image_mock = np.random.randint(0, 255, (10, 10, 3), dtype=np.uint8) + mock_cv2_cvtcolor.return_value = bgr_image_mock + + result = ipu.convert_rgb_to_bgr(rgb_image) + + mock_cv2_cvtcolor.assert_called_once_with(rgb_image, cv2.COLOR_RGB2BGR) + assert np.array_equal(result, bgr_image_mock) + +def test_convert_rgb_to_bgr_none_input(): + assert ipu.convert_rgb_to_bgr(None) is None + +def test_convert_rgb_to_bgr_grayscale_input(): + gray_image = np.random.randint(0, 255, (10, 10), dtype=np.uint8) + result = ipu.convert_rgb_to_bgr(gray_image) + assert np.array_equal(result, gray_image) # Should return as is + +def test_convert_rgb_to_bgr_4_channel_input(): + rgba_image = np.random.randint(0, 255, (10, 10, 4), dtype=np.uint8) + result = ipu.convert_rgb_to_bgr(rgba_image) + assert np.array_equal(result, rgba_image) # Should return as is + + +@mock.patch('cv2.resize') +def test_resize_image_downscale(mock_cv2_resize): + original_image = np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8) + resized_image_mock = np.random.randint(0, 255, (50, 50, 3), dtype=np.uint8) + mock_cv2_resize.return_value = resized_image_mock + target_w, target_h = 50, 50 + + result = ipu.resize_image(original_image, target_w, target_h) + + mock_cv2_resize.assert_called_once_with(original_image, (target_w, target_h), interpolation=cv2.INTER_LANCZOS4) + assert np.array_equal(result, resized_image_mock) + +@mock.patch('cv2.resize') +def test_resize_image_upscale(mock_cv2_resize): + original_image = np.random.randint(0, 255, (50, 50, 3), dtype=np.uint8) + resized_image_mock = np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8) + mock_cv2_resize.return_value = resized_image_mock + target_w, target_h = 100, 100 + + result = ipu.resize_image(original_image, target_w, target_h) + + mock_cv2_resize.assert_called_once_with(original_image, (target_w, target_h), interpolation=cv2.INTER_CUBIC) + assert np.array_equal(result, resized_image_mock) + +@mock.patch('cv2.resize') +def test_resize_image_custom_interpolation(mock_cv2_resize): + original_image = np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8) + resized_image_mock = np.random.randint(0, 255, (50, 50, 3), dtype=np.uint8) + mock_cv2_resize.return_value = resized_image_mock + target_w, target_h = 50, 50 + + result = ipu.resize_image(original_image, target_w, target_h, interpolation=cv2.INTER_NEAREST) + + mock_cv2_resize.assert_called_once_with(original_image, (target_w, target_h), interpolation=cv2.INTER_NEAREST) + assert np.array_equal(result, resized_image_mock) + +def test_resize_image_none_input(): + with pytest.raises(ValueError, match="Cannot resize a None image."): + ipu.resize_image(None, 50, 50) + +@pytest.mark.parametrize("w, h", [(0, 50), (50, 0), (-1, 50)]) +def test_resize_image_invalid_dims(w, h): + original_image = np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8) + with pytest.raises(ValueError, match="Target width and height must be positive."): + ipu.resize_image(original_image, w, h) + + +@mock.patch('cv2.imwrite') +@mock.patch('pathlib.Path.mkdir') # Mock mkdir to avoid actual directory creation +def test_save_image_success(mock_mkdir, mock_cv2_imwrite): + mock_cv2_imwrite.return_value = True + img_data = np.zeros((10,10,3), dtype=np.uint8) # RGB + save_path = "output/test.png" + + # ipu.save_image converts RGB to BGR by default for non-EXR + # So we expect convert_rgb_to_bgr to be called internally, + # and cv2.imwrite to receive BGR data. + # We can mock convert_rgb_to_bgr if we want to be very specific, + # or trust its own unit tests and check the data passed to imwrite. + # For simplicity, let's assume convert_rgb_to_bgr works and imwrite gets BGR. + # The function copies data, so we can check the mock call. + + success = ipu.save_image(save_path, img_data, convert_to_bgr_before_save=True) + + assert success is True + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) + + # Check that imwrite was called. The first arg to assert_called_once_with is the path. + # The second arg is the image data. We need to compare it carefully. + # Since convert_rgb_to_bgr is called internally, the data passed to imwrite will be BGR. + # Let's create expected BGR data. + expected_bgr_data = cv2.cvtColor(img_data, cv2.COLOR_RGB2BGR) + + args, kwargs = mock_cv2_imwrite.call_args + assert args[0] == str(Path(save_path)) + assert np.array_equal(args[1], expected_bgr_data) + + +@mock.patch('cv2.imwrite') +@mock.patch('pathlib.Path.mkdir') +def test_save_image_success_exr_no_bgr_conversion(mock_mkdir, mock_cv2_imwrite): + mock_cv2_imwrite.return_value = True + img_data_rgb_float = np.random.rand(10,10,3).astype(np.float32) # RGB float for EXR + save_path = "output/test.exr" + + success = ipu.save_image(save_path, img_data_rgb_float, output_format="exr", convert_to_bgr_before_save=False) + + assert success is True + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) + args, kwargs = mock_cv2_imwrite.call_args + assert args[0] == str(Path(save_path)) + assert np.array_equal(args[1], img_data_rgb_float) # Should be original RGB data + +@mock.patch('cv2.imwrite') +@mock.patch('pathlib.Path.mkdir') +def test_save_image_success_explicit_bgr_false_png(mock_mkdir, mock_cv2_imwrite): + mock_cv2_imwrite.return_value = True + img_data_rgb = np.zeros((10,10,3), dtype=np.uint8) # RGB + save_path = "output/test.png" + + # If convert_to_bgr_before_save is False, it should save RGB as is. + # However, OpenCV's imwrite for PNG might still expect BGR. + # The function's docstring says: "If True and image is 3-channel, converts RGB to BGR." + # So if False, it passes the data as is. + success = ipu.save_image(save_path, img_data_rgb, convert_to_bgr_before_save=False) + + assert success is True + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) + args, kwargs = mock_cv2_imwrite.call_args + assert args[0] == str(Path(save_path)) + assert np.array_equal(args[1], img_data_rgb) + + +@mock.patch('cv2.imwrite') +@mock.patch('pathlib.Path.mkdir') +def test_save_image_failure(mock_mkdir, mock_cv2_imwrite): + mock_cv2_imwrite.return_value = False + img_data = np.zeros((10,10,3), dtype=np.uint8) + save_path = "output/fail.png" + + success = ipu.save_image(save_path, img_data) + + assert success is False + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) + mock_cv2_imwrite.assert_called_once() # Check it was called + +def test_save_image_none_data(): + assert ipu.save_image("output/none.png", None) is False + +@mock.patch('cv2.imwrite', side_effect=Exception("CV2 Write Error")) +@mock.patch('pathlib.Path.mkdir') +def test_save_image_exception(mock_mkdir, mock_cv2_imwrite_exception): + img_data = np.zeros((10,10,3), dtype=np.uint8) + save_path = "output/exception.png" + + success = ipu.save_image(save_path, img_data) + + assert success is False + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) + mock_cv2_imwrite_exception.assert_called_once() + +# Test data type conversions in save_image +@pytest.mark.parametrize( + "input_dtype, input_data_producer, output_dtype_target, expected_conversion_dtype, check_scaling", + [ + (np.uint16, lambda: (np.random.randint(0, 65535, (10,10,3), dtype=np.uint16)), np.uint8, np.uint8, True), + (np.float32, lambda: np.random.rand(10,10,3).astype(np.float32), np.uint8, np.uint8, True), + (np.uint8, lambda: (np.random.randint(0, 255, (10,10,3), dtype=np.uint8)), np.uint16, np.uint16, True), + (np.float32, lambda: np.random.rand(10,10,3).astype(np.float32), np.uint16, np.uint16, True), + (np.uint8, lambda: (np.random.randint(0, 255, (10,10,3), dtype=np.uint8)), np.float16, np.float16, True), + (np.uint16, lambda: (np.random.randint(0, 65535, (10,10,3), dtype=np.uint16)), np.float32, np.float32, True), + ] +) +@mock.patch('cv2.imwrite') +@mock.patch('pathlib.Path.mkdir') +def test_save_image_dtype_conversion(mock_mkdir, mock_cv2_imwrite, input_dtype, input_data_producer, output_dtype_target, expected_conversion_dtype, check_scaling): + mock_cv2_imwrite.return_value = True + img_data = input_data_producer() + original_img_data_copy = img_data.copy() # For checking scaling if needed + + ipu.save_image("output/dtype_test.png", img_data, output_dtype_target=output_dtype_target) + + mock_cv2_imwrite.assert_called_once() + saved_img_data = mock_cv2_imwrite.call_args[0][1] # Get the image data passed to imwrite + + assert saved_img_data.dtype == expected_conversion_dtype + + if check_scaling: + # This is a basic check. More precise checks would require known input/output values. + if output_dtype_target == np.uint8: + if input_dtype == np.uint16: + expected_scaled_data = (original_img_data_copy.astype(np.float32) / 65535.0 * 255.0).astype(np.uint8) + assert np.allclose(saved_img_data, cv2.cvtColor(expected_scaled_data, cv2.COLOR_RGB2BGR), atol=1) # Allow small diff due to float precision + elif input_dtype in [np.float16, np.float32, np.float64]: + expected_scaled_data = (np.clip(original_img_data_copy, 0.0, 1.0) * 255.0).astype(np.uint8) + assert np.allclose(saved_img_data, cv2.cvtColor(expected_scaled_data, cv2.COLOR_RGB2BGR), atol=1) + elif output_dtype_target == np.uint16: + if input_dtype == np.uint8: + expected_scaled_data = (original_img_data_copy.astype(np.float32) / 255.0 * 65535.0).astype(np.uint16) + assert np.allclose(saved_img_data, cv2.cvtColor(expected_scaled_data, cv2.COLOR_RGB2BGR), atol=1) + elif input_dtype in [np.float16, np.float32, np.float64]: + expected_scaled_data = (np.clip(original_img_data_copy, 0.0, 1.0) * 65535.0).astype(np.uint16) + assert np.allclose(saved_img_data, cv2.cvtColor(expected_scaled_data, cv2.COLOR_RGB2BGR), atol=1) + # Add more scaling checks for float16, float32 if necessary + + +# --- Tests for calculate_image_stats --- + +def test_calculate_image_stats_grayscale_uint8(): + img_data = np.array([[0, 128], [255, 10]], dtype=np.uint8) + # Expected normalized: [[0, 0.50196], [1.0, 0.03921]] approx + stats = ipu.calculate_image_stats(img_data) + assert stats is not None + assert np.isclose(stats["min"], 0/255.0) + assert np.isclose(stats["max"], 255/255.0) + assert np.isclose(stats["mean"], np.mean(img_data.astype(np.float64)/255.0)) + +def test_calculate_image_stats_color_uint8(): + img_data = np.array([ + [[0, 50, 100], [10, 60, 110]], + [[255, 128, 200], [20, 70, 120]] + ], dtype=np.uint8) + stats = ipu.calculate_image_stats(img_data) + assert stats is not None + # Min per channel (normalized) + assert np.allclose(stats["min"], [0/255.0, 50/255.0, 100/255.0]) + # Max per channel (normalized) + assert np.allclose(stats["max"], [255/255.0, 128/255.0, 200/255.0]) + # Mean per channel (normalized) + expected_mean = np.mean(img_data.astype(np.float64)/255.0, axis=(0,1)) + assert np.allclose(stats["mean"], expected_mean) + +def test_calculate_image_stats_grayscale_uint16(): + img_data = np.array([[0, 32768], [65535, 1000]], dtype=np.uint16) + stats = ipu.calculate_image_stats(img_data) + assert stats is not None + assert np.isclose(stats["min"], 0/65535.0) + assert np.isclose(stats["max"], 65535/65535.0) + assert np.isclose(stats["mean"], np.mean(img_data.astype(np.float64)/65535.0)) + +def test_calculate_image_stats_color_float32(): + # Floats are assumed to be in 0-1 range already by the function's normalization logic + img_data = np.array([ + [[0.0, 0.2, 0.4], [0.1, 0.3, 0.5]], + [[1.0, 0.5, 0.8], [0.05, 0.25, 0.6]] + ], dtype=np.float32) + stats = ipu.calculate_image_stats(img_data) + assert stats is not None + assert np.allclose(stats["min"], [0.0, 0.2, 0.4]) + assert np.allclose(stats["max"], [1.0, 0.5, 0.8]) + expected_mean = np.mean(img_data.astype(np.float64), axis=(0,1)) + assert np.allclose(stats["mean"], expected_mean) + +def test_calculate_image_stats_none_input(): + assert ipu.calculate_image_stats(None) is None + +def test_calculate_image_stats_unsupported_shape(): + img_data = np.zeros((2,2,2,2), dtype=np.uint8) # 4D array + assert ipu.calculate_image_stats(img_data) is None + +@mock.patch('numpy.mean', side_effect=Exception("Numpy error")) +def test_calculate_image_stats_exception_during_calculation(mock_np_mean): + img_data = np.array([[0, 128], [255, 10]], dtype=np.uint8) + stats = ipu.calculate_image_stats(img_data) + assert stats == {"error": "Error calculating image stats"} + +# Example of mocking ipu.load_image for a function that uses it (if calculate_image_stats used it) +# For the current calculate_image_stats, it takes image_data directly, so this is not needed for it. +# This is just an example as requested in the prompt for a hypothetical scenario. +@mock.patch('processing.utils.image_processing_utils.load_image') +def test_hypothetical_function_using_load_image(mock_load_image): + # This test is for a function that would call ipu.load_image internally + # e.g. def process_image_from_path(path): + # img_data = ipu.load_image(path) + # return ipu.calculate_image_stats(img_data) + + mock_img_data = np.array([[[0.5]]], dtype=np.float32) + mock_load_image.return_value = mock_img_data + + # result = ipu.hypothetical_process_image_from_path("dummy.png") + # mock_load_image.assert_called_once_with("dummy.png") + # assert result["mean"] == 0.5 + pass # This is a conceptual example \ No newline at end of file diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..cfc5ffa --- /dev/null +++ b/tests/utils/__init__.py @@ -0,0 +1 @@ +# This file makes the 'tests/utils' directory a Python package. \ No newline at end of file diff --git a/tests/utils/test_path_utils.py b/tests/utils/test_path_utils.py new file mode 100644 index 0000000..56f7a0a --- /dev/null +++ b/tests/utils/test_path_utils.py @@ -0,0 +1,252 @@ +import pytest +from pathlib import Path +from utils.path_utils import sanitize_filename, generate_path_from_pattern + +# Tests for sanitize_filename +def test_sanitize_filename_valid(): + assert sanitize_filename("valid_filename.txt") == "valid_filename.txt" + +def test_sanitize_filename_with_spaces(): + assert sanitize_filename("file name with spaces.txt") == "file_name_with_spaces.txt" + +def test_sanitize_filename_with_special_characters(): + assert sanitize_filename("file!@#$%^&*()[]{};:'\",.<>/?\\|.txt") == "file____________________.txt" + +def test_sanitize_filename_with_leading_trailing_whitespace(): + assert sanitize_filename(" filename_with_spaces .txt") == "filename_with_spaces.txt" + +def test_sanitize_filename_empty_string(): + assert sanitize_filename("") == "" + +def test_sanitize_filename_with_none(): + with pytest.raises(TypeError): + sanitize_filename(None) + +def test_sanitize_filename_mixed_case(): + assert sanitize_filename("MixedCaseFileName.PNG") == "MixedCaseFileName.PNG" + +def test_sanitize_filename_long_filename(): + long_name = "a" * 255 + ".txt" + # Assuming the function doesn't truncate, but sanitizes. + # If it's meant to handle OS limits, this test might need adjustment + # based on the function's specific behavior for long names. + assert sanitize_filename(long_name) == long_name + +def test_sanitize_filename_unicode_characters(): + assert sanitize_filename("文件名前缀_文件名_后缀.jpg") == "文件名前缀_文件名_后缀.jpg" + +def test_sanitize_filename_multiple_extensions(): + assert sanitize_filename("archive.tar.gz") == "archive.tar.gz" + +def test_sanitize_filename_no_extension(): + assert sanitize_filename("filename") == "filename" + +def test_sanitize_filename_only_special_chars(): + assert sanitize_filename("!@#$%^") == "______" + +def test_sanitize_filename_with_hyphens_and_underscores(): + assert sanitize_filename("file-name_with-hyphens_and_underscores.zip") == "file-name_with-hyphens_and_underscores.zip" + +# Tests for generate_path_from_pattern +def test_generate_path_basic(): + result = generate_path_from_pattern( + base_path="output", + pattern="{asset_name}/{map_type}/{filename}", + asset_name="MyAsset", + map_type="Diffuse", + filename="MyAsset_Diffuse.png", + source_rule_name="TestRule", + incrementing_value=None, + sha5_value=None + ) + expected = Path("output/MyAsset/Diffuse/MyAsset_Diffuse.png") + assert Path(result) == expected + +def test_generate_path_all_placeholders(): + result = generate_path_from_pattern( + base_path="project_files", + pattern="{source_rule_name}/{asset_name}/{map_type}_{incrementing_value}_{sha5_value}/{filename}", + asset_name="AnotherAsset", + map_type="Normal", + filename="NormalMap.tif", + source_rule_name="ComplexRule", + incrementing_value="001", + sha5_value="abcde" + ) + expected = Path("project_files/ComplexRule/AnotherAsset/Normal_001_abcde/NormalMap.tif") + assert Path(result) == expected + +def test_generate_path_optional_placeholders_none(): + result = generate_path_from_pattern( + base_path="data", + pattern="{asset_name}/{filename}", + asset_name="SimpleAsset", + map_type="Albedo", # map_type is in pattern but not used if not in string + filename="texture.jpg", + source_rule_name="Basic", + incrementing_value=None, + sha5_value=None + ) + expected = Path("data/SimpleAsset/texture.jpg") + assert Path(result) == expected + +def test_generate_path_optional_incrementing_value_present(): + result = generate_path_from_pattern( + base_path="assets", + pattern="{asset_name}/{map_type}/v{incrementing_value}/{filename}", + asset_name="VersionedAsset", + map_type="Specular", + filename="spec.png", + source_rule_name="VersioningRule", + incrementing_value="3", + sha5_value=None + ) + expected = Path("assets/VersionedAsset/Specular/v3/spec.png") + assert Path(result) == expected + +def test_generate_path_optional_sha5_value_present(): + result = generate_path_from_pattern( + base_path="cache", + pattern="{asset_name}/{sha5_value}/{filename}", + asset_name="HashedAsset", + map_type="Roughness", + filename="rough.exr", + source_rule_name="HashingRule", + incrementing_value=None, + sha5_value="f1234" + ) + expected = Path("cache/HashedAsset/f1234/rough.exr") + assert Path(result) == expected + +def test_generate_path_base_path_is_path_object(): + result = generate_path_from_pattern( + base_path=Path("output_path"), + pattern="{asset_name}/{filename}", + asset_name="ObjectAsset", + map_type="AO", + filename="ao.png", + source_rule_name="PathObjectRule", + incrementing_value=None, + sha5_value=None + ) + expected = Path("output_path/ObjectAsset/ao.png") + assert Path(result) == expected + +def test_generate_path_empty_pattern(): + result = generate_path_from_pattern( + base_path="output", + pattern="", # Empty pattern should just use base_path and filename + asset_name="MyAsset", + map_type="Diffuse", + filename="MyAsset_Diffuse.png", + source_rule_name="TestRule", + incrementing_value=None, + sha5_value=None + ) + expected = Path("output/MyAsset_Diffuse.png") + assert Path(result) == expected + +def test_generate_path_pattern_with_no_placeholders(): + result = generate_path_from_pattern( + base_path="fixed_output", + pattern="some/static/path", # Pattern has no placeholders + asset_name="MyAsset", + map_type="Diffuse", + filename="MyAsset_Diffuse.png", + source_rule_name="TestRule", + incrementing_value=None, + sha5_value=None + ) + expected = Path("fixed_output/some/static/path/MyAsset_Diffuse.png") + assert Path(result) == expected + +def test_generate_path_filename_with_subdirs_in_pattern(): + result = generate_path_from_pattern( + base_path="output", + pattern="{asset_name}", # Filename itself will be appended + asset_name="AssetWithSubdirFile", + map_type="Color", + filename="textures/variant1/color.png", # Filename contains subdirectories + source_rule_name="SubdirRule", + incrementing_value=None, + sha5_value=None + ) + # The function is expected to join pattern result with filename + expected = Path("output/AssetWithSubdirFile/textures/variant1/color.png") + assert Path(result) == expected + +def test_generate_path_no_filename_provided(): + # This test assumes that if filename is None or empty, it might raise an error + # or behave in a specific way, e.g. not append anything or use a default. + # Adjust based on actual function behavior for missing filename. + # For now, let's assume it might raise TypeError if filename is critical. + with pytest.raises(TypeError): # Or ValueError, depending on implementation + generate_path_from_pattern( + base_path="output", + pattern="{asset_name}/{map_type}", + asset_name="MyAsset", + map_type="Diffuse", + filename=None, # No filename + source_rule_name="TestRule", + incrementing_value=None, + sha5_value=None + ) + +def test_generate_path_all_values_are_empty_strings_or_none_where_applicable(): + result = generate_path_from_pattern( + base_path="", # Empty base_path + pattern="{asset_name}/{map_type}/{incrementing_value}/{sha5_value}", + asset_name="", # Empty asset_name + map_type="", # Empty map_type + filename="empty_test.file", + source_rule_name="", # Empty source_rule_name + incrementing_value="", # Empty incrementing_value + sha5_value="" # Empty sha5_value + ) + # Behavior with empty strings might vary. Assuming they are treated as literal empty segments. + # Path("///empty_test.file") might resolve to "/empty_test.file" on POSIX + # or just "empty_test.file" if base_path is current dir. + # Let's assume Path() handles normalization. + # If base_path is "", it means current directory. + # So, "//empty_test.file" relative to current dir. + # Path objects normalize this. e.g. Path('//a') -> Path('/a') on POSIX + # Path('a//b') -> Path('a/b') + # Path('/a//b') -> Path('/a/b') + # Path('//a//b') -> Path('/a/b') + # If base_path is empty, it's like Path('.////empty_test.file') + expected = Path("empty_test.file") # Simplified, actual result might be OS dependent or Path lib norm. + # More robust check: + # result_path = Path(result) + # expected_path = Path.cwd() / "" / "" / "" / "" / "empty_test.file" # This is not quite right + # Let's assume the function joins them: "" + "/" + "" + "/" + "" + "/" + "" + "/" + "empty_test.file" + # which becomes "////empty_test.file" + # Path("////empty_test.file") on Windows becomes "\\empty_test.file" (network path attempt) + # Path("////empty_test.file") on Linux becomes "/empty_test.file" + # Given the function likely uses os.path.join or Path.joinpath, + # and base_path="", asset_name="", map_type="", inc_val="", sha5_val="" + # pattern = "{asset_name}/{map_type}/{incrementing_value}/{sha5_value}" -> "///" + # result = base_path / pattern_result / filename + # result = "" / "///" / "empty_test.file" + # Path("") / "///" / "empty_test.file" -> Path("///empty_test.file") + # This is tricky. Let's assume the function is robust. + # If all path segments are empty, it should ideally resolve to just the filename relative to base_path. + # If base_path is also empty, then filename relative to CWD. + # Let's test the expected output based on typical os.path.join behavior: + # os.path.join("", "", "", "", "", "empty_test.file") -> "empty_test.file" on Windows + # os.path.join("", "", "", "", "", "empty_test.file") -> "empty_test.file" on Linux + assert Path(result) == Path("empty_test.file") + + +def test_generate_path_with_dots_in_placeholders(): + result = generate_path_from_pattern( + base_path="output", + pattern="{asset_name}/{map_type}", + asset_name="My.Asset.V1", + map_type="Diffuse.Main", + filename="texture.png", + source_rule_name="DotsRule", + incrementing_value=None, + sha5_value=None + ) + expected = Path("output/My.Asset.V1/Diffuse.Main/texture.png") + assert Path(result) == expected \ No newline at end of file diff --git a/utils/hash_utils.py b/utils/hash_utils.py new file mode 100644 index 0000000..0be88e3 --- /dev/null +++ b/utils/hash_utils.py @@ -0,0 +1,45 @@ +import hashlib +import logging +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +def calculate_sha256(file_path: Path) -> Optional[str]: + """ + Calculates the SHA-256 hash of a file. + + Args: + file_path: The path to the file. + + Returns: + The SHA-256 hash as a hexadecimal string, or None if an error occurs. + """ + if not isinstance(file_path, Path): + try: + file_path = Path(file_path) + except TypeError: + logger.error(f"Invalid file path type: {type(file_path)}. Expected Path object or string.") + return None + + if not file_path.is_file(): + logger.error(f"File not found or is not a regular file: {file_path}") + return None + + sha256_hash = hashlib.sha256() + buffer_size = 65536 # Read in 64k chunks + + try: + with open(file_path, "rb") as f: + while True: + data = f.read(buffer_size) + if not data: + break + sha256_hash.update(data) + return sha256_hash.hexdigest() + except IOError as e: + logger.error(f"Error reading file {file_path}: {e}") + return None + except Exception as e: + logger.error(f"An unexpected error occurred while hashing {file_path}: {e}") + return None \ No newline at end of file diff --git a/utils/path_utils.py b/utils/path_utils.py new file mode 100644 index 0000000..5b86194 --- /dev/null +++ b/utils/path_utils.py @@ -0,0 +1,297 @@ +import os +import sys +import datetime +import re +import logging +from pathlib import Path +from typing import Optional, Dict + +logger = logging.getLogger(__name__) + +def generate_path_from_pattern(pattern_string: str, token_data: dict) -> str: + """ + Generates a file path by replacing tokens in a pattern string with values + from the provided token_data dictionary. + + Args: + pattern_string: The string containing tokens to be replaced (e.g., + "[Assettype]/[supplier]/[assetname]_[resolution].[ext]"). + token_data: A dictionary where keys are token names (without brackets, + case-insensitive) and values are the replacement strings. + Special tokens like 'IncrementingValue' or '####' should + be provided here if used in the pattern. + + Returns: + The generated path string with tokens replaced. + + Raises: + ValueError: If a token required by the pattern (excluding date/time/apppath) + is not found in token_data. + KeyError: If internal logic fails to find expected date/time components. + """ + if not isinstance(pattern_string, str): + raise TypeError("pattern_string must be a string") + if not isinstance(token_data, dict): + raise TypeError("token_data must be a dictionary") + + # Normalize token keys in the input data for case-insensitive matching + normalized_token_data = {k.lower(): v for k, v in token_data.items()} + + # --- Prepare dynamic/default token values --- + now = datetime.datetime.now() + dynamic_tokens = { + 'date': now.strftime('%Y%m%d'), + 'time': now.strftime('%H%M%S'), + # Provide a default ApplicationPath, can be overridden by token_data + 'applicationpath': os.path.abspath(os.getcwd()) + } + + # Merge dynamic tokens with provided data, allowing overrides + # Provided data takes precedence + full_token_data = {**dynamic_tokens, **normalized_token_data} + + # --- Define known tokens (lowercase) --- + # Add variations like #### for IncrementingValue + known_tokens_lc = { + 'assettype', 'supplier', 'assetname', 'resolution', 'ext', + 'incrementingvalue', '####', 'date', 'time', 'sha5', 'applicationpath' + } + + output_path = pattern_string + + # --- Regex to find all tokens like [TokenName] --- + token_pattern = re.compile(r'\[([^\]]+)\]') + tokens_found = token_pattern.findall(pattern_string) + + processed_tokens_lc = set() + + for token_name in tokens_found: + token_name_lc = token_name.lower() + + # Handle alias #### for IncrementingValue + lookup_key = 'incrementingvalue' if token_name_lc == '####' else token_name_lc + + if lookup_key in processed_tokens_lc: + continue # Already processed this token type + + if lookup_key in full_token_data: + replacement_value = str(full_token_data[lookup_key]) # Ensure string + # Replace all occurrences of this token (case-insensitive original name) + # We use a regex finditer to replace only the specific token format + # to avoid replacing substrings within other words. + current_token_pattern = re.compile(re.escape(f'[{token_name}]'), re.IGNORECASE) + output_path = current_token_pattern.sub(replacement_value, output_path) + processed_tokens_lc.add(lookup_key) + elif lookup_key in known_tokens_lc: + # Known token but not found in data (and not a dynamic one we generated) + logger.warning(f"Token '[{token_name}]' found in pattern but not in token_data.") + # Raise error for non-optional tokens if needed, or replace with placeholder + # For now, let's raise an error to be explicit + raise ValueError(f"Required token '[{token_name}]' not found in token_data.") + else: + # Token not recognized + logger.warning(f"Unknown token '[{token_name}]' found in pattern string. Leaving it unchanged.") + + # --- Final path cleaning (optional, e.g., normalize separators) --- + # output_path = os.path.normpath(output_path) # Consider implications on mixed separators + + return output_path +def get_next_incrementing_value(output_base_path: Path, output_directory_pattern: str) -> str: + """Determines the next incrementing value based on existing directories.""" + logger.debug(f"Calculating next increment value for pattern '{output_directory_pattern}' in '{output_base_path}'") + match = re.match(r"(.*?)(\[IncrementingValue\]|(#+))(.*)", output_directory_pattern) + if not match: + logger.warning(f"Could not find incrementing token ([IncrementingValue] or #+) in pattern '{output_directory_pattern}'. Defaulting to '00'.") + return "00" # Default fallback if pattern doesn't contain the token + + prefix_pattern, increment_token, suffix_pattern = match.groups() + num_digits = len(increment_token) if increment_token.startswith("#") else 2 # Default to 2 for [IncrementingValue] if not specified otherwise + logger.debug(f"Parsed pattern: prefix='{prefix_pattern}', token='{increment_token}' ({num_digits} digits), suffix='{suffix_pattern}'") + + # Replace other tokens in prefix/suffix with '*' for globbing + glob_prefix = re.sub(r'\[[^\]]+\]', '*', prefix_pattern) + glob_suffix = re.sub(r'\[[^\]]+\]', '*', suffix_pattern) + # Construct the glob pattern part for the number itself + glob_increment_part = f"[{'0-9' * num_digits}]" # Matches exactly num_digits + glob_pattern = f"{glob_prefix}{glob_increment_part}{glob_suffix}" + logger.debug(f"Constructed glob pattern: {glob_pattern}") + + max_value = -1 + try: + # Prepare regex to extract the number from directory names matching the full pattern + # Escape regex special characters in the literal parts of the pattern + extract_prefix_re = re.escape(prefix_pattern) + extract_suffix_re = re.escape(suffix_pattern) + # The regex captures exactly num_digits between the escaped prefix and suffix + extract_regex = re.compile(rf"^{extract_prefix_re}(\d{{{num_digits}}}){extract_suffix_re}.*") + logger.debug(f"Constructed extraction regex: {extract_regex.pattern}") + + if not output_base_path.is_dir(): + logger.warning(f"Output base path '{output_base_path}' does not exist or is not a directory. Cannot scan for existing values.") + else: + for item in output_base_path.glob(glob_pattern): + if item.is_dir(): + logger.debug(f"Checking directory: {item.name}") + num_match = extract_regex.match(item.name) + if num_match: + try: + current_val = int(num_match.group(1)) + logger.debug(f"Extracted value {current_val} from {item.name}") + max_value = max(max_value, current_val) + except (ValueError, IndexError) as e: + logger.warning(f"Could not parse number from matching directory '{item.name}': {e}") + else: + logger.debug(f"Directory '{item.name}' matched glob but not extraction regex.") + + except Exception as e: + logger.error(f"Error searching for incrementing values using glob pattern '{glob_pattern}' in '{output_base_path}': {e}", exc_info=True) + # Decide on fallback behavior - returning "00" might be safer than raising + return "00" # Fallback on error during search + + next_value = max_value + 1 + format_string = f"{{:0{num_digits}d}}" + next_value_str = format_string.format(next_value) + logger.info(f"Determined next incrementing value: {next_value_str} (Max found: {max_value})") + return next_value_str + +def sanitize_filename(name: str) -> str: + """Removes or replaces characters invalid for filenames/directory names.""" + if not isinstance(name, str): name = str(name) + name = re.sub(r'[^\w.\-]+', '_', name) # Allow alphanumeric, underscore, hyphen, dot + name = re.sub(r'_+', '_', name) + name = name.strip('_') + if not name: name = "invalid_name" + return name + +def get_filename_friendly_map_type(internal_map_type: str, file_type_definitions: Optional[Dict[str, Dict]]) -> str: + """Derives a filename-friendly map type from the internal map type.""" + filename_friendly_map_type = internal_map_type # Fallback + if not file_type_definitions or not isinstance(file_type_definitions, dict) or not file_type_definitions: + logger.warning(f"Filename-friendly lookup: FILE_TYPE_DEFINITIONS not available or invalid. Falling back to internal type: {internal_map_type}") + return filename_friendly_map_type + + base_map_key_val = None + suffix_part = "" + # Sort keys by length descending to match longest prefix first (e.g., MAP_ROUGHNESS before MAP_ROUGH) + sorted_known_base_keys = sorted(list(file_type_definitions.keys()), key=len, reverse=True) + + for known_key in sorted_known_base_keys: + if internal_map_type.startswith(known_key): + base_map_key_val = known_key + suffix_part = internal_map_type[len(known_key):] + break + + if base_map_key_val: + definition = file_type_definitions.get(base_map_key_val) + if definition and isinstance(definition, dict): + standard_type_alias = definition.get("standard_type") + if standard_type_alias and isinstance(standard_type_alias, str) and standard_type_alias.strip(): + filename_friendly_map_type = standard_type_alias.strip() + suffix_part + logger.debug(f"Filename-friendly lookup: Transformed '{internal_map_type}' -> '{filename_friendly_map_type}'") + else: + logger.warning(f"Filename-friendly lookup: Standard type alias for '{base_map_key_val}' is missing or invalid. Falling back.") + else: + logger.warning(f"Filename-friendly lookup: No valid definition for '{base_map_key_val}'. Falling back.") + else: + logger.warning(f"Filename-friendly lookup: Could not parse base key from '{internal_map_type}'. Falling back.") + + return filename_friendly_map_type +# --- Basic Unit Tests --- +if __name__ == "__main__": + print("Running basic tests for path_utils.generate_path_from_pattern...") + + test_pattern_1 = "[Assettype]/[supplier]/[assetname]_[resolution]_[Date]_[Time].[ext]" + test_data_1 = { + "AssetType": "Texture", + "supplier": "MegaScans", + "assetName": "RustyMetalPanel", + "Resolution": "4k", + "EXT": "png", + "Sha5": "abcde" # Included but not in pattern + } + expected_1_base = f"Texture/MegaScans/RustyMetalPanel_4k_" + try: + result_1 = generate_path_from_pattern(test_pattern_1, test_data_1) + assert result_1.startswith(expected_1_base) + assert result_1.endswith(".png") + assert len(result_1.split('_')) == 5 # Check date and time were added + print(f"PASS: Test 1 - Basic replacement: {result_1}") + except Exception as e: + print(f"FAIL: Test 1 - {e}") + + test_pattern_2 = "Output/[assetname]/[assetname]_####.[ext]" + test_data_2 = { + "assetname": "WoodFloor", + "IncrementingValue": "001", + "ext": "jpg" + } + expected_2 = "Output/WoodFloor/WoodFloor_001.jpg" + try: + result_2 = generate_path_from_pattern(test_pattern_2, test_data_2) + assert result_2 == expected_2 + print(f"PASS: Test 2 - IncrementingValue (####): {result_2}") + except Exception as e: + print(f"FAIL: Test 2 - {e}") + + test_pattern_3 = "AppPath=[ApplicationPath]/[assetname].[ext]" + test_data_3 = {"assetname": "Test", "ext": "txt"} + expected_3_start = f"AppPath={os.path.abspath(os.getcwd())}/Test.txt" + try: + result_3 = generate_path_from_pattern(test_pattern_3, test_data_3) + assert result_3 == expected_3_start + print(f"PASS: Test 3 - ApplicationPath (default): {result_3}") + except Exception as e: + print(f"FAIL: Test 3 - {e}") + + test_pattern_4 = "AppPath=[ApplicationPath]/[assetname].[ext]" + test_data_4 = {"assetname": "Test", "ext": "txt", "ApplicationPath": "/custom/path"} + expected_4 = "/custom/path/Test.txt" # Note: AppPath= part is replaced by the token logic + # Correction: The pattern includes "AppPath=", so it should remain. + expected_4_corrected = "AppPath=/custom/path/Test.txt" + try: + result_4 = generate_path_from_pattern(test_pattern_4, test_data_4) + assert result_4 == expected_4_corrected + print(f"PASS: Test 4 - ApplicationPath (override): {result_4}") + except Exception as e: + print(f"FAIL: Test 4 - {e}") + + + test_pattern_5 = "[assetname]/[MissingToken].[ext]" + test_data_5 = {"assetname": "FailureTest", "ext": "err"} + try: + generate_path_from_pattern(test_pattern_5, test_data_5) + print("FAIL: Test 5 - Expected ValueError for missing token") + except ValueError as e: + assert "MissingToken" in str(e) + print(f"PASS: Test 5 - Correctly raised ValueError for missing token: {e}") + except Exception as e: + print(f"FAIL: Test 5 - Incorrect exception type: {e}") + + + test_pattern_6 = "[assetname]/[UnknownToken].[ext]" + test_data_6 = {"assetname": "UnknownTest", "ext": "dat"} + expected_6 = "UnknownTest/[UnknownToken].dat" # Unknown tokens are left as is + try: + # Capture warnings + logging.basicConfig() + with logging.catch_warnings(record=True) as w: + result_6 = generate_path_from_pattern(test_pattern_6, test_data_6) + assert result_6 == expected_6 + assert len(w) == 1 + assert "Unknown token '[UnknownToken]'" in str(w[0].message) + print(f"PASS: Test 6 - Unknown token left unchanged: {result_6}") + except Exception as e: + print(f"FAIL: Test 6 - {e}") + + test_pattern_7 = "[assetname]/[assetname].png" # Case check + test_data_7 = {"AssetName": "CaseTest"} + expected_7 = "CaseTest/CaseTest.png" + try: + result_7 = generate_path_from_pattern(test_pattern_7, test_data_7) + assert result_7 == expected_7 + print(f"PASS: Test 7 - Case insensitivity: {result_7}") + except Exception as e: + print(f"FAIL: Test 7 - {e}") + + + print("Basic tests finished.") \ No newline at end of file diff --git a/utils/prediction_utils.py b/utils/prediction_utils.py new file mode 100644 index 0000000..8914f21 --- /dev/null +++ b/utils/prediction_utils.py @@ -0,0 +1,132 @@ + +import logging +import re +from pathlib import Path +from typing import Optional, Dict, Any + +from rule_structure import SourceRule, RuleSet, MapRule, AssetRule +from configuration import load_preset +from utils.structure_analyzer import analyze_archive_structure # Hypothetical utility + +log = logging.getLogger(__name__) + +# Regex to extract preset name (similar to monitor.py) +# Matches "[PresetName]_anything.zip/rar/7z" +PRESET_FILENAME_REGEX = re.compile(r"^\[?([a-zA-Z0-9_-]+)\]?_.*\.(zip|rar|7z)$", re.IGNORECASE) + +class PredictionError(Exception): + """Custom exception for prediction failures.""" + pass + +def generate_source_rule_from_archive(archive_path: Path, config: Dict[str, Any]) -> SourceRule: + """ + Generates a SourceRule hierarchy based on rules defined in a preset, + determined by the archive filename. + + Args: + archive_path: Path to the input archive file. + config: The loaded application configuration dictionary, expected + to contain preset information or a way to load it. + + Returns: + The generated SourceRule hierarchy. + + Raises: + PredictionError: If the preset cannot be determined, loaded, or + if rule generation fails. + FileNotFoundError: If the archive_path does not exist. + """ + if not archive_path.is_file(): + raise FileNotFoundError(f"Archive file not found: {archive_path}") + + log.debug(f"Generating SourceRule for archive: {archive_path.name}") + + match = PRESET_FILENAME_REGEX.match(archive_path.name) + if not match: + raise PredictionError(f"Filename '{archive_path.name}' does not match expected format '[preset]_filename.ext'. Cannot determine preset.") + + preset_name = match.group(1) + log.info(f"Extracted preset name: '{preset_name}' from {archive_path.name}") + + try: + # Assuming load_preset takes the name and maybe the base config/path + # Adjust based on the actual signature of load_preset + preset_config = load_preset(preset_name) # This might need config path or dict + if not preset_config: + raise PredictionError(f"Preset '{preset_name}' configuration is empty or invalid.") + # Assuming the preset config directly contains the RuleSet structure + # or needs parsing into RuleSet. Let's assume it needs parsing. + # This part is highly dependent on how presets are stored and loaded. + # For now, let's assume preset_config IS the RuleSet dictionary. + if not isinstance(preset_config.get('rules'), dict): + raise PredictionError(f"Preset '{preset_name}' does not contain a valid 'rules' dictionary.") + rule_set_dict = preset_config['rules'] + # Assuming RuleSet has a class method or similar for this + rule_set = RuleSet.from_dict(rule_set_dict) # Placeholder for actual deserialization + + except FileNotFoundError: + raise PredictionError(f"Preset file for '{preset_name}' not found.") + except Exception as e: + log.exception(f"Failed to load or parse preset '{preset_name}': {e}") + raise PredictionError(f"Failed to load or parse preset '{preset_name}': {e}") + + if not rule_set: + raise PredictionError(f"Failed to obtain RuleSet for preset '{preset_name}'.") + + log.debug(f"Successfully loaded RuleSet for preset: {preset_name}") + + # This simulates what a RuleBasedPredictionHandler might do, but without + # needing the actual extracted files for *this* step. The rules themselves + # define the expected structure. The ProcessingEngine will later use this + # rule against the actual extracted files. + + # The actual structure (AssetRules, MapRules) comes directly from the RuleSet. + # We might need to adapt the archive name slightly (e.g., remove preset prefix) + # for the root node name, depending on desired output structure. + root_name = archive_path.stem + source_rule = SourceRule(name=root_name, rule_set=rule_set) + + # Potentially add logic here if basic archive structure analysis *is* needed + # for rule generation (e.g., using utils.structure_analyzer if it exists) + + log.info(f"Generated initial SourceRule for '{archive_path.name}' based on preset '{preset_name}'.") + + # No temporary workspace needed/created in this function based on current plan. + # Cleanup is not required here. + return source_rule + +# Example Usage (Conceptual - requires actual config/presets) +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + log.info("Testing prediction_utils...") + + dummy_archive = Path("./[TestPreset]_MyAsset.zip") + dummy_archive.touch() + + preset_dir = Path(__file__).parent.parent / "Presets" + preset_dir.mkdir(exist_ok=True) + dummy_preset_path = preset_dir / "TestPreset.json" + dummy_preset_content = """ + { + "name": "TestPreset", + "description": "A dummy preset for testing", + "rules": { + "map_rules": [ + {"pattern": ".*albedo.*", "map_type": "Albedo", "color_space": "sRGB"}, + {"pattern": ".*normal.*", "map_type": "Normal", "color_space": "Non-Color"} + ], + "asset_rules": [ + {"pattern": ".*", "material_name": "{asset_name}"} + ] + }, + "settings": {} + } + """ + + + + + + + + log.warning("Note: Main execution block is commented out as it requires specific implementations of load_preset and RuleSet.from_dict.") \ No newline at end of file diff --git a/utils/workspace_utils.py b/utils/workspace_utils.py new file mode 100644 index 0000000..bd69d2c --- /dev/null +++ b/utils/workspace_utils.py @@ -0,0 +1,77 @@ + +import tempfile +import shutil +import zipfile +import logging +from pathlib import Path +from typing import Union + +log = logging.getLogger(__name__) + +# Add more archive extensions as needed (e.g., '.rar', '.7z'). +# Non-zip formats may require additional libraries like patoolib. +SUPPORTED_ARCHIVES = {'.zip'} + +def prepare_processing_workspace(input_path_str: Union[str, Path]) -> Path: + """ + Prepares a temporary workspace for processing an asset source. + + Handles copying directory contents or extracting supported archives + into a unique temporary directory. + + Args: + input_path_str: The path (as a string or Path object) to the input + directory or archive file. + + Returns: + The Path object representing the created temporary workspace directory. + The caller is responsible for cleaning up this directory. + + Raises: + FileNotFoundError: If the input_path does not exist. + ValueError: If the input_path is not a directory or a supported archive type. + zipfile.BadZipFile: If a zip file is corrupted. + OSError: If there are issues creating the temp directory or copying files. + """ + input_path = Path(input_path_str) + log.info(f"Preparing workspace for input: {input_path}") + + if not input_path.exists(): + raise FileNotFoundError(f"Input path does not exist: {input_path}") + + try: + temp_workspace_dir = tempfile.mkdtemp(prefix="asset_proc_") + prepared_workspace_path = Path(temp_workspace_dir) + log.info(f"Created temporary workspace: {prepared_workspace_path}") + except OSError as e: + log.error(f"Failed to create temporary directory: {e}") + raise + + try: + if input_path.is_dir(): + log.info(f"Input is a directory, copying contents to workspace: {input_path}") + shutil.copytree(input_path, prepared_workspace_path, dirs_exist_ok=True) + elif input_path.is_file() and input_path.suffix.lower() in SUPPORTED_ARCHIVES: + log.info(f"Input is a supported archive ({input_path.suffix}), extracting to workspace: {input_path}") + if input_path.suffix.lower() == '.zip': + with zipfile.ZipFile(input_path, 'r') as zip_ref: + zip_ref.extractall(prepared_workspace_path) + # Add elif blocks here for other archive types (e.g., using patoolib) + else: + # This case should ideally not be reached if SUPPORTED_ARCHIVES is correct + raise ValueError(f"Archive type {input_path.suffix} marked as supported but no extraction logic defined.") + else: + raise ValueError(f"Unsupported input type: {input_path}. Must be a directory or a supported archive ({', '.join(SUPPORTED_ARCHIVES)}).") + + log.debug(f"Workspace preparation successful for: {input_path}") + return prepared_workspace_path + + except (FileNotFoundError, ValueError, zipfile.BadZipFile, OSError, ImportError) as e: + log.error(f"Error during workspace preparation for {input_path}: {e}. Cleaning up workspace.") + if prepared_workspace_path.exists(): + try: + shutil.rmtree(prepared_workspace_path) + log.info(f"Cleaned up failed workspace: {prepared_workspace_path}") + except OSError as cleanup_error: + log.error(f"Failed to cleanup workspace {prepared_workspace_path} after error: {cleanup_error}") + raise \ No newline at end of file