From 4ffb2ff78c4de076126168769fac55b0e83eb158 Mon Sep 17 00:00:00 2001 From: Rusfort Date: Mon, 12 May 2025 13:31:58 +0200 Subject: [PATCH] Pipeline simplification - Needs testing! --- ProjectNotes/PipelineRefactoringPlan.md | 154 +++ ProjectNotes/ProcessingEngineRefactorPlan.md | 181 --- config/app_settings.json | 7 +- .../stages/individual_map_processing.py | 1169 ++++++++--------- processing/pipeline/stages/map_merging.py | 303 +---- processing/utils/image_processing_utils.py | 31 + processing/utils/image_saving_utils.py | 250 ++++ 7 files changed, 1046 insertions(+), 1049 deletions(-) create mode 100644 ProjectNotes/PipelineRefactoringPlan.md delete mode 100644 ProjectNotes/ProcessingEngineRefactorPlan.md create mode 100644 processing/utils/image_saving_utils.py diff --git a/ProjectNotes/PipelineRefactoringPlan.md b/ProjectNotes/PipelineRefactoringPlan.md new file mode 100644 index 0000000..18b8df8 --- /dev/null +++ b/ProjectNotes/PipelineRefactoringPlan.md @@ -0,0 +1,154 @@ +# Revised Refactoring Plan: Processing Pipeline + +**Overall Goal:** To simplify the processing pipeline by refactoring the map merging process, consolidating map transformations (Gloss-to-Rough, Normal Green Invert), and creating a unified, configurable image saving utility. This plan aims to improve clarity, significantly reduce I/O by favoring in-memory operations, and make Power-of-Two (POT) scaling an optional, integrated step. + +**I. Map Merging Stage (`processing/pipeline/stages/map_merging.py`)** + +* **Objective:** Transform this stage from performing merges to generating tasks for merged images. +* **Changes to `MapMergingStage.execute()`:** + 1. Iterate through `context.config_obj.map_merge_rules`. + 2. Identify required input map types and find their corresponding source file paths (potentially original paths or outputs of prior essential stages if any). + 3. Create "merged image tasks" and add them to `context.merged_image_tasks`. + 4. Each task entry will contain: + * `output_map_type`: Target map type (e.g., "MAP_NRMRGH"). + * `input_map_sources`: Details of source map types and file paths. + * `merge_rule_config`: Complete merge rule configuration (including fallback values). + * `source_dimensions`: Dimensions for the high-resolution merged map basis. + * `source_bit_depths`: Information about the bit depth of original source maps (needed for "respect_inputs" rule in save utility). + +**II. Individual Map Processing Stage (`processing/pipeline/stages/individual_map_processing.py`)** + +* **Objective:** Adapt this stage to handle both individual raw maps and `merged_image_tasks`. It will perform necessary in-memory transformations (Gloss-to-Rough, Normal Green Invert) and prepare a single "high-resolution" source image (in memory) to be passed to the `UnifiedSaveUtility`. +* **Changes to `IndividualMapProcessingStage.execute()`:** + 1. **Input Handling Loop:** Iterate through `context.files_to_process` (regular maps) and `context.merged_image_tasks`. + 2. **Image Data Preparation:** + * **For regular maps:** Load the source image file into memory (`current_image_data`). Determine `base_map_type` from the `FileRule`. Determine source bit depth. + * **For `merged_image_tasks`:** + * Attempt to load input map files specified in `input_map_sources`. If a file is missing, log a warning and generate placeholder data using fallback values from `merge_rule_config`. Handle other load errors. + * Check dimensions of loaded/fallback data. Apply `MERGE_DIMENSION_MISMATCH_STRATEGY` (e.g., resize, log warning) or handle "ERROR_SKIP" strategy (log error, mark task failed, continue). + * Perform the merge operation in memory according to `merge_rule_config`. Result is `current_image_data`. `base_map_type` is the task's `output_map_type`. + 3. **In-Memory Transformations:** + * **Gloss-to-Rough Conversion:** + * If `base_map_type` starts with "MAP_GLOSS": + * Perform inversion on `current_image_data` (in memory). + * Update `base_map_type` to "MAP_ROUGH". + * Log the conversion. + * **Normal Map Green Channel Inversion:** + * If `base_map_type` is "NORMAL" *and* `context.config_obj.general_settings.invert_normal_map_green_channel_globally` is true: + * Perform green channel inversion on `current_image_data` (in memory). + * Log the inversion. + 4. **Optional Initial Scaling (POT or other):** + * Check `INITIAL_SCALING_MODE` from config. + * If `"POT_DOWNSCALE"`: Perform POT downscaling on `current_image_data` (in memory) -> `image_to_save`. + * If `"NONE"`: `image_to_save` = `current_image_data`. + * *(Note: `image_to_save` now reflects any prior transformations)*. + 5. **Color Management:** Apply necessary color management to `image_to_save`. + 6. **Pass to Save Utility:** Pass `image_to_save`, the (potentially updated) `base_map_type`, original source bit depth info (for "respect_inputs" rule), and other necessary details (like specific config values) to the `UnifiedSaveUtility`. + 7. **Remove Old Logic:** Remove old save logic, separate Gloss/Normal stage calls. + 8. **Context Update:** Update `context.processed_maps_details` with results from the `UnifiedSaveUtility`, including notes about any conversions/inversions performed or merge task failures. + +**III. Unified Image Save Utility (New file: `processing/utils/image_saving_utils.py`)** + +* **Objective:** Centralize all image saving logic (resolution variants, format, bit depth, compression). +* **Interface (e.g., `save_image_variants` function):** + * **Inputs:** + * `source_image_data (np.ndarray)`: High-res image data (in memory, potentially transformed). + * `base_map_type (str)`: Final map type (e.g., "COL", "ROUGH", "NORMAL", "MAP_NRMRGH"). + * `source_bit_depth_info (list)`: List of original source bit depth(s). + * Specific config values (e.g., `image_resolutions: dict`, `file_type_defs: dict`, `output_format_8bit: str`, etc.). + * `output_filename_pattern_tokens (dict)`. + * `output_base_directory (Path)`. + * **Core Functionality:** + 1. Use provided configuration inputs. + 2. Determine Target Bit Depth: + * Use `bit_depth_rule` for `base_map_type` from `file_type_defs`. + * If "force_8bit": target 8-bit. + * If "respect_inputs": If `any(depth > 8 for depth in source_bit_depth_info)`, target 16-bit, else 8-bit. + 3. Determine Output File Format(s) (based on target bit depth, config). + 4. Generate and Save Resolution Variants: + * Iterate through `image_resolutions`. + * Resize `source_image_data` (in memory) for each variant (no upscaling). + * Construct filename and path. + * Prepare save parameters. + * Convert variant data to target bit depth/color space just before saving. + * Save variant using `cv2.imwrite` or similar. + * Discard in-memory variant after saving. + 5. Return List of Saved File Details: `{'path': str, 'resolution_key': str, 'format': str, 'bit_depth': int, 'dimensions': (w,h)}`. + * **Memory Management:** Holds `source_image_data` + one variant in memory at a time. + +**IV. Configuration Changes (`config/app_settings.json`)** + +1. **Add/Confirm Settings:** + * `"INITIAL_SCALING_MODE": "POT_DOWNSCALE"` (Options: "POT_DOWNSCALE", "NONE"). + * `"MERGE_DIMENSION_MISMATCH_STRATEGY": "USE_LARGEST"` (Options: "USE_LARGEST", "USE_FIRST", "ERROR_SKIP"). + * Ensure `general_settings.invert_normal_map_green_channel_globally` exists (boolean). +2. **Review/Confirm Existing Settings:** + * Ensure `IMAGE_RESOLUTIONS`, `FILE_TYPE_DEFINITIONS` (`bit_depth_rule`), `MAP_MERGE_RULES` (`output_bit_depth`, fallback values), format settings, quality settings are comprehensive. +3. **Remove Obsolete Setting:** + * `RESPECT_VARIANT_MAP_TYPES`. + +**V. Data Flow Diagram (Mermaid)** + +```mermaid +graph TD + A[Start Asset Processing] --> B[File Rules Filter]; + B --> STAGE_INDIVIDUAL_MAP_PROCESSING[Individual Map Processing Stage]; + + subgraph STAGE_INDIVIDUAL_MAP_PROCESSING [Individual Map Processing Stage] + direction LR + C1{Is it a regular map or merged task?} + C1 -- Regular Map --> C2[Load Source Image File into Memory (current_image_data)]; + C1 -- Merged Task (from Map Merging Stage) --> C3[Load Inputs (Handle Missing w/ Fallbacks) & Merge in Memory (Handle Dim Mismatch) (current_image_data)]; + + C2 --> C4[current_image_data]; + C3 --> C4; + + C4 --> C4_TRANSFORM{Transformations?}; + C4_TRANSFORM -- Gloss Map? --> C4a[Invert Data (in memory), Update base_map_type to ROUGH]; + C4_TRANSFORM -- Normal Map & Invert Config? --> C4b[Invert Green Channel (in memory)]; + C4_TRANSFORM -- No Transformation Needed --> C4_POST_TRANSFORM; + C4a --> C4_POST_TRANSFORM; + C4b --> C4_POST_TRANSFORM; + + C4_POST_TRANSFORM[current_image_data (potentially transformed)] --> C5{INITIAL_SCALING_MODE}; + C5 -- "POT_DOWNSCALE" --> C6[Perform POT Scale (in memory) --> image_to_save]; + C5 -- "NONE" --> C7[image_to_save = current_image_data]; + + C6 --> C8[Apply Color Management to image_to_save (in memory)]; + C7 --> C8; + + C8 --> UNIFIED_SAVE_UTILITY[Call Unified Save Utility with image_to_save, final base_map_type, source bit depth info, config]; + end + + UNIFIED_SAVE_UTILITY --> H[Update context.processed_maps_details with list of saved files & notes]; + H --> STAGE_METADATA_SAVE[Metadata Finalization & Save Stage]; + + STAGE_MAP_MERGING[Map Merging Stage] --> N{Identify Merge Rules}; + N --> O[Create Merged Image Tasks (incl. inputs, config, source bit depths)]; + O --> STAGE_INDIVIDUAL_MAP_PROCESSING; %% Feed tasks + + A --> STAGE_OTHER_INITIAL[Other Initial Stages] + STAGE_OTHER_INITIAL --> STAGE_MAP_MERGING; + + STAGE_METADATA_SAVE --> Z[End Asset Processing]; + + subgraph UNIFIED_SAVE_UTILITY_DETAILS [Unified Save Utility (processing.utils.image_saving_utils)] + direction TB + INPUTS[Input: in-memory image_to_save, final base_map_type, source_bit_depth_info, config_params, tokens, out_base_dir] + INPUTS --> CONFIG_LOAD[1. Use Provided Config Params] + CONFIG_LOAD --> DETERMINE_BIT_DEPTH[2. Determine Target Bit Depth (using rule & source_bit_depth_info)] + DETERMINE_BIT_DEPTH --> DETERMINE_FORMAT[3. Determine Output Format] + DETERMINE_FORMAT --> LOOP_VARIANTS[4. For each Resolution:] + LOOP_VARIANTS --> RESIZE_VARIANT[4a. Resize image_to_save to Variant (in memory)] + RESIZE_VARIANT --> PREPARE_SAVE[4b. Prepare Filename & Save Params] + PREPARE_SAVE --> SAVE_IMAGE[4c. Convert & Save Variant to Disk] + SAVE_IMAGE --> LOOP_VARIANTS; + LOOP_VARIANTS --> OUTPUT_LIST[5. Return List of Saved File Details] + end + + style STAGE_INDIVIDUAL_MAP_PROCESSING fill:#f9f,stroke:#333,stroke-width:2px; + style STAGE_MAP_MERGING fill:#f9f,stroke:#333,stroke-width:2px; + style UNIFIED_SAVE_UTILITY fill:#ccf,stroke:#333,stroke-width:2px; + style UNIFIED_SAVE_UTILITY_DETAILS fill:#ccf,stroke:#333,stroke-width:1px,dashed; + style O fill:#lightgrey,stroke:#333,stroke-width:2px; + style C4_POST_TRANSFORM fill:#e6ffe6,stroke:#333,stroke-width:1px; \ No newline at end of file diff --git a/ProjectNotes/ProcessingEngineRefactorPlan.md b/ProjectNotes/ProcessingEngineRefactorPlan.md deleted file mode 100644 index 1364ab6..0000000 --- a/ProjectNotes/ProcessingEngineRefactorPlan.md +++ /dev/null @@ -1,181 +0,0 @@ -# Project Plan: Modularizing the Asset Processing Engine - -**Last Updated:** May 9, 2025 - -**1. Project Vision & Goals** - -* **Vision:** Transform the asset processing pipeline into a highly modular, extensible, and testable system. -* **Primary Goals:** - 1. Decouple processing steps into independent, reusable stages. - 2. Simplify the addition of new processing capabilities (e.g., GLOSS > ROUGH conversion, Alpha to MASK, Normal Map Green Channel inversion). - 3. Improve code maintainability and readability. - 4. Enhance unit and integration testing capabilities for each processing component. - 5. Centralize common utility functions (image manipulation, path generation). - -**2. Proposed Architecture Overview** - -* **Core Concept:** A `PipelineOrchestrator` will manage a sequence of `ProcessingStage`s. Each stage will operate on an `AssetProcessingContext` object, which carries all necessary data and state for a single asset through the pipeline. -* **Key Components:** - * `AssetProcessingContext`: Data class holding asset-specific data, configuration, temporary paths, and status. - * `PipelineOrchestrator`: Class to manage the overall processing flow for a `SourceRule`, iterating through assets and executing the pipeline of stages for each. - * `ProcessingStage` (Base Class/Interface): Defines the contract for all individual processing stages (e.g., `execute(context)` method). - * Specific Stage Classes: (e.g., `SupplierDeterminationStage`, `IndividualMapProcessingStage`, etc.) - * Utility Modules: `image_processing_utils.py`, enhancements to `utils/path_utils.py`. - -**3. Proposed File Structure** - -* `processing/` - * `pipeline/` - * `__init__.py` - * `asset_context.py` (Defines `AssetProcessingContext`) - * `orchestrator.py` (Defines `PipelineOrchestrator`) - * `stages/` - * `__init__.py` - * `base_stage.py` (Defines `ProcessingStage` interface) - * `supplier_determination.py` - * `asset_skip_logic.py` - * `metadata_initialization.py` - * `file_rule_filter.py` - * `gloss_to_rough_conversion.py` - * `alpha_extraction_to_mask.py` - * `normal_map_green_channel.py` - * `individual_map_processing.py` - * `map_merging.py` - * `metadata_finalization.py` - * `output_organization.py` - * `utils/` - * `__init__.py` - * `image_processing_utils.py` (New module for image functions) -* `utils/` (Top-level existing directory) - * `path_utils.py` (To be enhanced with `sanitize_filename` from `processing_engine.py`) - -**4. Detailed Phases and Tasks** - -**Phase 0: Setup & Core Structures Definition** -*Goal: Establish the foundational classes for the new pipeline.* -* **Task 0.1: Define `AssetProcessingContext`** - * Create `processing/pipeline/asset_context.py`. - * Define the `AssetProcessingContext` data class with fields: `source_rule: SourceRule`, `asset_rule: AssetRule`, `workspace_path: Path`, `engine_temp_dir: Path`, `output_base_path: Path`, `effective_supplier: Optional[str]`, `asset_metadata: Dict`, `processed_maps_details: Dict[str, Dict[str, Dict]]`, `merged_maps_details: Dict[str, Dict[str, Dict]]`, `files_to_process: List[FileRule]`, `loaded_data_cache: Dict`, `config_obj: Configuration`, `status_flags: Dict`, `incrementing_value: Optional[str]`, `sha5_value: Optional[str]`. - * Ensure proper type hinting. -* **Task 0.2: Define `ProcessingStage` Base Class/Interface** - * Create `processing/pipeline/stages/base_stage.py`. - * Define an abstract base class `ProcessingStage` with an abstract method `execute(self, context: AssetProcessingContext) -> AssetProcessingContext`. -* **Task 0.3: Implement Initial `PipelineOrchestrator`** - * Create `processing/pipeline/orchestrator.py`. - * Define the `PipelineOrchestrator` class. - * Implement `__init__(self, config_obj: Configuration, stages: List[ProcessingStage])`. - * Implement `process_source_rule(self, source_rule: SourceRule, workspace_path: Path, output_base_path: Path, overwrite: bool, incrementing_value: Optional[str], sha5_value: Optional[str]) -> Dict[str, List[str]]`. - * Handles creation/cleanup of the main engine temporary directory. - * Loops through `source_rule.assets`, initializes `AssetProcessingContext` for each. - * Iterates `self.stages`, calling `stage.execute(context)`. - * Collects overall status. - -**Phase 1: Utility Module Refactoring** -*Goal: Consolidate and centralize common utility functions.* -* **Task 1.1: Refactor Path Utilities** - * Move `_sanitize_filename` from `processing_engine.py` to `utils/path_utils.py`. - * Update uses to call the new utility function. -* **Task 1.2: Create `image_processing_utils.py`** - * Create `processing/utils/image_processing_utils.py`. - * Move general-purpose image functions from `processing_engine.py`: - * `is_power_of_two` - * `get_nearest_pot` - * `calculate_target_dimensions` - * `calculate_image_stats` - * `normalize_aspect_ratio_change` - * Core image loading, BGR<>RGB conversion, generic resizing (from `_load_and_transform_source`). - * Core data type conversion for saving, color conversion for saving, `cv2.imwrite` call (from `_save_image`). - * Ensure functions are pure and testable. - -**Phase 2: Implementing Core Processing Stages (Migrating Existing Logic)** -*Goal: Migrate existing functionalities from `processing_engine.py` into the new stage-based architecture.* -(For each task: create stage file, implement class, move logic, adapt to `AssetProcessingContext`) -* **Task 2.1: Implement `SupplierDeterminationStage`** -* **Task 2.2: Implement `AssetSkipLogicStage`** -* **Task 2.3: Implement `MetadataInitializationStage`** -* **Task 2.4: Implement `FileRuleFilterStage`** (New logic for `item_type == "FILE_IGNORE"`) -* **Task 2.5: Implement `IndividualMapProcessingStage`** (Adapts `_process_individual_maps`, uses `image_processing_utils.py`) -* **Task 2.6: Implement `MapMergingStage`** (Adapts `_merge_maps`, uses `image_processing_utils.py`) -* **Task 2.7: Implement `MetadataFinalizationAndSaveStage`** (Adapts `_generate_metadata_file`, uses `utils.path_utils.generate_path_from_pattern`) -* **Task 2.8: Implement `OutputOrganizationStage`** (Adapts `_organize_output_files`) - -**Phase 3: Implementing New Feature Stages** -*Goal: Add the new desired processing capabilities as distinct stages.* -* **Task 3.1: Implement `GlossToRoughConversionStage`** (Identify gloss, convert, invert, save temp, update `FileRule`) -* **Task 3.2: Implement `AlphaExtractionToMaskStage`** (Check existing mask, find MAP_COL with alpha, extract, save temp, add new `FileRule`) -* **Task 3.3: Implement `NormalMapGreenChannelStage`** (Identify normal maps, invert green based on config, save temp, update `FileRule`) - -**Phase 4: Integration, Testing & Finalization** -*Goal: Assemble the pipeline, test thoroughly, and deprecate old code.* -* **Task 4.1: Configure `PipelineOrchestrator`** - * Instantiate `PipelineOrchestrator` in main application logic with the ordered list of stage instances. -* **Task 4.2: Unit Testing** - * Unit tests for each `ProcessingStage` (mocking `AssetProcessingContext`). - * Unit tests for `image_processing_utils.py` and `utils/path_utils.py` functions. -* **Task 4.3: Integration Testing** - * Test `PipelineOrchestrator` end-to-end with sample data. - * Compare outputs with the existing engine for consistency. -* **Task 4.4: Documentation Update** - * Update developer documentation (e.g., `Documentation/02_Developer_Guide/05_Processing_Pipeline.md`). - * Document `AssetProcessingContext` and stage responsibilities. -* **Task 4.5: Deprecate/Remove Old `ProcessingEngine` Code** - * Gradually remove refactored logic from `processing_engine.py`. - -**5. Workflow Diagram** - -```mermaid -graph TD - AA[Load SourceRule & Config] --> BA(PipelineOrchestrator: process_source_rule); - BA --> CA{For Each Asset in SourceRule}; - CA -- Yes --> DA(Orchestrator: Create AssetProcessingContext); - DA --> EA(SupplierDeterminationStage); - EA -- context --> FA(AssetSkipLogicStage); - FA -- context --> GA{context.skip_asset?}; - GA -- Yes --> HA(Orchestrator: Record Skipped); - HA --> CA; - GA -- No --> IA(MetadataInitializationStage); - IA -- context --> JA(FileRuleFilterStage); - JA -- context --> KA(GlossToRoughConversionStage); - KA -- context --> LA(AlphaExtractionToMaskStage); - LA -- context --> MA(NormalMapGreenChannelStage); - MA -- context --> NA(IndividualMapProcessingStage); - NA -- context --> OA(MapMergingStage); - OA -- context --> PA(MetadataFinalizationAndSaveStage); - PA -- context --> QA(OutputOrganizationStage); - QA -- context --> RA(Orchestrator: Record Processed/Failed); - RA --> CA; - CA -- No --> SA(Orchestrator: Cleanup Engine Temp Dir); - SA --> TA[Processing Complete]; - - subgraph Stages - direction LR - EA - FA - IA - JA - KA - LA - MA - NA - OA - PA - QA - end - - subgraph Utils - direction LR - U1[image_processing_utils.py] - U2[utils/path_utils.py] - end - - NA -.-> U1; - OA -.-> U1; - KA -.-> U1; - LA -.-> U1; - MA -.-> U1; - - PA -.-> U2; - QA -.-> U2; - - classDef context fill:#f9f,stroke:#333,stroke-width:2px; - class DA,EA,FA,IA,JA,KA,LA,MA,NA,OA,PA,QA context; \ No newline at end of file diff --git a/config/app_settings.json b/config/app_settings.json index cec7393..efe09ac 100644 --- a/config/app_settings.json +++ b/config/app_settings.json @@ -284,5 +284,10 @@ ], "CALCULATE_STATS_RESOLUTION": "1K", "DEFAULT_ASSET_CATEGORY": "Surface", - "TEMP_DIR_PREFIX": "_PROCESS_ASSET_" + "TEMP_DIR_PREFIX": "_PROCESS_ASSET_", + "INITIAL_SCALING_MODE": "POT_DOWNSCALE", + "MERGE_DIMENSION_MISMATCH_STRATEGY": "USE_LARGEST", + "general_settings": { + "invert_normal_map_green_channel_globally": false + } } \ No newline at end of file diff --git a/processing/pipeline/stages/individual_map_processing.py b/processing/pipeline/stages/individual_map_processing.py index acbe8bd..21b6d47 100644 --- a/processing/pipeline/stages/individual_map_processing.py +++ b/processing/pipeline/stages/individual_map_processing.py @@ -4,7 +4,7 @@ import re import os import logging from pathlib import Path -from typing import Optional, Tuple, Dict +from typing import Optional, Tuple, Dict, List, Any, Union import cv2 import numpy as np @@ -13,21 +13,58 @@ from .base_stage import ProcessingStage from ..asset_context import AssetProcessingContext from rule_structure import FileRule from utils.path_utils import sanitize_filename -from ...utils import image_processing_utils as ipu +from ...utils import image_processing_utils as ipu # Includes get_image_bit_depth implicitly now +from ...utils.image_saving_utils import save_image_variants # Added import logger = logging.getLogger(__name__) +# Helper function to get filename-friendly map type (adapted from old logic) +def get_filename_friendly_map_type(internal_map_type: str, file_type_definitions: Optional[Dict[str, Dict]]) -> str: + """Derives a filename-friendly map type from the internal map type.""" + filename_friendly_map_type = internal_map_type # Fallback + if not file_type_definitions or not isinstance(file_type_definitions, dict) or not file_type_definitions: + logger.warning(f"Filename-friendly lookup: FILE_TYPE_DEFINITIONS not available or invalid. Falling back to internal type: {internal_map_type}") + return filename_friendly_map_type + + base_map_key_val = None + suffix_part = "" + sorted_known_base_keys = sorted(list(file_type_definitions.keys()), key=len, reverse=True) + + for known_key in sorted_known_base_keys: + if internal_map_type.startswith(known_key): + base_map_key_val = known_key + suffix_part = internal_map_type[len(known_key):] + break + + if base_map_key_val: + definition = file_type_definitions.get(base_map_key_val) + if definition and isinstance(definition, dict): + standard_type_alias = definition.get("standard_type") + if standard_type_alias and isinstance(standard_type_alias, str) and standard_type_alias.strip(): + filename_friendly_map_type = standard_type_alias.strip() + suffix_part + logger.debug(f"Filename-friendly lookup: Transformed '{internal_map_type}' -> '{filename_friendly_map_type}'") + else: + logger.warning(f"Filename-friendly lookup: Standard type alias for '{base_map_key_val}' is missing or invalid. Falling back.") + else: + logger.warning(f"Filename-friendly lookup: No valid definition for '{base_map_key_val}'. Falling back.") + else: + logger.warning(f"Filename-friendly lookup: Could not parse base key from '{internal_map_type}'. Falling back.") + + return filename_friendly_map_type + + class IndividualMapProcessingStage(ProcessingStage): """ - Processes individual texture map files based on FileRules. - This stage finds the source file, loads it, applies transformations - (resize, color space), saves a temporary processed version, and updates - the AssetProcessingContext with details. + Processes individual texture maps and merged map tasks. + This stage loads source images (or merges inputs for tasks), performs + in-memory transformations (Gloss-to-Rough, Normal Green Invert, optional scaling), + and passes the result to the UnifiedSaveUtility for final output generation. + It updates the AssetProcessingContext with detailed results. """ def execute(self, context: AssetProcessingContext) -> AssetProcessingContext: """ - Executes the individual map processing logic. + Executes the individual map and merged task processing logic. """ asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" if context.status_flags.get('skip_asset', False): @@ -38,663 +75,549 @@ class IndividualMapProcessingStage(ProcessingStage): context.processed_maps_details = {} logger.debug(f"Asset '{asset_name_for_log}': Initialized processed_maps_details.") - if not context.files_to_process: - logger.info(f"Asset '{asset_name_for_log}': No files to process in this stage.") - return context - - # Source path for the asset group comes from SourceRule - if not context.source_rule or not context.source_rule.input_path: - logger.error(f"Asset '{asset_name_for_log}': SourceRule or SourceRule.input_path is not set. Cannot determine source base path.") - context.status_flags['individual_map_processing_failed'] = True - # Mark all file_rules as failed - for fr_idx, file_rule_to_fail in enumerate(context.files_to_process): - # Use fr_idx as the key for status update for these early failures - map_type_for_fail = file_rule_to_fail.item_type_override or file_rule_to_fail.item_type or "UnknownMapType" - self._update_file_rule_status(context, fr_idx, 'Failed', map_type=map_type_for_fail, details="SourceRule.input_path missing") - return context - - # The workspace_path in the context should be the directory where files are extracted/available. - source_base_path = context.workspace_path - if not source_base_path.is_dir(): - logger.error(f"Asset '{asset_name_for_log}': Workspace path '{source_base_path}' is not a valid directory.") - context.status_flags['individual_map_processing_failed'] = True - for fr_idx, file_rule_to_fail in enumerate(context.files_to_process): - # Use fr_idx as the key for status update - map_type_for_fail = file_rule_to_fail.item_type_override or file_rule_to_fail.item_type or "UnknownMapType" - self._update_file_rule_status(context, fr_idx, 'Failed', map_type=map_type_for_fail, details="Workspace path invalid") - return context + # --- Configuration Fetching --- + config = context.config_obj + file_type_definitions = getattr(config, "FILE_TYPE_DEFINITIONS", {}) + respect_variant_map_types = getattr(config, "respect_variant_map_types", []) # Needed for suffixing logic + initial_scaling_mode = getattr(config, "INITIAL_SCALING_MODE", "NONE") + merge_dimension_mismatch_strategy = getattr(config, "MERGE_DIMENSION_MISMATCH_STRATEGY", "USE_LARGEST") + invert_normal_green = getattr(config.general_settings, "invert_normal_map_green_channel_globally", False) + output_base_dir = context.output_dir # Assuming output_dir is set in context + asset_name = context.asset_rule.asset_name if context.asset_rule else "UnknownAsset" + output_filename_pattern_tokens = {'asset_name': asset_name, 'output_base_directory': str(output_base_dir)} - # Fetch config settings once before the loop - respect_variant_map_types = getattr(context.config_obj, "respect_variant_map_types", []) - image_resolutions = getattr(context.config_obj, "image_resolutions", {}) - output_filename_pattern = getattr(context.config_obj, "output_filename_pattern", "[assetname]_[maptype]_[resolution].[ext]") + # --- Prepare Items to Process --- + items_to_process: List[Union[Tuple[int, FileRule], Tuple[str, Dict]]] = [] - for file_rule_idx, file_rule in enumerate(context.files_to_process): - # file_rule_idx will be the key for processed_maps_details. - # processing_instance_tag is for unique temp files and detailed logging for this specific run. - processing_instance_tag = f"map_{file_rule_idx}_{uuid.uuid4().hex[:8]}" - current_map_key = file_rule_idx # Key for processed_maps_details + # Add regular files + if context.files_to_process: + # Validate source path early for regular files + if not context.source_rule or not context.source_rule.input_path: + logger.error(f"Asset '{asset_name_for_log}': SourceRule or SourceRule.input_path is not set. Cannot process regular files.") + context.status_flags['individual_map_processing_failed'] = True + # Mark all file_rules as failed if source path is missing + for fr_idx, file_rule_to_fail in enumerate(context.files_to_process): + map_type_for_fail = file_rule_to_fail.item_type_override or file_rule_to_fail.item_type or "UnknownMapType" + ff_map_type = get_filename_friendly_map_type(map_type_for_fail, file_type_definitions) + context.processed_maps_details[fr_idx] = { + 'status': 'Failed', + 'map_type': ff_map_type, + 'processing_map_type': map_type_for_fail, + 'notes': "SourceRule.input_path missing", + 'saved_files_info': [] + } + # Don't add regular files if source path is bad + elif not context.workspace_path or not context.workspace_path.is_dir(): + logger.error(f"Asset '{asset_name_for_log}': Workspace path '{context.workspace_path}' is not a valid directory. Cannot process regular files.") + context.status_flags['individual_map_processing_failed'] = True + for fr_idx, file_rule_to_fail in enumerate(context.files_to_process): + map_type_for_fail = file_rule_to_fail.item_type_override or file_rule_to_fail.item_type or "UnknownMapType" + ff_map_type = get_filename_friendly_map_type(map_type_for_fail, file_type_definitions) + context.processed_maps_details[fr_idx] = { + 'status': 'Failed', + 'map_type': ff_map_type, + 'processing_map_type': map_type_for_fail, + 'notes': "Workspace path invalid", + 'saved_files_info': [] + } + # Don't add regular files if workspace path is bad + else: + for idx, file_rule in enumerate(context.files_to_process): + items_to_process.append((idx, file_rule)) - if not file_rule.file_path: # Ensure file_path exists, critical for later stages if they rely on it from FileRule - logger.error(f"Asset '{asset_name_for_log}', FileRule at index {file_rule_idx} has an empty or None file_path. Skipping this rule.") - self._update_file_rule_status(context, current_map_key, 'Failed', - processing_tag=processing_instance_tag, - details="FileRule has no file_path") - continue - - initial_current_map_type = file_rule.item_type_override or file_rule.item_type or "UnknownMapType" + # Add merged tasks + if hasattr(context, 'merged_image_tasks') and context.merged_image_tasks: + for task_idx, task_data in enumerate(context.merged_image_tasks): + task_key = f"merged_task_{task_idx}" + items_to_process.append((task_key, task_data)) - # --- START NEW SUFFIXING LOGIC --- - final_current_map_type = initial_current_map_type # Default to initial - - # 1. Determine Base Map Type from initial_current_map_type - base_map_type_match = re.match(r"(MAP_[A-Z]{3})", initial_current_map_type) - - if base_map_type_match and context.asset_rule: - true_base_map_type = base_map_type_match.group(1) # This is "MAP_XXX" + if not items_to_process: + logger.info(f"Asset '{asset_name_for_log}': No regular files or merged tasks to process in this stage.") + return context - # 2. Count Occurrences and Find Index of current_file_rule in context.asset_rule.files - peers_of_same_base_type_in_asset_rule = [] - for fr_asset in context.asset_rule.files: - fr_asset_item_type = fr_asset.item_type_override or fr_asset.item_type or "UnknownMapType" - fr_asset_base_map_type_match = re.match(r"(MAP_[A-Z]{3})", fr_asset_item_type) - - if fr_asset_base_map_type_match: - fr_asset_base_map_type = fr_asset_base_map_type_match.group(1) - if fr_asset_base_map_type == true_base_map_type: - peers_of_same_base_type_in_asset_rule.append(fr_asset) - - num_occurrences_of_base_type = len(peers_of_same_base_type_in_asset_rule) - current_instance_index = 0 # 1-based + # --- Unified Processing Loop --- + for item_key, item_data in items_to_process: + current_image_data: Optional[np.ndarray] = None + base_map_type: str = "Unknown" # Filename-friendly + processing_map_type: str = "Unknown" # Internal MAP_XXX type + source_bit_depth_info_for_save_util: List[int] = [] + is_merged_task: bool = False + status_notes: List[str] = [] + processing_status: str = "Started" + saved_files_details_list: List[Dict] = [] + original_dimensions: Optional[Tuple[int, int]] = None + source_file_path_regular: Optional[Path] = None # For regular maps + merge_task_config_output_type: Optional[str] = None # For merged tasks + inputs_used_for_merge: Optional[Dict[str, str]] = None # For merged tasks + processing_instance_tag = f"item_{item_key}_{uuid.uuid4().hex[:8]}" # Unique tag for logging this item - try: - current_instance_index = peers_of_same_base_type_in_asset_rule.index(file_rule) + 1 - except ValueError: - logger.warning( - f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Initial Type: '{initial_current_map_type}', Base: '{true_base_map_type}'): " - f"Could not find its own instance in the list of peers from asset_rule.files. " - f"Number of peers found: {num_occurrences_of_base_type}. Suffixing may be affected." - ) - - # 3. Determine Suffix - map_type_for_respect_check = true_base_map_type.replace("MAP_", "") # e.g., "COL" - is_in_respect_list = map_type_for_respect_check in respect_variant_map_types - - suffix_to_append = "" - if num_occurrences_of_base_type > 1: - if current_instance_index > 0: - suffix_to_append = f"-{current_instance_index}" - else: - logger.warning(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}': Index for multi-occurrence map type '{true_base_map_type}' (count: {num_occurrences_of_base_type}) not determined. Omitting numeric suffix.") - elif num_occurrences_of_base_type == 1 and is_in_respect_list: - suffix_to_append = "-1" - - # 4. Form the final_current_map_type - if suffix_to_append: - final_current_map_type = true_base_map_type + suffix_to_append - else: - final_current_map_type = initial_current_map_type - - current_map_type = final_current_map_type - # --- END NEW SUFFIXING LOGIC --- - - # --- START: Filename-friendly map type derivation --- - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: --- Starting Filename-Friendly Map Type Logic for: {current_map_type} ---") - filename_friendly_map_type = current_map_type # Fallback - - # 1. Access FILE_TYPE_DEFINITIONS - file_type_definitions = None - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Attempting to access context.config_obj.FILE_TYPE_DEFINITIONS.") try: - file_type_definitions = context.config_obj.FILE_TYPE_DEFINITIONS - if not file_type_definitions: # Check if it's None or empty - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS is present but empty or None.") - else: - sample_defs_log = {k: file_type_definitions[k] for k in list(file_type_definitions.keys())[:2]} # Log first 2 for brevity - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Accessed FILE_TYPE_DEFINITIONS. Sample: {sample_defs_log}, Total keys: {len(file_type_definitions)}.") - except AttributeError: - logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Could not access context.config_obj.FILE_TYPE_DEFINITIONS via direct attribute.") - - base_map_key_val = None # Renamed from base_map_key to avoid conflict with current_map_key - suffix_part = "" + # --- A. Regular Map Processing --- + if isinstance(item_data, FileRule): + file_rule: FileRule = item_data + file_rule_idx: int = item_key # Key is the index for regular maps + is_merged_task = False + logger.info(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: Processing Regular Map from FileRule: {file_rule.file_path}") - if file_type_definitions and isinstance(file_type_definitions, dict) and len(file_type_definitions) > 0: - base_map_key_val = None - suffix_part = "" - - sorted_known_base_keys = sorted(list(file_type_definitions.keys()), key=len, reverse=True) - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Sorted known base keys for parsing: {sorted_known_base_keys}") + if not file_rule.file_path: + logger.error(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: FileRule has an empty or None file_path. Skipping.") + processing_status = "Failed" + status_notes.append("FileRule has no file_path") + continue # To finally block - for known_key in sorted_known_base_keys: - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Checking if '{current_map_type}' starts with '{known_key}'") - if current_map_type.startswith(known_key): - base_map_key_val = known_key - suffix_part = current_map_type[len(known_key):] - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Match found! current_map_type: '{current_map_type}', base_map_key_val: '{base_map_key_val}', suffix_part: '{suffix_part}'") - break + # Determine internal map type (MAP_XXX) with suffixing + initial_internal_map_type = file_rule.item_type_override or file_rule.item_type or "UnknownMapType" + processing_map_type = self._get_suffixed_internal_map_type(context, file_rule, initial_internal_map_type, respect_variant_map_types) + base_map_type = get_filename_friendly_map_type(processing_map_type, file_type_definitions) # Get filename friendly version - if base_map_key_val is None: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Could not parse base_map_key_val from '{current_map_type}' using known keys. Fallback: filename_friendly_map_type = '{filename_friendly_map_type}'.") - else: - definition = file_type_definitions.get(base_map_key_val) - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Definition for '{base_map_key_val}': {definition}") - if definition and isinstance(definition, dict): - standard_type_alias = definition.get("standard_type") - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Standard type alias for '{base_map_key_val}': '{standard_type_alias}'") - if standard_type_alias and isinstance(standard_type_alias, str) and standard_type_alias.strip(): - filename_friendly_map_type = standard_type_alias.strip() + suffix_part - logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Successfully transformed map type: '{current_map_type}' -> '{filename_friendly_map_type}' (standard_type_alias: '{standard_type_alias}', suffix_part: '{suffix_part}').") + # Skip types not meant for individual processing (e.g., composites handled elsewhere) + if not processing_map_type or not processing_map_type.startswith("MAP_") or processing_map_type == "MAP_GEN_COMPOSITE": + logger.debug(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: Skipping, type '{processing_map_type}' (Filename: '{base_map_type}') not targeted for individual processing.") + processing_status = "Skipped" + status_notes.append(f"Type '{processing_map_type}' not processed individually.") + continue # To finally block + + # Find source file (relative to workspace_path) + source_base_path = context.workspace_path + # Use the file_rule.file_path directly as it should be relative now + potential_source_path = source_base_path / file_rule.file_path + if potential_source_path.is_file(): + source_file_path_regular = potential_source_path + logger.info(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: Found source file: {source_file_path_regular}") + else: + # Attempt globbing as a fallback if direct path fails (optional, based on previous logic) + found_files = list(source_base_path.glob(file_rule.file_path)) + if len(found_files) == 1: + source_file_path_regular = found_files[0] + logger.info(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: Found source file via glob: {source_file_path_regular}") + elif len(found_files) > 1: + logger.warning(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: Multiple files found for pattern '{file_rule.file_path}' in '{source_base_path}'. Using first: {found_files[0]}") + source_file_path_regular = found_files[0] else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Standard type alias for '{base_map_key_val}' is missing, empty, or not a string (value: '{standard_type_alias}'). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") - else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: No definition or invalid definition for '{base_map_key_val}' (value: {definition}). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") - elif file_type_definitions is None: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS not available for lookup (was None). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") - elif not isinstance(file_type_definitions, dict): - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS is not a dictionary (type: {type(file_type_definitions)}). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") - else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS is an empty dictionary. Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") - - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Final filename_friendly_map_type: '{filename_friendly_map_type}'") - # --- END: Filename-friendly map type derivation --- + logger.error(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: Source file not found using path/pattern '{file_rule.file_path}' in '{source_base_path}'.") + processing_status = "Failed" + status_notes.append("Source file not found") + continue # To finally block - if not current_map_type or not current_map_type.startswith("MAP_") or current_map_type == "MAP_GEN_COMPOSITE": - logger.debug(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}': Skipping, item_type '{current_map_type}' (initial: '{initial_current_map_type}') not targeted for individual processing.") - continue + # Load image + source_image_data = ipu.load_image(str(source_file_path_regular)) + if source_image_data is None: + logger.error(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: Failed to load image from '{source_file_path_regular}'.") + processing_status = "Failed" + status_notes.append("Image load failed") + continue # To finally block - logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Type: {current_map_type}, Initial Type: {initial_current_map_type}, Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Starting individual processing.") + original_height, original_width = source_image_data.shape[:2] + original_dimensions = (original_width, original_height) + logger.debug(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: Loaded image with dimensions {original_width}x{original_height}.") - # A. Find Source File (using file_rule.file_path as the pattern relative to source_base_path) - source_file_path = self._find_source_file(source_base_path, file_rule.file_path, asset_name_for_log, processing_instance_tag) - if not source_file_path: - logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Source file not found with path/pattern '{file_rule.file_path}' in '{source_base_path}'.") - self._update_file_rule_status(context, current_map_key, 'Failed', - map_type=filename_friendly_map_type, - processing_map_type=current_map_type, - source_file_rule_index=file_rule_idx, - processing_tag=processing_instance_tag, - details="Source file not found") - continue + # Get original bit depth + try: + original_source_bit_depth = ipu.get_image_bit_depth(str(source_file_path_regular)) + source_bit_depth_info_for_save_util = [original_source_bit_depth] + logger.info(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: Determined source bit depth: {original_source_bit_depth}") + except Exception as e: + logger.warning(f"Asset '{asset_name_for_log}', Key {file_rule_idx}, Proc. Tag {processing_instance_tag}: Could not determine source bit depth for {source_file_path_regular}: {e}. Using default [8].") + source_bit_depth_info_for_save_util = [8] # Default fallback + status_notes.append("Could not determine source bit depth, defaulted to 8.") - # B. Load and Transform Image - image_data: Optional[np.ndarray] = ipu.load_image(str(source_file_path)) - if image_data is None: - logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Failed to load image from '{source_file_path}'.") - self._update_file_rule_status(context, current_map_key, 'Failed', - map_type=filename_friendly_map_type, - processing_map_type=current_map_type, - source_file_rule_index=file_rule_idx, - processing_tag=processing_instance_tag, - source_file=str(source_file_path), - details="Image load failed") - continue - - original_height, original_width = image_data.shape[:2] - logger.debug(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Loaded image '{source_file_path}' with dimensions {original_width}x{original_height}.") + current_image_data = source_image_data.copy() - # 1. Initial Power-of-Two (POT) Downscaling - pot_width = ipu.get_nearest_power_of_two_downscale(original_width) - pot_height = ipu.get_nearest_power_of_two_downscale(original_height) + # --- B. Merged Image Task Processing --- + elif isinstance(item_data, dict): + task: Dict = item_data + task_key: str = item_key # Key is the generated string for merged tasks + is_merged_task = True + merge_task_config_output_type = task.get('output_map_type', 'UnknownMergeOutput') + logger.info(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Processing Merged Task for output type: {merge_task_config_output_type}") - # Maintain aspect ratio for initial POT scaling, using the smaller of the scaled dimensions - # This ensures we only downscale. - if original_width > 0 and original_height > 0 : # Avoid division by zero - aspect_ratio = original_width / original_height - - # Calculate new dimensions based on POT width, then POT height, and pick the one that results in downscale or same size - pot_h_from_w = int(pot_width / aspect_ratio) - pot_w_from_h = int(pot_height * aspect_ratio) + processing_map_type = merge_task_config_output_type # Internal type is the output type from config + base_map_type = get_filename_friendly_map_type(processing_map_type, file_type_definitions) # Get filename friendly version + source_bit_depth_info_for_save_util = task.get('source_bit_depths', []) + merge_rule_config = task.get('merge_rule_config', {}) + input_map_sources = task.get('input_map_sources', {}) + target_dimensions = task.get('source_dimensions') # Expected dimensions (h, w) - # Option 1: Scale by width, adjust height - candidate1_w, candidate1_h = pot_width, ipu.get_nearest_power_of_two_downscale(pot_h_from_w) - # Option 2: Scale by height, adjust width - candidate2_w, candidate2_h = ipu.get_nearest_power_of_two_downscale(pot_w_from_h), pot_height - - # Ensure candidates are not upscaling - if candidate1_w > original_width or candidate1_h > original_height: - candidate1_w, candidate1_h = original_width, original_height # Fallback to original if upscaling - if candidate2_w > original_width or candidate2_h > original_height: - candidate2_w, candidate2_h = original_width, original_height # Fallback to original if upscaling - - # Choose the candidate that results in a larger area (preferring less downscaling if multiple POT options) - # but still respects the POT downscale logic for each dimension individually. - # The actual POT dimensions are already calculated by get_nearest_power_of_two_downscale. - # We need to decide if we base the aspect ratio calc on pot_width or pot_height. - # The goal is to make one dimension POT and the other POT while maintaining aspect as much as possible, only downscaling. + if not merge_rule_config or not input_map_sources or not target_dimensions: + logger.error(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Merge task data is incomplete (missing config, sources, or dimensions). Skipping.") + processing_status = "Failed" + status_notes.append("Incomplete merge task data") + continue # To finally block - final_pot_width = ipu.get_nearest_power_of_two_downscale(original_width) - final_pot_height = ipu.get_nearest_power_of_two_downscale(original_height) + loaded_inputs_for_merge: Dict[str, np.ndarray] = {} + actual_input_dimensions: List[Tuple[int, int]] = [] # List of (h, w) + inputs_used_for_merge = {} # Track actual files/fallbacks used - # If original aspect is not 1:1, one of the POT dimensions might need further adjustment to maintain aspect - # after the other dimension is set to its POT. - # We prioritize fitting within the *downscaled* POT dimensions. - - # Scale to fit within final_pot_width, adjust height, then make height POT (downscale) - scaled_h_for_pot_w = max(1, round(final_pot_width / aspect_ratio)) - h1 = ipu.get_nearest_power_of_two_downscale(scaled_h_for_pot_w) - w1 = final_pot_width - if h1 > final_pot_height: # If this adjustment made height too big, re-evaluate - h1 = final_pot_height - w1 = ipu.get_nearest_power_of_two_downscale(max(1, round(h1 * aspect_ratio))) + # Load/Prepare Inputs for Merge + merge_inputs_config = merge_rule_config.get('inputs', {}) + merge_defaults = merge_rule_config.get('defaults', {}) + for channel_char, required_map_type_from_rule in merge_inputs_config.items(): + input_info = input_map_sources.get(required_map_type_from_rule) + input_image_data = None + input_source_desc = f"Fallback for {required_map_type_from_rule}" - # Scale to fit within final_pot_height, adjust width, then make width POT (downscale) - scaled_w_for_pot_h = max(1, round(final_pot_height * aspect_ratio)) - w2 = ipu.get_nearest_power_of_two_downscale(scaled_w_for_pot_h) - h2 = final_pot_height - if w2 > final_pot_width: # If this adjustment made width too big, re-evaluate - w2 = final_pot_width - h2 = ipu.get_nearest_power_of_two_downscale(max(1, round(w2 / aspect_ratio))) - - # Choose the option that results in larger area (less aggressive downscaling) - # while ensuring both dimensions are POT and not upscaled from original. - if w1 * h1 >= w2 * h2: - base_pot_width, base_pot_height = w1, h1 - else: - base_pot_width, base_pot_height = w2, h2 - - # Final check to ensure no upscaling from original dimensions - base_pot_width = min(base_pot_width, original_width) - base_pot_height = min(base_pot_height, original_height) - # And ensure they are POT - base_pot_width = ipu.get_nearest_power_of_two_downscale(base_pot_width) - base_pot_height = ipu.get_nearest_power_of_two_downscale(base_pot_height) - - else: # Handle cases like 0-dim images, though load_image should prevent this - base_pot_width, base_pot_height = 1, 1 - - - logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Original dims: ({original_width},{original_height}), Initial POT Scaled Dims: ({base_pot_width},{base_pot_height}).") - - # Calculate and store aspect ratio change string - if original_width > 0 and original_height > 0 and base_pot_width > 0 and base_pot_height > 0: - aspect_change_str = ipu.normalize_aspect_ratio_change( - original_width, original_height, - base_pot_width, base_pot_height - ) - if aspect_change_str: - # This will overwrite if multiple maps are processed; specified by requirements. - context.asset_metadata['aspect_ratio_change_string'] = aspect_change_str - logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type {current_map_type}: Calculated aspect ratio change string: '{aspect_change_str}' (Original: {original_width}x{original_height}, Base POT: {base_pot_width}x{base_pot_height}). Stored in asset_metadata.") - else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type {current_map_type}: Failed to calculate aspect ratio change string.") - else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type {current_map_type}: Skipping aspect ratio change string calculation due to invalid dimensions (Original: {original_width}x{original_height}, Base POT: {base_pot_width}x{base_pot_height}).") - - base_pot_image_data = image_data.copy() - if (base_pot_width, base_pot_height) != (original_width, original_height): - interpolation = cv2.INTER_AREA # Good for downscaling - base_pot_image_data = ipu.resize_image(base_pot_image_data, base_pot_width, base_pot_height, interpolation=interpolation) - if base_pot_image_data is None: - logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Failed to resize image to base POT dimensions.") - self._update_file_rule_status(context, current_map_key, 'Failed', - map_type=filename_friendly_map_type, - processing_map_type=current_map_type, - source_file_rule_index=file_rule_idx, - processing_tag=processing_instance_tag, - source_file=str(source_file_path), - original_dimensions=(original_width, original_height), - details="Base POT resize failed") - continue - - # Color Profile Management (after initial POT resize, before multi-res saving) - # Initialize transform settings with defaults for color management - transform_settings = { - "color_profile_management": False, # Default, can be overridden by FileRule - "target_color_profile": "sRGB", # Default - "output_format_settings": None # For JPG quality, PNG compression - } - if file_rule.channel_merge_instructions and 'transform' in file_rule.channel_merge_instructions: - custom_transform_settings = file_rule.channel_merge_instructions['transform'] - if isinstance(custom_transform_settings, dict): - transform_settings.update(custom_transform_settings) - logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Loaded transform settings for color/output from file_rule.") - - if transform_settings['color_profile_management'] and transform_settings['target_color_profile'] == "RGB": - if len(base_pot_image_data.shape) == 3 and base_pot_image_data.shape[2] == 3: # BGR to RGB - logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Converting BGR to RGB for base POT image.") - base_pot_image_data = ipu.convert_bgr_to_rgb(base_pot_image_data) - elif len(base_pot_image_data.shape) == 3 and base_pot_image_data.shape[2] == 4: # BGRA to RGBA - logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Converting BGRA to RGBA for base POT image.") - base_pot_image_data = ipu.convert_bgra_to_rgba(base_pot_image_data) - - # Ensure engine_temp_dir exists before saving base POT - if not context.engine_temp_dir.exists(): - try: - context.engine_temp_dir.mkdir(parents=True, exist_ok=True) - logger.info(f"Asset '{asset_name_for_log}': Created engine_temp_dir at '{context.engine_temp_dir}'") - except OSError as e: - logger.error(f"Asset '{asset_name_for_log}': Failed to create engine_temp_dir '{context.engine_temp_dir}': {e}") - self._update_file_rule_status(context, current_map_key, 'Failed', - map_type=filename_friendly_map_type, - processing_map_type=current_map_type, - source_file_rule_index=file_rule_idx, - processing_tag=processing_instance_tag, - source_file=str(source_file_path), - details="Failed to create temp directory for base POT") - continue - - temp_filename_suffix = Path(source_file_path).suffix - base_pot_temp_filename = f"{processing_instance_tag}_basePOT{temp_filename_suffix}" # Use processing_instance_tag - base_pot_temp_path = context.engine_temp_dir / base_pot_temp_filename - - # Determine save parameters for base POT image (can be different from variants if needed) - base_save_params = [] - base_output_ext = temp_filename_suffix.lstrip('.') # Default to original, can be overridden by format rules - # TODO: Add logic here to determine base_output_ext and base_save_params based on bit depth and config, similar to variants. - # For now, using simple save. - - if not ipu.save_image(str(base_pot_temp_path), base_pot_image_data, params=base_save_params): - logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Failed to save base POT image to '{base_pot_temp_path}'.") - self._update_file_rule_status(context, current_map_key, 'Failed', - map_type=filename_friendly_map_type, - processing_map_type=current_map_type, - source_file_rule_index=file_rule_idx, - processing_tag=processing_instance_tag, - source_file=str(source_file_path), - original_dimensions=(original_width, original_height), - base_pot_dimensions=(base_pot_width, base_pot_height), - details="Base POT image save failed") - continue - - logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Successfully saved base POT image to '{base_pot_temp_path}' with dims ({base_pot_width}x{base_pot_height}).") - - # Initialize/update the status for this map in processed_maps_details - self._update_file_rule_status( - context, - current_map_key, # Use file_rule_idx as key - 'BasePOTSaved', # Intermediate status, will be updated after variant check - map_type=filename_friendly_map_type, - processing_map_type=current_map_type, - source_file_rule_index=file_rule_idx, - processing_tag=processing_instance_tag, # Store the tag - source_file=str(source_file_path), - original_dimensions=(original_width, original_height), - base_pot_dimensions=(base_pot_width, base_pot_height), - temp_processed_file=str(base_pot_temp_path) # Store path to the saved base POT - ) - - # 2. Multiple Resolution Output (Variants) - processed_at_least_one_resolution_variant = False - # Resolution variants are attempted for all map types individually processed. - # The filter at the beginning of the loop ensures only relevant maps reach this stage. - generate_variants_for_this_map_type = True - - if generate_variants_for_this_map_type: # This will now always be true if code execution reaches here - logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Map type '{current_map_type}' is eligible for individual processing. Attempting to generate resolution variants.") - # Sort resolutions from largest to smallest - sorted_resolutions = sorted(image_resolutions.items(), key=lambda item: item[1], reverse=True) - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Sorted resolutions for variant processing: {sorted_resolutions}") - - for res_key, res_max_dim in sorted_resolutions: - current_w, current_h = base_pot_image_data.shape[1], base_pot_image_data.shape[0] - - if current_w <= 0 or current_h <=0: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Base POT image has zero dimension ({current_w}x{current_h}). Skipping this resolution variant.") - continue - - if max(current_w, current_h) >= res_max_dim: - target_w_res, target_h_res = current_w, current_h - if max(current_w, current_h) > res_max_dim: - if current_w >= current_h: - target_w_res = res_max_dim - target_h_res = max(1, round(target_w_res / (current_w / current_h))) + if input_info and input_info.get('file_path'): + # Paths in merged tasks should ideally be absolute or relative to a known base (e.g., workspace) + # Assuming they are resolvable as is for now. + input_file_path = Path(input_info['file_path']) + if input_file_path.is_file(): + try: + input_image_data = ipu.load_image(str(input_file_path)) + if input_image_data is not None: + logger.info(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Loaded input '{required_map_type_from_rule}' for channel '{channel_char}' from: {input_file_path}") + actual_input_dimensions.append(input_image_data.shape[:2]) # (h, w) + input_source_desc = str(input_file_path) + else: + logger.warning(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Failed to load input '{required_map_type_from_rule}' from {input_file_path}. Attempting fallback.") + except Exception as e: + logger.warning(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Error loading input '{required_map_type_from_rule}' from {input_file_path}: {e}. Attempting fallback.") else: - target_h_res = res_max_dim - target_w_res = max(1, round(target_h_res * (current_w / current_h))) + logger.warning(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Input file path for '{required_map_type_from_rule}' not found: {input_file_path}. Attempting fallback.") + else: + logger.warning(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: No file path provided for required input '{required_map_type_from_rule}'. Attempting fallback.") + + # Fallback if load failed or no path + if input_image_data is None: + fallback_value = merge_defaults.get(channel_char) + if fallback_value is not None: + try: + # Determine shape and dtype for fallback + h, w = target_dimensions + # Infer channels needed based on typical usage or config (e.g., RGB default, single channel for masks) + # This might need refinement based on how defaults are structured. Assuming uint8 for now. + # If fallback_value is a single number, assume grayscale, else assume color based on length? + num_channels = 1 if isinstance(fallback_value, (int, float)) else len(fallback_value) if isinstance(fallback_value, (list, tuple)) else 3 # Default to 3? Risky. + dtype = np.uint8 # Default dtype, might need adjustment based on context + shape = (h, w) if num_channels == 1 else (h, w, num_channels) + + input_image_data = np.full(shape, fallback_value, dtype=dtype) + logger.warning(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Using fallback value {fallback_value} for channel '{channel_char}' (Target Dims: {target_dimensions}).") + # Fallback uses target dimensions, don't add to actual_input_dimensions for mismatch check unless required + # actual_input_dimensions.append(target_dimensions) # Optional: Treat fallback as having target dims + status_notes.append(f"Used fallback for {required_map_type_from_rule}") + except Exception as e: + logger.error(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Error creating fallback for channel '{channel_char}': {e}. Cannot proceed with merge.") + processing_status = "Failed" + status_notes.append(f"Fallback creation failed for {required_map_type_from_rule}") + break # Break inner loop + else: + logger.error(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Missing input '{required_map_type_from_rule}' and no fallback default provided for channel '{channel_char}'. Cannot proceed.") + processing_status = "Failed" + status_notes.append(f"Missing input {required_map_type_from_rule} and no fallback") + break # Break inner loop + + if processing_status == "Failed": break # Exit outer loop if inner loop failed + + loaded_inputs_for_merge[channel_char] = input_image_data + inputs_used_for_merge[required_map_type_from_rule] = input_source_desc + + if processing_status == "Failed": continue # To finally block + + # Dimension Mismatch Handling + unique_dimensions = set(actual_input_dimensions) + target_merge_dims = target_dimensions # Default + if len(unique_dimensions) > 1: + logger.warning(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Mismatched dimensions found among loaded inputs: {unique_dimensions}. Applying strategy: {merge_dimension_mismatch_strategy}") + status_notes.append(f"Mismatched input dimensions ({unique_dimensions}), applied {merge_dimension_mismatch_strategy}") + + if merge_dimension_mismatch_strategy == "ERROR_SKIP": + logger.error(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Dimension mismatch strategy is ERROR_SKIP. Failing task.") + processing_status = "Failed" + status_notes.append("Dimension mismatch (ERROR_SKIP)") + continue # To finally block + elif merge_dimension_mismatch_strategy == "USE_LARGEST": + max_h = max(h for h, w in unique_dimensions) + max_w = max(w for h, w in unique_dimensions) + target_merge_dims = (max_h, max_w) + elif merge_dimension_mismatch_strategy == "USE_FIRST": + target_merge_dims = actual_input_dimensions[0] if actual_input_dimensions else target_dimensions + else: # Default or unknown: Use largest + logger.warning(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Unknown dimension mismatch strategy '{merge_dimension_mismatch_strategy}'. Defaulting to USE_LARGEST.") + max_h = max(h for h, w in unique_dimensions) + max_w = max(w for h, w in unique_dimensions) + target_merge_dims = (max_h, max_w) + + logger.info(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Resizing inputs to target merge dimensions: {target_merge_dims}") + # Resize loaded inputs (not fallbacks unless they were added to actual_input_dimensions) + for channel_char, img_data in loaded_inputs_for_merge.items(): + # Only resize if it was a loaded input that contributed to the mismatch check + if img_data.shape[:2] in unique_dimensions and img_data.shape[:2] != target_merge_dims: + resized_img = ipu.resize_image(img_data, target_merge_dims[1], target_merge_dims[0]) # w, h + if resized_img is None: + logger.error(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Failed to resize input for channel '{channel_char}' to {target_merge_dims}. Failing task.") + processing_status = "Failed" + status_notes.append(f"Input resize failed for {channel_char}") + break + loaded_inputs_for_merge[channel_char] = resized_img + if processing_status == "Failed": continue # To finally block + + # Perform Merge (Example: Simple Channel Packing - Adapt as needed) + # This needs to be robust based on merge_rule_config structure + try: + merge_channels_order = merge_rule_config.get('channel_order', 'RGB') # e.g., 'RGB', 'BGR', 'R', 'RGBA' etc. + output_channels = len(merge_channels_order) + h, w = target_merge_dims # Use the potentially adjusted dimensions + + if output_channels == 1: + # Assume the first channel in order is the one to use + channel_char_to_use = merge_channels_order[0] + source_img = loaded_inputs_for_merge[channel_char_to_use] + # Ensure it's grayscale (take first channel if it's multi-channel) + if len(source_img.shape) == 3: + current_image_data = source_img[:, :, 0].copy() + else: + current_image_data = source_img.copy() + elif output_channels > 1: + # Assume uint8 dtype for merged output unless specified otherwise + merged_image = np.zeros((h, w, output_channels), dtype=np.uint8) + for i, channel_char in enumerate(merge_channels_order): + source_img = loaded_inputs_for_merge.get(channel_char) + if source_img is not None: + # Extract the correct channel (e.g., R from RGB, or use grayscale directly) + if len(source_img.shape) == 3: + # Assuming standard RGB/BGR order in source based on channel_char? Needs clear definition. + # Example: If source is RGB and channel_char is 'R', take channel 0. + # This mapping needs to be defined in merge_rule_config or conventions. + # Simple approach: take the first channel if source is color. + merged_image[:, :, i] = source_img[:, :, 0] + else: # Grayscale source + merged_image[:, :, i] = source_img + else: + # This case should have been caught by fallback logic earlier + logger.error(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Missing prepared input for channel '{channel_char}' during final merge assembly. This shouldn't happen.") + processing_status = "Failed" + status_notes.append(f"Internal error: Missing input '{channel_char}' at merge assembly") + break + if processing_status != "Failed": + current_image_data = merged_image + else: + logger.error(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Invalid channel_order '{merge_channels_order}' in merge config.") + processing_status = "Failed" + status_notes.append("Invalid merge channel_order") + + if processing_status != "Failed": + logger.info(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Successfully merged inputs into image with shape {current_image_data.shape}") + original_dimensions = (current_image_data.shape[1], current_image_data.shape[0]) # Set original dims after merge + + except Exception as e: + logger.exception(f"Asset '{asset_name_for_log}', Key {task_key}, Proc. Tag {processing_instance_tag}: Error during merge operation: {e}") + processing_status = "Failed" + status_notes.append(f"Merge operation failed: {e}") + continue # To finally block + + else: + logger.error(f"Asset '{asset_name_for_log}', Key {item_key}: Unknown item type in processing loop: {type(item_data)}. Skipping.") + processing_status = "Failed" + status_notes.append("Unknown item type in loop") + continue # To finally block + + # --- C. Common Processing Path --- + if current_image_data is None: + logger.error(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: current_image_data is None before common processing. Status: {processing_status}. Skipping common path.") + # Status should already be Failed or Skipped from A or B + if processing_status not in ["Failed", "Skipped"]: + processing_status = "Failed" + status_notes.append("Internal error: Image data missing before common processing") + continue # To finally block + + logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Entering common processing path for '{base_map_type}' (Internal: '{processing_map_type}')") + + # In-Memory Transformations + transformation_applied = False + # Gloss-to-Rough + # Use filename-friendly 'GLOSS' or internal 'MAP_GLOSS' + if base_map_type == "GLOSS" or processing_map_type.startswith("MAP_GLOSS"): + logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Applying Gloss-to-Rough conversion.") + current_image_data = ipu.invert_image_colors(current_image_data) + # Update map types + new_processing_map_type = processing_map_type.replace("GLOSS", "ROUGH") + new_base_map_type = get_filename_friendly_map_type(new_processing_map_type, file_type_definitions) + logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Map type updated: '{processing_map_type}' -> '{new_processing_map_type}', Filename type: '{base_map_type}' -> '{new_base_map_type}'") + processing_map_type = new_processing_map_type + base_map_type = new_base_map_type + status_notes.append("Gloss-to-Rough applied") + transformation_applied = True + + # Normal Green Invert + # Use filename-friendly 'NRM' or internal 'MAP_NRM' + if (base_map_type == "NRM" or processing_map_type == "MAP_NRM") and invert_normal_green: + logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Applying Normal Map Green Channel Inversion (Global Setting).") + current_image_data = ipu.invert_normal_map_green_channel(current_image_data) + status_notes.append("Normal Green Inverted (Global)") + transformation_applied = True + + # Optional Initial Scaling (In Memory) + image_to_save = None + scaling_applied = False + h_pre_scale, w_pre_scale = current_image_data.shape[:2] + + if initial_scaling_mode == "POT_DOWNSCALE": + pot_w = ipu.get_nearest_power_of_two_downscale(w_pre_scale) + pot_h = ipu.get_nearest_power_of_two_downscale(h_pre_scale) + if (pot_w, pot_h) != (w_pre_scale, h_pre_scale): + logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Applying Initial Scaling: POT Downscale from ({w_pre_scale},{h_pre_scale}) to ({pot_w},{pot_h}).") + # Use aspect ratio preserving POT logic if needed, or simple independent POT per dim? Plan implies simple POT. + # Let's use the more robust aspect-preserving POT downscale logic from ipu if available, otherwise simple resize. + # Simple resize for now based on calculated pot_w, pot_h: + resized_img = ipu.resize_image(current_image_data, pot_w, pot_h, interpolation=cv2.INTER_AREA) + if resized_img is not None: + image_to_save = resized_img + scaling_applied = True + status_notes.append(f"Initial POT Downscale applied ({pot_w}x{pot_h})") + else: + logger.warning(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: POT Downscale resize failed. Using original data for saving.") + image_to_save = current_image_data.copy() + status_notes.append("Initial POT Downscale failed, used original") else: - logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Base POT image ({current_w}x{current_h}) is smaller than target max dim {res_max_dim}. Skipping this resolution variant.") - continue + logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Initial Scaling: POT Downscale - Image already POT or smaller. No scaling needed.") + image_to_save = current_image_data.copy() + elif initial_scaling_mode == "NONE": + logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Initial Scaling: Mode is NONE.") + image_to_save = current_image_data.copy() + else: + logger.warning(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Unknown INITIAL_SCALING_MODE '{initial_scaling_mode}'. Defaulting to NONE.") + image_to_save = current_image_data.copy() + status_notes.append(f"Unknown initial scale mode '{initial_scaling_mode}', used original") - target_w_res = min(target_w_res, current_w) - target_h_res = min(target_h_res, current_h) - - if target_w_res <=0 or target_h_res <=0: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Calculated target variant dims are zero or negative ({target_w_res}x{target_h_res}). Skipping.") - continue + if image_to_save is None: # Should not happen if logic above is correct + logger.error(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: image_to_save is None after scaling block. This indicates an error. Failing.") + processing_status = "Failed" + status_notes.append("Internal error: image_to_save is None post-scaling") + continue # To finally block - logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Processing variant for {res_max_dim}. Base POT Dims: ({current_w}x{current_h}), Target Dims for {res_key}: ({target_w_res}x{target_h_res}).") + # Color Management (Example: BGR to RGB if needed) + # This logic might need refinement based on actual requirements and ipu capabilities + # Assuming save_image_variants expects RGB by default if color conversion is needed. + # Let's assume save_image_variants handles color internally based on format/config for now. + # If specific BGR->RGB conversion is needed *before* saving based on map type: + # if base_map_type in ["COL", "DIFF", "ALB"] and len(image_to_save.shape) == 3 and image_to_save.shape[2] == 3: + # logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Applying BGR to RGB conversion before saving.") + # image_to_save = ipu.convert_bgr_to_rgb(image_to_save) + # status_notes.append("BGR->RGB applied") - output_image_data_for_res = base_pot_image_data - if (target_w_res, target_h_res) != (current_w, current_h): - interpolation_res = cv2.INTER_AREA - output_image_data_for_res = ipu.resize_image(base_pot_image_data, target_w_res, target_h_res, interpolation=interpolation_res) - if output_image_data_for_res is None: - logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Failed to resize image for resolution variant {res_key}.") - continue - - assetname_placeholder = context.asset_rule.asset_name if context.asset_rule else "UnknownAsset" - resolution_placeholder = res_key - - # TODO: Implement proper output format/extension determination for variants - output_ext_variant = temp_filename_suffix.lstrip('.') + # Call Unified Save Utility + logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Calling Unified Save Utility for map type '{base_map_type}' (Internal: '{processing_map_type}')") - temp_output_filename_variant = output_filename_pattern.replace("[assetname]", sanitize_filename(assetname_placeholder)) \ - .replace("[maptype]", sanitize_filename(filename_friendly_map_type)) \ - .replace("[resolution]", sanitize_filename(resolution_placeholder)) \ - .replace("[ext]", output_ext_variant) - temp_output_filename_variant = f"{processing_instance_tag}_variant_{temp_output_filename_variant}" # Use processing_instance_tag - temp_output_path_variant = context.engine_temp_dir / temp_output_filename_variant + try: + # Prepare arguments for save_image_variants + save_args = { + "source_image_data": image_to_save, + "base_map_type": base_map_type, # Filename-friendly + "source_bit_depth_info": source_bit_depth_info_for_save_util, + "output_filename_pattern_tokens": output_filename_pattern_tokens, + "config_obj": config, # Pass the whole config object + "asset_name_for_log": asset_name_for_log, # Pass asset name for logging within save util + "processing_instance_tag": processing_instance_tag # Pass tag for logging within save util + } - save_params_variant = [] - if transform_settings.get('output_format_settings'): - if output_ext_variant.lower() in ['jpg', 'jpeg']: - quality = transform_settings['output_format_settings'].get('quality', context.config_obj.get("JPG_QUALITY", 95)) - save_params_variant = [cv2.IMWRITE_JPEG_QUALITY, quality] - elif output_ext_variant.lower() == 'png': - compression = transform_settings['output_format_settings'].get('compression_level', context.config_obj.get("PNG_COMPRESSION_LEVEL", 6)) - save_params_variant = [cv2.IMWRITE_PNG_COMPRESSION, compression] - - save_success_variant = ipu.save_image(str(temp_output_path_variant), output_image_data_for_res, params=save_params_variant) + saved_files_details_list = save_image_variants(**save_args) - if not save_success_variant: - logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Failed to save temporary variant image to '{temp_output_path_variant}'.") - continue - - logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Successfully saved temporary variant map to '{temp_output_path_variant}' with dims ({target_w_res}x{target_h_res}).") - processed_at_least_one_resolution_variant = True - - if 'variants' not in context.processed_maps_details[current_map_key]: # Use current_map_key (file_rule_idx) - context.processed_maps_details[current_map_key]['variants'] = [] - - context.processed_maps_details[current_map_key]['variants'].append({ # Use current_map_key (file_rule_idx) - 'resolution_key': res_key, - 'temp_path': str(temp_output_path_variant), - 'dimensions': (target_w_res, target_h_res), - 'resolution_name': f"{target_w_res}x{target_h_res}" - }) - - if 'processed_files' not in context.asset_metadata: - context.asset_metadata['processed_files'] = [] - context.asset_metadata['processed_files'].append({ - 'processed_map_key': current_map_key, # Use current_map_key (file_rule_idx) - 'resolution_key': res_key, - 'path': str(temp_output_path_variant), - 'type': 'temporary_map_variant', - 'map_type': current_map_type, - 'dimensions_w': target_w_res, - 'dimensions_h': target_h_res - }) - # Calculate and store image statistics for the lowest resolution output - lowest_res_image_data_for_stats = None - image_to_stat_path_for_log = "N/A" - source_of_stats_image = "unknown" - - if processed_at_least_one_resolution_variant and \ - current_map_key in context.processed_maps_details and \ - 'variants' in context.processed_maps_details[current_map_key] and \ - context.processed_maps_details[current_map_key]['variants']: - - variants_list = context.processed_maps_details[current_map_key]['variants'] - valid_variants_for_stats = [ - v for v in variants_list - if isinstance(v.get('dimensions'), tuple) and len(v['dimensions']) == 2 and v['dimensions'][0] > 0 and v['dimensions'][1] > 0 - ] - - if valid_variants_for_stats: - smallest_variant = min(valid_variants_for_stats, key=lambda v: v['dimensions'][0] * v['dimensions'][1]) - - if smallest_variant and 'temp_path' in smallest_variant and smallest_variant.get('dimensions'): - smallest_res_w, smallest_res_h = smallest_variant['dimensions'] - logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Identified smallest variant for stats: {smallest_variant.get('resolution_key', 'N/A')} ({smallest_res_w}x{smallest_res_h}) at {smallest_variant['temp_path']}") - lowest_res_image_data_for_stats = ipu.load_image(smallest_variant['temp_path']) - image_to_stat_path_for_log = smallest_variant['temp_path'] - source_of_stats_image = f"variant {smallest_variant.get('resolution_key', 'N/A')}" - if lowest_res_image_data_for_stats is None: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Failed to load smallest variant image '{smallest_variant['temp_path']}' for stats.") + if saved_files_details_list: + logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Unified Save Utility completed successfully. Saved {len(saved_files_details_list)} variants.") + processing_status = "Processed_Via_Save_Utility" else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Could not determine smallest variant for stats from valid variants list (details missing).") - else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: No valid variants found to determine the smallest one for stats.") + logger.warning(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Unified Save Utility returned no saved file details. Check utility logs.") + processing_status = "Processed_Save_Utility_No_Output" # Or potentially "Failed" depending on severity + status_notes.append("Save utility reported no files saved") - if lowest_res_image_data_for_stats is None: - if base_pot_image_data is not None: - logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Using base POT image for stats (dimensions: {base_pot_width}x{base_pot_height}). Smallest variant not available/loaded or no variants generated.") - lowest_res_image_data_for_stats = base_pot_image_data - image_to_stat_path_for_log = f"In-memory base POT image (dims: {base_pot_width}x{base_pot_height})" - source_of_stats_image = "base POT" - else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Base POT image data is also None. Cannot calculate stats.") + except Exception as e: + logger.exception(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Error calling or executing save_image_variants: {e}") + processing_status = "Failed" + status_notes.append(f"Save utility call failed: {e}") + # saved_files_details_list remains empty - if lowest_res_image_data_for_stats is not None: - stats_dict = ipu.calculate_image_stats(lowest_res_image_data_for_stats) - if stats_dict and "error" not in stats_dict: - if 'image_stats_lowest_res' not in context.asset_metadata: - context.asset_metadata['image_stats_lowest_res'] = {} - - context.asset_metadata['image_stats_lowest_res'][current_map_type] = stats_dict # Keyed by map_type - logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': Calculated and stored image stats from '{source_of_stats_image}' (source ref: '{image_to_stat_path_for_log}').") - elif stats_dict and "error" in stats_dict: - logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': Error calculating image stats from '{source_of_stats_image}': {stats_dict['error']}.") - else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': Failed to calculate image stats from '{source_of_stats_image}' (result was None or empty).") - else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': No image data available (from variant or base POT) to calculate stats.") + except Exception as e: + logger.exception(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Unhandled exception during processing loop for item: {e}") + processing_status = "Failed" + status_notes.append(f"Unhandled exception: {e}") - # Final status update based on whether variants were generated (and expected) - if generate_variants_for_this_map_type: - if processed_at_least_one_resolution_variant: - self._update_file_rule_status(context, current_map_key, 'Processed_With_Variants', - map_type=filename_friendly_map_type, - processing_map_type=current_map_type, - source_file_rule_index=file_rule_idx, - processing_tag=processing_instance_tag, - details="Successfully processed with multiple resolution variants.") + finally: + # --- Update Context --- + logger.debug(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Updating context. Status: {processing_status}, Notes: {status_notes}") + details_entry = { + 'status': processing_status, + 'map_type': base_map_type, # Final filename-friendly type + 'processing_map_type': processing_map_type, # Final internal type + 'notes': " | ".join(status_notes), + 'saved_files_info': saved_files_details_list, + 'original_dimensions': original_dimensions, # (w, h) + } + if is_merged_task: + details_entry['merge_task_config_output_type'] = merge_task_config_output_type + details_entry['inputs_used_for_merge'] = inputs_used_for_merge + details_entry['source_bit_depths'] = source_bit_depth_info_for_save_util # Store the list used else: - logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Variants were expected for map type '{current_map_type}', but none were generated (e.g., base POT too small for any variant tier).") - self._update_file_rule_status(context, current_map_key, 'Processed_No_Variants', - map_type=filename_friendly_map_type, - processing_map_type=current_map_type, - source_file_rule_index=file_rule_idx, - processing_tag=processing_instance_tag, - details="Variants expected but none generated (e.g., base POT too small).") - else: # No variants were expected for this map type - self._update_file_rule_status(context, current_map_key, 'Processed_No_Variants', - map_type=filename_friendly_map_type, - processing_map_type=current_map_type, - source_file_rule_index=file_rule_idx, - processing_tag=processing_instance_tag, - details="Processed to base POT; variants not applicable for this map type.") + # Regular map specific details + details_entry['source_file'] = str(source_file_path_regular) if source_file_path_regular else "N/A" + details_entry['original_bit_depth'] = source_bit_depth_info_for_save_util[0] if source_bit_depth_info_for_save_util else None + details_entry['source_file_rule_index'] = item_key # Store original index + + context.processed_maps_details[item_key] = details_entry + logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Context updated for this item.") logger.info(f"Asset '{asset_name_for_log}': Finished individual map processing stage.") return context - def _find_source_file(self, base_path: Path, pattern: str, asset_name_for_log: str, processing_instance_tag: str) -> Optional[Path]: + def _get_suffixed_internal_map_type(self, context: AssetProcessingContext, current_file_rule: FileRule, initial_internal_map_type: str, respect_variant_map_types: List[str]) -> str: """ - Finds a single source file matching the pattern within the base_path. - Logs use processing_instance_tag for specific run tracing. + Determines the potentially suffixed internal map type (e.g., MAP_COL-1) + based on occurrences within the asset rule's file list. """ - if not pattern: - logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Empty file_path provided in FileRule.") - return None - - # If pattern is an absolute path, use it directly - potential_abs_path = Path(pattern) - if potential_abs_path.is_absolute() and potential_abs_path.exists(): - logger.debug(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: file_path '{pattern}' is absolute and exists. Using it directly.") - return potential_abs_path - elif potential_abs_path.is_absolute(): - logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: file_path '{pattern}' is absolute but does not exist.") - # Fall through to try resolving against base_path if it's just a name/relative pattern - - # Treat pattern as relative to base_path - # This could be an exact name or a glob pattern - try: - # First, check if pattern is an exact relative path - exact_match_path = base_path / pattern - if exact_match_path.exists() and exact_match_path.is_file(): - logger.debug(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Found exact match for '{pattern}' at '{exact_match_path}'.") - return exact_match_path - - # If not an exact match, try as a glob pattern (recursive) - matched_files_rglob = list(base_path.rglob(pattern)) - if matched_files_rglob: - if len(matched_files_rglob) > 1: - logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Multiple files ({len(matched_files_rglob)}) found for pattern '{pattern}' in '{base_path}' (recursive). Using first: {matched_files_rglob[0]}. Files: {matched_files_rglob}") - return matched_files_rglob[0] - - # Try non-recursive glob if rglob fails - matched_files_glob = list(base_path.glob(pattern)) - if matched_files_glob: - if len(matched_files_glob) > 1: - logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Multiple files ({len(matched_files_glob)}) found for pattern '{pattern}' in '{base_path}' (non-recursive). Using first: {matched_files_glob[0]}. Files: {matched_files_glob}") - return matched_files_glob[0] - - logger.debug(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: No files found matching pattern '{pattern}' in '{base_path}' (exact, recursive, or non-recursive).") - return None - except Exception as e: - logger.error(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Error searching for file with pattern '{pattern}' in '{base_path}': {e}") - return None - - def _update_file_rule_status(self, context: AssetProcessingContext, map_key_index: int, status: str, **kwargs): # Renamed map_id_hex to map_key_index - """Helper to update processed_maps_details for a map, keyed by file_rule_idx.""" + final_internal_map_type = initial_internal_map_type # Default asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" - if map_key_index not in context.processed_maps_details: - context.processed_maps_details[map_key_index] = {} - - context.processed_maps_details[map_key_index]['status'] = status - for key, value in kwargs.items(): - # Ensure source_file_rule_id_hex is not added if it was somehow passed (it shouldn't be) - if key == 'source_file_rule_id_hex': - continue - context.processed_maps_details[map_key_index][key] = value - - if 'map_type' not in context.processed_maps_details[map_key_index] and 'map_type' in kwargs: - context.processed_maps_details[map_key_index]['map_type'] = kwargs['map_type'] - - # Add formatted resolution names - if 'original_dimensions' in kwargs and isinstance(kwargs['original_dimensions'], tuple) and len(kwargs['original_dimensions']) == 2: - orig_w, orig_h = kwargs['original_dimensions'] - context.processed_maps_details[map_key_index]['original_resolution_name'] = f"{orig_w}x{orig_h}" - - # Determine the correct dimensions to use for 'processed_resolution_name' - # This name refers to the base POT scaled image dimensions before variant generation. - dims_to_log_as_base_processed = None - if 'base_pot_dimensions' in kwargs and isinstance(kwargs['base_pot_dimensions'], tuple) and len(kwargs['base_pot_dimensions']) == 2: - # This key is used when status is 'Processed_With_Variants' - dims_to_log_as_base_processed = kwargs['base_pot_dimensions'] - elif 'processed_dimensions' in kwargs and isinstance(kwargs['processed_dimensions'], tuple) and len(kwargs['processed_dimensions']) == 2: - # This key is used when status is 'Processed_No_Variants' (and potentially others) - dims_to_log_as_base_processed = kwargs['processed_dimensions'] - - if dims_to_log_as_base_processed: - proc_w, proc_h = dims_to_log_as_base_processed - resolution_name_str = f"{proc_w}x{proc_h}" - context.processed_maps_details[map_key_index]['base_pot_resolution_name'] = resolution_name_str - # Ensure 'processed_resolution_name' is also set for OutputOrganizationStage compatibility - context.processed_maps_details[map_key_index]['processed_resolution_name'] = resolution_name_str - elif 'processed_dimensions' in kwargs or 'base_pot_dimensions' in kwargs: - details_for_warning = kwargs.get('processed_dimensions', kwargs.get('base_pot_dimensions')) - logger.warning(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: 'processed_dimensions' or 'base_pot_dimensions' key present but its value is not a valid 2-element tuple: {details_for_warning}") - - # If temp_processed_file was passed, ensure it's in the details - if 'temp_processed_file' in kwargs: - context.processed_maps_details[map_key_index]['temp_processed_file'] = kwargs['temp_processed_file'] - # Log all details being stored for clarity, including the newly added resolution names - log_details = context.processed_maps_details[map_key_index].copy() - # Avoid logging full image data if it accidentally gets into kwargs - if 'image_data' in log_details: del log_details['image_data'] - if 'base_pot_image_data' in log_details: del log_details['base_pot_image_data'] - logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: Status updated to '{status}'. Details: {log_details}") \ No newline at end of file + base_map_type_match = re.match(r"(MAP_[A-Z]{3})", initial_internal_map_type) + if not base_map_type_match or not context.asset_rule or not context.asset_rule.files: + return final_internal_map_type # Cannot determine suffix without base type or asset rule files + + true_base_map_type = base_map_type_match.group(1) # This is "MAP_XXX" + + peers_of_same_base_type = [] + for fr_asset in context.asset_rule.files: + fr_asset_item_type = fr_asset.item_type_override or fr_asset.item_type or "UnknownMapType" + fr_asset_base_match = re.match(r"(MAP_[A-Z]{3})", fr_asset_item_type) + if fr_asset_base_match and fr_asset_base_match.group(1) == true_base_map_type: + peers_of_same_base_type.append(fr_asset) + + num_occurrences = len(peers_of_same_base_type) + current_instance_index = 0 # 1-based index + + try: + # Find the index based on the FileRule object itself + current_instance_index = peers_of_same_base_type.index(current_file_rule) + 1 + except ValueError: + # Fallback: try matching by file_path if object identity fails (less reliable) + try: + current_instance_index = [fr.file_path for fr in peers_of_same_base_type].index(current_file_rule.file_path) + 1 + logger.warning(f"Asset '{asset_name_for_log}', FileRule path '{current_file_rule.file_path}': Found peer index using file_path fallback.") + except (ValueError, AttributeError): # Catch AttributeError if file_path is None + logger.warning( + f"Asset '{asset_name_for_log}', FileRule path '{current_file_rule.file_path}' (Initial Type: '{initial_internal_map_type}', Base: '{true_base_map_type}'): " + f"Could not find its own instance in the list of {num_occurrences} peers from asset_rule.files using object identity or path. Suffixing may be incorrect." + ) + # Keep index 0, suffix logic below will handle it + + # Determine Suffix + map_type_for_respect_check = true_base_map_type.replace("MAP_", "") # e.g., "COL" + is_in_respect_list = map_type_for_respect_check in respect_variant_map_types + + suffix_to_append = "" + if num_occurrences > 1: + if current_instance_index > 0: + suffix_to_append = f"-{current_instance_index}" + else: + # If index is still 0 (not found), don't add suffix to avoid ambiguity + logger.warning(f"Asset '{asset_name_for_log}', FileRule path '{current_file_rule.file_path}': Index for multi-occurrence map type '{true_base_map_type}' (count: {num_occurrences}) not determined. Omitting numeric suffix.") + elif num_occurrences == 1 and is_in_respect_list: + suffix_to_append = "-1" # Add suffix even for single instance if in respect list + + if suffix_to_append: + final_internal_map_type = true_base_map_type + suffix_to_append + # else: final_internal_map_type remains the initial_internal_map_type if no suffix needed + + if final_internal_map_type != initial_internal_map_type: + logger.debug(f"Asset '{asset_name_for_log}', FileRule path '{current_file_rule.file_path}': Suffixed internal map type determined: '{initial_internal_map_type}' -> '{final_internal_map_type}'") + + return final_internal_map_type \ No newline at end of file diff --git a/processing/pipeline/stages/map_merging.py b/processing/pipeline/stages/map_merging.py index 5dacc73..696b05e 100644 --- a/processing/pipeline/stages/map_merging.py +++ b/processing/pipeline/stages/map_merging.py @@ -2,14 +2,12 @@ import logging from pathlib import Path from typing import Dict, Optional, List, Tuple -import numpy as np -import cv2 # For potential direct cv2 operations if ipu doesn't cover all merge needs from .base_stage import ProcessingStage from ..asset_context import AssetProcessingContext from rule_structure import FileRule from utils.path_utils import sanitize_filename -from ...utils import image_processing_utils as ipu + logger = logging.getLogger(__name__) @@ -34,48 +32,20 @@ class MapMergingStage(ProcessingStage): if context.status_flags.get('skip_asset'): logger.info(f"Skipping map merging for asset {asset_name_for_log} as skip_asset flag is set.") return context - if not hasattr(context, 'merged_maps_details'): context.merged_maps_details = {} + + if not hasattr(context, 'merged_image_tasks'): + context.merged_image_tasks = [] if not hasattr(context, 'processed_maps_details'): - logger.warning(f"Asset {asset_name_for_log}: 'processed_maps_details' not found in context. Cannot perform map merging.") + logger.warning(f"Asset {asset_name_for_log}: 'processed_maps_details' not found in context. Cannot generate merge tasks.") return context - if not context.files_to_process: # This list might not be relevant if merge rules are defined elsewhere or implicitly - logger.info(f"Asset {asset_name_for_log}: No files_to_process defined. This stage might rely on config or processed_maps_details directly for merge rules.") - # Depending on design, this might not be an error, so we don't return yet. - logger.info(f"Starting MapMergingStage for asset: {asset_name_for_log}") - # TODO: The logic for identifying merge rules and their inputs needs significant rework - # as FileRule no longer has 'id' or 'merge_settings' directly in the way this stage expects. - # Merge rules are likely defined in the main configuration (context.config_obj.map_merge_rules) - # and need to be matched against available maps in context.processed_maps_details. - - # Placeholder for the loop that would iterate over context.config_obj.map_merge_rules - # For now, this stage will effectively do nothing until that logic is implemented. - - # Example of how one might start to adapt: - # for configured_merge_rule in context.config_obj.map_merge_rules: - # output_map_type = configured_merge_rule.get('output_map_type') - # inputs_config = configured_merge_rule.get('inputs') # e.g. {"R": "NORMAL", "G": "ROUGHNESS"} - # # ... then find these input map_types in context.processed_maps_details ... - # # ... and perform the merge ... - # # This is a complex change beyond simple attribute renaming. - - # The following is the original loop structure, which will likely fail due to missing attributes on FileRule. - # Keeping it commented out to show what was there. - """ - for merge_rule in context.files_to_process: # This iteration logic is likely incorrect for merge rules - if not isinstance(merge_rule, FileRule) or merge_rule.item_type != "MAP_MERGE": - continue - - # FileRule does not have merge_settings or id.hex - # This entire block needs to be re-thought based on where merge rules are defined. - # Assuming merge_rule_id_hex would be a generated UUID for this operation. - merge_rule_id_hex = f"merge_op_{uuid.uuid4().hex[:8]}" - current_map_type = merge_rule.item_type_override or merge_rule.item_type + # The core merge rules are in context.config_obj.map_merge_rules + # Each rule in there defines an output_map_type and its inputs. logger.error(f"Asset {asset_name_for_log}, Potential Merge for {current_map_type}: Merge rule processing needs rework. FileRule lacks 'merge_settings' and 'id'. Skipping this rule.") context.merged_maps_details[merge_rule_id_hex] = { @@ -84,7 +54,7 @@ class MapMergingStage(ProcessingStage): 'reason': 'Merge rule processing logic in MapMergingStage needs refactor due to FileRule changes.' } continue - """ + # For now, let's assume no merge rules are processed until the logic is fixed. num_merge_rules_attempted = 0 @@ -115,24 +85,20 @@ class MapMergingStage(ProcessingStage): merge_op_id = f"merge_{sanitize_filename(output_map_type)}_{rule_idx}" logger.info(f"Asset {asset_name_for_log}: Processing configured merge rule for '{output_map_type}' (Op ID: {merge_op_id})") - loaded_input_maps: Dict[str, np.ndarray] = {} # Key: input_map_type (e.g. "NRM"), Value: image_data - input_map_paths: Dict[str, str] = {} # Key: input_map_type, Value: path_str - target_dims: Optional[Tuple[int, int]] = None - all_inputs_valid = True - - # Find and load input maps from processed_maps_details - # This assumes one processed map per map_type. If multiple variants exist, this needs refinement. + input_map_sources_list = [] + source_bit_depths_list = [] + primary_source_dimensions = None + + # Find required input maps from processed_maps_details required_input_map_types = set(inputs_map_type_to_channel.values()) - + for required_map_type in required_input_map_types: found_processed_map_details = None - # The key `p_key_idx` is the file_rule_idx from the IndividualMapProcessingStage - for p_key_idx, p_details in context.processed_maps_details.items(): # p_key_idx is an int + # Iterate through processed_maps_details to find the required map type + for p_key_idx, p_details in context.processed_maps_details.items(): processed_map_identifier = p_details.get('processing_map_type', p_details.get('map_type')) - - # Comprehensive list of valid statuses for an input map to be used in merging - valid_input_statuses = ['BasePOTSaved', 'Processed_With_Variants', 'Processed_No_Variants', 'Converted_To_Rough'] + # Check for a match, considering both "MAP_TYPE" and "TYPE" formats is_match = False if processed_map_identifier == required_map_type: is_match = True @@ -141,207 +107,56 @@ class MapMergingStage(ProcessingStage): elif not required_map_type.startswith("MAP_") and processed_map_identifier == f"MAP_{required_map_type}": is_match = True - if is_match and p_details.get('status') in valid_input_statuses: - found_processed_map_details = p_details - # The key `p_key_idx` (which is the FileRule index) is implicitly associated with these details. - break - - if not found_processed_map_details: - can_be_fully_defaulted = True - channels_requiring_this_map = [ - ch_key for ch_key, map_type_val in inputs_map_type_to_channel.items() - if map_type_val == required_map_type - ] + # Check if the found map is in a usable status and has a temporary file + valid_input_statuses = ['BasePOTSaved', 'Processed_With_Variants', 'Processed_No_Variants', 'Converted_To_Rough'] # Add other relevant statuses if needed + if is_match and p_details.get('status') in valid_input_statuses and p_details.get('temp_processed_file'): + # Also check if the temp file actually exists on disk + if Path(p_details.get('temp_processed_file')).exists(): + found_processed_map_details = p_details + break # Found a suitable input, move to the next required map type - if not channels_requiring_this_map: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Internal logic error. Required map_type '{required_map_type}' is not actually used by any output channel. Configuration: {inputs_map_type_to_channel}") - all_inputs_valid = False - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Internal error: required map_type '{required_map_type}' not in use."} - break - - for channel_char_needing_default in channels_requiring_this_map: - if default_values.get(channel_char_needing_default) is None: - can_be_fully_defaulted = False - break - - if can_be_fully_defaulted: - logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Required input map_type '{required_map_type}' for output '{output_map_type}' not found or not in usable state. Will attempt to use default values for its channels: {channels_requiring_this_map}.") - else: - logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Required input map_type '{required_map_type}' for output '{output_map_type}' not found/unusable, AND not all its required channels ({channels_requiring_this_map}) have defaults. Failing merge op.") - all_inputs_valid = False - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Input '{required_map_type}' missing and defaults incomplete."} - break - if found_processed_map_details: - temp_file_path_str = found_processed_map_details.get('temp_processed_file') - if not temp_file_path_str: - # Log with p_key_idx if available, or just the map type if not (though it should be if found_processed_map_details is set) - log_key_info = f"(Associated Key Index: {p_key_idx})" if 'p_key_idx' in locals() and found_processed_map_details else "" # Use locals() to check if p_key_idx is defined in this scope - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: 'temp_processed_file' missing in details for found map_type '{required_map_type}' {log_key_info}.") - all_inputs_valid = False - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Temp file path missing for input '{required_map_type}'."} - break - - temp_file_path = Path(temp_file_path_str) - if not temp_file_path.exists(): - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Temp file {temp_file_path} for input map_type '{required_map_type}' does not exist.") - all_inputs_valid = False - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Temp file for input '{required_map_type}' missing."} - break + file_path = found_processed_map_details.get('temp_processed_file') + dimensions = found_processed_map_details.get('base_pot_dimensions') - try: - image_data = ipu.load_image(str(temp_file_path)) - if image_data is None: raise ValueError("Loaded image is None") - except Exception as e: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error loading image {temp_file_path} for input map_type '{required_map_type}': {e}") - all_inputs_valid = False - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Error loading input '{required_map_type}'."} - break - - loaded_input_maps[required_map_type] = image_data - input_map_paths[required_map_type] = str(temp_file_path) + # Attempt to get original_bit_depth, log warning if not found + original_bit_depth = found_processed_map_details.get('original_bit_depth') + if original_bit_depth is None: + logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: 'original_bit_depth' not found in processed_maps_details for map type '{required_map_type}'. This value is pending IndividualMapProcessingStage refactoring and will be None or a default for now.") - current_dims = (image_data.shape[1], image_data.shape[0]) - if target_dims is None: - target_dims = current_dims - elif current_dims != target_dims: - logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{required_map_type}' dims {current_dims} differ from target {target_dims}. Resizing.") - try: - image_data_resized = ipu.resize_image(image_data, target_dims[0], target_dims[1]) - if image_data_resized is None: raise ValueError("Resize returned None") - loaded_input_maps[required_map_type] = image_data_resized - except Exception as e: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to resize '{required_map_type}': {e}") - all_inputs_valid = False - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Failed to resize input '{required_map_type}'."} - break - - if not all_inputs_valid: - logger.warning(f"Asset {asset_name_for_log}: Skipping merge for Op ID {merge_op_id} ('{output_map_type}') due to invalid inputs.") - continue + input_map_sources_list.append({ + 'map_type': required_map_type, + 'file_path': file_path, + 'dimensions': dimensions, + 'original_bit_depth': original_bit_depth + }) + source_bit_depths_list.append(original_bit_depth) - if not loaded_input_maps and not any(default_values.get(ch) is not None for ch in inputs_map_type_to_channel.keys()): - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: No input maps loaded and no defaults available for any channel for '{output_map_type}'. Cannot proceed.") - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'No input maps loaded and no defaults available.'} - continue - - if target_dims is None: - default_res_key = context.config_obj.get("default_output_resolution_key_for_merge", "1K") - image_resolutions_cfg = getattr(context.config_obj, "image_resolutions", {}) - default_max_dim = image_resolutions_cfg.get(default_res_key) - - if default_max_dim: - target_dims = (default_max_dim, default_max_dim) - logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Target dimensions not set by inputs (all defaulted). Using configured default resolution '{default_res_key}': {target_dims}.") + # Set primary_source_dimensions from the first valid input found + if primary_source_dimensions is None and dimensions: + primary_source_dimensions = dimensions else: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Target dimensions could not be determined for '{output_map_type}' (all inputs defaulted and no default output resolution configured).") - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'Target dimensions undetermined for fully defaulted merge.'} - continue - - output_channel_keys = sorted(list(inputs_map_type_to_channel.keys())) - num_output_channels = len(output_channel_keys) - - if num_output_channels == 0: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: No output channels defined in 'inputs' for '{output_map_type}'.") - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'No output channels defined.'} - continue - - try: - output_dtype = np.uint8 - - if num_output_channels == 1: - merged_image = np.zeros((target_dims[1], target_dims[0]), dtype=output_dtype) - else: - merged_image = np.zeros((target_dims[1], target_dims[0], num_output_channels), dtype=output_dtype) - except Exception as e: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error creating empty merged image for '{output_map_type}': {e}") - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f'Error creating output canvas: {e}'} - continue - - merge_op_failed_detail = False - for i, out_channel_char in enumerate(output_channel_keys): - input_map_type_for_this_channel = inputs_map_type_to_channel[out_channel_char] - source_image = loaded_input_maps.get(input_map_type_for_this_channel) - - source_data_this_channel = None - if source_image is not None: - if source_image.dtype != np.uint8: - logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{input_map_type_for_this_channel}' has dtype {source_image.dtype}, expected uint8. Attempting conversion.") - source_image = ipu.convert_to_uint8(source_image) - if source_image is None: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to convert input '{input_map_type_for_this_channel}' to uint8.") - merge_op_failed_detail = True; break + # If a required map is not found, log a warning but don't fail the task generation. + # The consuming stage will handle missing inputs and fallbacks. + logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Required input map type '{required_map_type}' not found or not in a usable state in context.processed_maps_details. This input will be skipped for task generation.") - if source_image.ndim == 2: - source_data_this_channel = source_image - elif source_image.ndim == 3: - semantic_to_bgr_idx = {'R': 2, 'G': 1, 'B': 0, 'A': 3} - - idx_to_extract = semantic_to_bgr_idx.get(out_channel_char.upper()) - - if idx_to_extract is not None and idx_to_extract < source_image.shape[2]: - source_data_this_channel = source_image[:, :, idx_to_extract] - logger.debug(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: For output '{out_channel_char}', using source '{input_map_type_for_this_channel}' semantic '{out_channel_char}' (BGR(A) index {idx_to_extract}).") - else: - logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Could not map output '{out_channel_char}' to a specific BGR(A) channel of '{input_map_type_for_this_channel}' (shape {source_image.shape}). Defaulting to its channel 0 (Blue).") - source_data_this_channel = source_image[:, :, 0] - else: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Source image '{input_map_type_for_this_channel}' has unexpected dimensions: {source_image.ndim} (shape {source_image.shape}).") - merge_op_failed_detail = True; break - - else: - default_val_for_channel = default_values.get(out_channel_char) - if default_val_for_channel is not None: - try: - scaled_default_val = int(float(default_val_for_channel) * 255) - source_data_this_channel = np.full((target_dims[1], target_dims[0]), scaled_default_val, dtype=np.uint8) - logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Using default value {default_val_for_channel} (scaled to {scaled_default_val}) for output channel '{out_channel_char}' as input map '{input_map_type_for_this_channel}' was missing.") - except ValueError: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Default value '{default_val_for_channel}' for channel '{out_channel_char}' is not a valid float. Cannot scale.") - merge_op_failed_detail = True; break - else: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{input_map_type_for_this_channel}' for output channel '{out_channel_char}' is missing and no default value provided.") - merge_op_failed_detail = True; break - - if source_data_this_channel is None: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to get source data for output channel '{out_channel_char}'.") - merge_op_failed_detail = True; break - - try: - if merged_image.ndim == 2: - merged_image = source_data_this_channel - else: - merged_image[:, :, i] = source_data_this_channel - except Exception as e: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error assigning data to output channel '{out_channel_char}' (index {i}): {e}. Merged shape: {merged_image.shape}, Source data shape: {source_data_this_channel.shape}") - merge_op_failed_detail = True; break - - if merge_op_failed_detail: - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'Error during channel assignment.'} - continue - - output_format = 'png' - temp_merged_filename = f"merged_{sanitize_filename(output_map_type)}_{merge_op_id}.{output_format}" - temp_merged_path = context.engine_temp_dir / temp_merged_filename - - try: - save_success = ipu.save_image(str(temp_merged_path), merged_image) - if not save_success: raise ValueError("Save image returned false") - except Exception as e: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error saving merged image {temp_merged_path}: {e}") - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f'Failed to save merged image: {e}'} - continue - - logger.info(f"Asset {asset_name_for_log}: Successfully merged and saved '{output_map_type}' (Op ID: {merge_op_id}) to {temp_merged_path}") - context.merged_maps_details[merge_op_id] = { - 'map_type': output_map_type, - 'temp_merged_file': str(temp_merged_path), - 'input_map_types_used': list(inputs_map_type_to_channel.values()), - 'input_map_files_used': input_map_paths, - 'merged_dimensions': target_dims, - 'status': 'Processed' + # Create the merged image task dictionary + merged_task = { + 'output_map_type': output_map_type, + 'input_map_sources': input_map_sources_list, + 'merge_rule_config': configured_merge_rule, + 'source_dimensions': primary_source_dimensions, # Can be None if no inputs were found + 'source_bit_depths': source_bit_depths_list } - logger.info(f"Finished MapMergingStage for asset: {asset_name_for_log}. Merged maps operations attempted: {num_merge_rules_attempted}, Succeeded: {len([d for d in context.merged_maps_details.values() if d.get('status') == 'Processed'])}") + # Append the task to the context + context.merged_image_tasks.append(merged_task) + logger.info(f"Asset {asset_name_for_log}: Generated merge task for '{output_map_type}' (Op ID: {merge_op_id}). Task details: {merged_task}") + + # Note: We no longer populate context.merged_maps_details with 'Processed' status here, + # as this stage only generates tasks, it doesn't perform the merge or save files. + # The merged_maps_details will be populated by the stage that consumes these tasks. + + logger.info(f"Finished MapMergingStage for asset: {asset_name_for_log}. Merge tasks generated: {len(context.merged_image_tasks)}") return context \ No newline at end of file diff --git a/processing/utils/image_processing_utils.py b/processing/utils/image_processing_utils.py index 81e3e08..e9181b8 100644 --- a/processing/utils/image_processing_utils.py +++ b/processing/utils/image_processing_utils.py @@ -163,6 +163,37 @@ def calculate_target_dimensions( # --- Image Statistics --- +def get_image_bit_depth(image_path_str: str) -> Optional[int]: + """ + Determines the bit depth of an image file. + """ + try: + # Use IMREAD_UNCHANGED to preserve original bit depth + img = cv2.imread(image_path_str, cv2.IMREAD_UNCHANGED) + if img is None: + # logger.error(f"Failed to read image for bit depth: {image_path_str}") # Use print for utils + print(f"Warning: Failed to read image for bit depth: {image_path_str}") + return None + + dtype_to_bit_depth = { + np.dtype('uint8'): 8, + np.dtype('uint16'): 16, + np.dtype('float32'): 32, # Typically for EXR etc. + np.dtype('int8'): 8, # Unlikely for images but good to have + np.dtype('int16'): 16, # Unlikely + # Add other dtypes if necessary + } + bit_depth = dtype_to_bit_depth.get(img.dtype) + if bit_depth is None: + # logger.warning(f"Unknown dtype {img.dtype} for image {image_path_str}, cannot determine bit depth.") # Use print for utils + print(f"Warning: Unknown dtype {img.dtype} for image {image_path_str}, cannot determine bit depth.") + pass # Return None + return bit_depth + except Exception as e: + # logger.error(f"Error getting bit depth for {image_path_str}: {e}") # Use print for utils + print(f"Error getting bit depth for {image_path_str}: {e}") + return None + def calculate_image_stats(image_data: np.ndarray) -> Optional[Dict]: """ Calculates min, max, mean for a given numpy image array. diff --git a/processing/utils/image_saving_utils.py b/processing/utils/image_saving_utils.py new file mode 100644 index 0000000..7a51d14 --- /dev/null +++ b/processing/utils/image_saving_utils.py @@ -0,0 +1,250 @@ +import logging +import cv2 +import numpy as np +from pathlib import Path +from typing import List, Dict, Any, Tuple, Optional + +# Potentially import ipu from ...utils import image_processing_utils as ipu +# Assuming ipu is available in the same utils directory or parent +try: + from . import image_processing_utils as ipu +except ImportError: + # Fallback for different import structures if needed, adjust based on actual project structure + # For this project structure, the relative import should work. + logging.warning("Could not import image_processing_utils using relative path. Attempting absolute import.") + try: + from processing.utils import image_processing_utils as ipu + except ImportError: + logging.error("Could not import image_processing_utils.") + ipu = None # Handle case where ipu is not available + +logger = logging.getLogger(__name__) + +def save_image_variants( + source_image_data: np.ndarray, + base_map_type: str, # Filename-friendly map type + source_bit_depth_info: List[Optional[int]], + image_resolutions: Dict[str, int], + file_type_defs: Dict[str, Dict[str, Any]], + output_format_8bit: str, + output_format_16bit_primary: str, + output_format_16bit_fallback: str, + png_compression_level: int, + jpg_quality: int, + output_filename_pattern_tokens: Dict[str, Any], # Must include 'output_base_directory': Path and 'asset_name': str + output_filename_pattern: str, + # Consider adding ipu or relevant parts of it if not importing globally +) -> List[Dict[str, Any]]: + """ + Centralizes image saving logic, generating and saving various resolution variants + according to configuration. + + Args: + source_image_data (np.ndarray): High-res image data (in memory, potentially transformed). + base_map_type (str): Final map type (e.g., "COL", "ROUGH", "NORMAL", "MAP_NRMRGH"). + This is the filename-friendly map type. + source_bit_depth_info (List[Optional[int]]): List of original source bit depth(s) + (e.g., [8], [16], [8, 16]). Can contain None. + image_resolutions (Dict[str, int]): Dictionary mapping resolution keys (e.g., "4K") + to max dimensions (e.g., 4096). + file_type_defs (Dict[str, Dict[str, Any]]): Dictionary defining properties for map types, + including 'bit_depth_rule'. + output_format_8bit (str): File extension for 8-bit output (e.g., "jpg", "png"). + output_format_16bit_primary (str): Primary file extension for 16-bit output (e.g., "png", "tif"). + output_format_16bit_fallback (str): Fallback file extension for 16-bit output. + png_compression_level (int): Compression level for PNG output (0-9). + jpg_quality (int): Quality level for JPG output (0-100). + output_filename_pattern_tokens (Dict[str, Any]): Dictionary of tokens for filename + pattern replacement. Must include + 'output_base_directory' (Path) and + 'asset_name' (str). + output_filename_pattern (str): Pattern string for generating output filenames + (e.g., "[assetname]_[maptype]_[resolution].[ext]"). + + Returns: + List[Dict[str, Any]]: A list of dictionaries, each containing details about a saved file. + Example: [{'path': str, 'resolution_key': str, 'format': str, + 'bit_depth': int, 'dimensions': (w,h)}, ...] + """ + if ipu is None: + logger.error("image_processing_utils is not available. Cannot save images.") + return [] + + saved_file_details = [] + source_h, source_w = source_image_data.shape[:2] + source_max_dim = max(source_h, source_w) + + # 1. Use provided configuration inputs (already available as function arguments) + logger.info(f"Saving variants for map type: {base_map_type}") + + # 2. Determine Target Bit Depth + target_bit_depth = 8 # Default + bit_depth_rule = file_type_defs.get(base_map_type, {}).get('bit_depth_rule', 'force_8bit') + if bit_depth_rule not in ['force_8bit', 'respect_inputs']: + logger.warning(f"Unknown bit_depth_rule '{bit_depth_rule}' for map type '{base_map_type}'. Defaulting to 'force_8bit'.") + bit_depth_rule = 'force_8bit' + + if bit_depth_rule == 'respect_inputs': + # Check if any source bit depth is > 8, ignoring None + if any(depth is not None and depth > 8 for depth in source_bit_depth_info): + target_bit_depth = 16 + else: + target_bit_depth = 8 + logger.info(f"Bit depth rule 'respect_inputs' applied. Source bit depths: {source_bit_depth_info}. Target bit depth: {target_bit_depth}") + else: # force_8bit + target_bit_depth = 8 + logger.info(f"Bit depth rule 'force_8bit' applied. Target bit depth: {target_bit_depth}") + + + # 3. Determine Output File Format(s) + if target_bit_depth == 8: + output_ext = output_format_8bit.lstrip('.').lower() + elif target_bit_depth == 16: + # Prioritize primary, fallback to fallback if primary is not supported/desired + # For now, just use primary. More complex logic might be needed later. + output_ext = output_format_16bit_primary.lstrip('.').lower() + # Basic fallback logic example (can be expanded) + if output_ext not in ['png', 'tif']: # Assuming common 16-bit formats + output_ext = output_format_16bit_fallback.lstrip('.').lower() + logger.warning(f"Primary 16-bit format '{output_format_16bit_primary}' might not be suitable. Using fallback '{output_format_16bit_fallback}'.") + else: + logger.error(f"Unsupported target bit depth: {target_bit_depth}. Defaulting to 8-bit format.") + output_ext = output_format_8bit.lstrip('.').lower() + + logger.info(f"Target bit depth: {target_bit_depth}, Output format: {output_ext}") + + # 4. Generate and Save Resolution Variants + # Sort resolutions by max dimension descending + sorted_resolutions = sorted(image_resolutions.items(), key=lambda item: item[1], reverse=True) + + for res_key, res_max_dim in sorted_resolutions: + logger.info(f"Processing resolution variant: {res_key} ({res_max_dim} max dim)") + + # Calculate target dimensions, ensuring no upscaling + if source_max_dim <= res_max_dim: + # If source is smaller or equal, use source dimensions + target_w_res, target_h_res = source_w, source_h + if source_max_dim < res_max_dim: + logger.info(f"Source image ({source_w}x{source_h}) is smaller than target resolution {res_key} ({res_max_dim}). Saving at source resolution.") + else: + # Downscale, maintaining aspect ratio + aspect_ratio = source_w / source_h + if source_w > source_h: + target_w_res = res_max_dim + target_h_res = int(res_max_dim / aspect_ratio) + else: + target_h_res = res_max_dim + target_w_res = int(res_max_dim * aspect_ratio) + logger.info(f"Resizing source image ({source_w}x{source_h}) to {target_w_res}x{target_h_res} for {res_key} variant.") + + + # Resize source_image_data + # Use INTER_AREA for downscaling, INTER_LINEAR or INTER_CUBIC for upscaling (though we avoid upscaling here) + interpolation_method = cv2.INTER_AREA # Good for downscaling + # If we were allowing upscaling, we might add logic like: + # if target_w_res > source_w or target_h_res > source_h: + # interpolation_method = cv2.INTER_LINEAR # Or INTER_CUBIC + + try: + variant_data = ipu.resize_image(source_image_data, (target_w_res, target_h_res), interpolation=interpolation_method) + logger.debug(f"Resized variant data shape: {variant_data.shape}") + except Exception as e: + logger.error(f"Error resizing image for {res_key} variant: {e}") + continue # Skip this variant if resizing fails + + # Filename Construction + current_tokens = output_filename_pattern_tokens.copy() + current_tokens['maptype'] = base_map_type + current_tokens['resolution'] = res_key + current_tokens['ext'] = output_ext + + try: + # Replace placeholders in the pattern + filename = output_filename_pattern + for token, value in current_tokens.items(): + # Ensure value is string for replacement, handle Path objects later + filename = filename.replace(f"[{token}]", str(value)) + + # Construct full output path + output_base_directory = current_tokens.get('output_base_directory') + if not isinstance(output_base_directory, Path): + logger.error(f"'output_base_directory' token is missing or not a Path object: {output_base_directory}. Cannot save file.") + continue # Skip this variant + + output_path = output_base_directory / filename + logger.info(f"Constructed output path: {output_path}") + + # Ensure parent directory exists + output_path.parent.mkdir(parents=True, exist_ok=True) + logger.debug(f"Ensured directory exists: {output_path.parent}") + + except Exception as e: + logger.error(f"Error constructing filepath for {res_key} variant: {e}") + continue # Skip this variant if path construction fails + + + # Prepare Save Parameters + save_params_cv2 = [] + if output_ext == 'jpg': + save_params_cv2.append(cv2.IMWRITE_JPEG_QUALITY) + save_params_cv2.append(jpg_quality) + logger.debug(f"Using JPG quality: {jpg_quality}") + elif output_ext == 'png': + save_params_cv2.append(cv2.IMWRITE_PNG_COMPRESSION) + save_params_cv2.append(png_compression_level) + logger.debug(f"Using PNG compression level: {png_compression_level}") + # Add other format specific parameters if needed (e.g., TIFF compression) + + + # Bit Depth Conversion (just before saving) + image_data_for_save = variant_data + try: + if target_bit_depth == 8: + image_data_for_save = ipu.convert_to_uint8(variant_data) + logger.debug("Converted variant data to uint8.") + elif target_bit_depth == 16: + # ipu.convert_to_uint16 might handle different input types (float, uint8) + # Assuming variant_data might be float after resizing, convert to uint16 + image_data_for_save = ipu.convert_to_uint16(variant_data) + logger.debug("Converted variant data to uint16.") + # Add other bit depth conversions if needed + except Exception as e: + logger.error(f"Error converting image data to target bit depth {target_bit_depth} for {res_key} variant: {e}") + continue # Skip this variant if conversion fails + + + # Saving + try: + # ipu.save_image is expected to handle the actual cv2.imwrite call + success = ipu.save_image(str(output_path), image_data_for_save, params=save_params_cv2) + if success: + logger.info(f"Successfully saved {res_key} variant to {output_path}") + # Collect details for the returned list + saved_file_details.append({ + 'path': str(output_path), + 'resolution_key': res_key, + 'format': output_ext, + 'bit_depth': target_bit_depth, + 'dimensions': (target_w_res, target_h_res) + }) + else: + logger.error(f"Failed to save {res_key} variant to {output_path}") + + except Exception as e: + logger.error(f"Error saving image for {res_key} variant to {output_path}: {e}") + # Continue to next variant even if one fails + + + # Discard in-memory variant after saving (Python's garbage collection handles this) + del variant_data + del image_data_for_save + + + # 5. Return List of Saved File Details + logger.info(f"Finished saving variants for map type: {base_map_type}. Saved {len(saved_file_details)} variants.") + return saved_file_details + +# Optional Helper Functions (can be added here if needed) +# def _determine_target_bit_depth(...): ... +# def _determine_output_format(...): ... +# def _construct_variant_filepath(...): ... \ No newline at end of file