diff --git a/processing/pipeline/stages/gloss_to_rough_conversion.py b/processing/pipeline/stages/gloss_to_rough_conversion.py index 7c61919..2de863c 100644 --- a/processing/pipeline/stages/gloss_to_rough_conversion.py +++ b/processing/pipeline/stages/gloss_to_rough_conversion.py @@ -2,6 +2,7 @@ import logging from pathlib import Path import numpy as np from typing import List +import dataclasses from .base_stage import ProcessingStage from ..asset_context import AssetProcessingContext @@ -35,135 +36,158 @@ class GlossToRoughConversionStage(ProcessingStage): logger.debug(f"Asset '{asset_name_for_log}': Skipping GlossToRoughConversionStage due to skip_asset flag.") return context - if not context.files_to_process or not context.processed_maps_details: + if not context.processed_maps_details: # files_to_process might be empty if only gloss maps existed and all are converted logger.debug( - f"Asset '{asset_name_for_log}': No files to process or processed_maps_details empty " - f"in GlossToRoughConversionStage. Skipping." + f"Asset '{asset_name_for_log}': processed_maps_details is empty in GlossToRoughConversionStage. Skipping." ) return context - new_files_to_process: List[FileRule] = [] + # Start with a copy of the current file rules. We will modify this list. + new_files_to_process: List[FileRule] = list(context.files_to_process) if context.files_to_process else [] processed_a_gloss_map = False + successful_conversion_statuses = ['BasePOTSaved', 'Processed_With_Variants', 'Processed_No_Variants'] - logger.info(f"Asset '{asset_name_for_log}': Starting Gloss to Roughness Conversion Stage.") + logger.info(f"Asset '{asset_name_for_log}': Starting Gloss to Roughness Conversion Stage. Examining {len(context.processed_maps_details)} processed map entries.") - for idx, file_rule in enumerate(context.files_to_process): - # Assuming FileRule has 'map_type' and 'id' (with a .hex attribute) and 'source_file_path' - # These might need to be checked with hasattr if they are optional or could be missing - if hasattr(file_rule, 'map_type') and file_rule.map_type == "GLOSS": - if not hasattr(file_rule, 'id') or not hasattr(file_rule.id, 'hex'): - logger.warning(f"Asset '{asset_name_for_log}': GLOSS FileRule missing 'id.hex'. Skipping conversion for this rule: {file_rule}") - new_files_to_process.append(file_rule) - continue - map_detail_key = file_rule.id.hex + # Iterate using the index (map_key_index) as the key, which is now standard. + for map_key_index, map_details in context.processed_maps_details.items(): + processing_map_type = map_details.get('processing_map_type', '') + map_status = map_details.get('status') + original_temp_path_str = map_details.get('temp_processed_file') + # source_file_rule_idx from details should align with map_key_index. + # We primarily use map_key_index for accessing FileRule from context.files_to_process. + source_file_rule_idx_from_details = map_details.get('source_file_rule_index') + processing_tag = map_details.get('processing_tag') + + if map_key_index != source_file_rule_idx_from_details: + logger.warning( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: Mismatch between map key index and 'source_file_rule_index' ({source_file_rule_idx_from_details}) in details. " + f"Using map_key_index ({map_key_index}) for FileRule lookup. This might indicate a data consistency issue from previous stage." + ) + + if not processing_tag: + logger.warning(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: 'processing_tag' is missing in map_details. Using a fallback for temp filename. This is unexpected.") + processing_tag = f"mki_{map_key_index}_fallback_tag" + + + if not processing_map_type.startswith("MAP_GLOSS"): + # logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: Type '{processing_map_type}' is not GLOSS. Skipping.") + continue + + logger.info(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Identified potential GLOSS map (Type: {processing_map_type}).") + + if map_status not in successful_conversion_statuses: + logger.warning( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): Status '{map_status}' is not one of {successful_conversion_statuses}. " + f"Skipping conversion for this map." + ) + continue + + if not original_temp_path_str: + logger.warning( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): 'temp_processed_file' missing in details. " + f"Skipping conversion." + ) + continue + + original_temp_path = Path(original_temp_path_str) + if not original_temp_path.exists(): + logger.error( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): Temporary file {original_temp_path_str} " + f"does not exist. Skipping conversion." + ) + continue + + # Use map_key_index directly to access the FileRule + # Ensure map_key_index is a valid index for context.files_to_process + if not isinstance(map_key_index, int) or map_key_index < 0 or map_key_index >= len(context.files_to_process): + logger.error( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): Invalid map_key_index ({map_key_index}) for accessing files_to_process (len: {len(context.files_to_process)}). " + f"Skipping conversion." + ) + continue + + original_file_rule = context.files_to_process[map_key_index] + source_file_path_for_log = original_file_rule.file_path if hasattr(original_file_rule, 'file_path') else "Unknown source path" + logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Processing GLOSS map from '{original_temp_path_str}' (Original FileRule path: '{source_file_path_for_log}') for conversion.") + + image_data = ipu.load_image(str(original_temp_path)) + if image_data is None: + logger.error( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Failed to load image data from {original_temp_path_str}. " + f"Skipping conversion." + ) + continue + + # Perform Inversion + inverted_image_data: np.ndarray + if np.issubdtype(image_data.dtype, np.floating): + inverted_image_data = 1.0 - image_data + inverted_image_data = np.clip(inverted_image_data, 0.0, 1.0) + logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Inverted float image data.") + elif np.issubdtype(image_data.dtype, np.integer): + max_val = np.iinfo(image_data.dtype).max + inverted_image_data = max_val - image_data + logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Inverted integer image data (max_val: {max_val}).") + else: + logger.error( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Unsupported image data type {image_data.dtype} " + f"for GLOSS map. Cannot invert. Skipping conversion." + ) + continue + + # Save New Temporary (Roughness) Map + new_temp_filename = f"rough_from_gloss_{processing_tag}{original_temp_path.suffix}" + new_temp_path = context.engine_temp_dir / new_temp_filename + + save_success = ipu.save_image(str(new_temp_path), inverted_image_data) + + if save_success: + logger.info( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Converted GLOSS map {original_temp_path_str} " + f"to ROUGHNESS map {new_temp_path}." + ) - source_file_path_for_log = file_rule.source_file_path if hasattr(file_rule, 'source_file_path') else "Unknown source path" - - if map_detail_key not in context.processed_maps_details: - logger.warning( - f"Asset '{asset_name_for_log}': GLOSS map '{source_file_path_for_log}' " - f"(ID: {map_detail_key}) found in files_to_process but not in processed_maps_details. " - f"Adding original rule and skipping conversion for this map." - ) - new_files_to_process.append(file_rule) - continue - - map_details = context.processed_maps_details[map_detail_key] + update_dict = {'item_type': "MAP_ROUGH", 'item_type_override': "MAP_ROUGH"} - if map_details.get('status') != 'Processed' or 'temp_processed_file' not in map_details: - logger.warning( - f"Asset '{asset_name_for_log}': GLOSS map '{source_file_path_for_log}' " - f"(ID: {map_detail_key}) not successfully processed by previous stage or temp file missing. " - f"Status: {map_details.get('status')}. Adding original rule and skipping conversion." - ) - new_files_to_process.append(file_rule) - continue - - original_temp_path_str = map_details['temp_processed_file'] - original_temp_path = Path(original_temp_path_str) - - if not original_temp_path.exists(): - logger.error( - f"Asset '{asset_name_for_log}': Temporary file {original_temp_path_str} for GLOSS map " - f"(ID: {map_detail_key}) does not exist. Adding original rule and skipping conversion." - ) - new_files_to_process.append(file_rule) - continue - - logger.debug(f"Asset '{asset_name_for_log}': Processing GLOSS map {original_temp_path} for conversion.") - image_data = ipu.load_image(original_temp_path) - - if image_data is None: - logger.error( - f"Asset '{asset_name_for_log}': Failed to load image data from {original_temp_path} " - f"for GLOSS map (ID: {map_detail_key}). Adding original rule and skipping conversion." - ) - new_files_to_process.append(file_rule) - continue - - # Perform Inversion - inverted_image_data: np.ndarray - if np.issubdtype(image_data.dtype, np.floating): - inverted_image_data = 1.0 - image_data - inverted_image_data = np.clip(inverted_image_data, 0.0, 1.0) # Ensure range for floats - logger.debug(f"Asset '{asset_name_for_log}': Inverted float image data for {original_temp_path}.") - elif np.issubdtype(image_data.dtype, np.integer): - max_val = np.iinfo(image_data.dtype).max - inverted_image_data = max_val - image_data - logger.debug(f"Asset '{asset_name_for_log}': Inverted integer image data (max_val: {max_val}) for {original_temp_path}.") + modified_file_rule: Optional[FileRule] = None + if hasattr(original_file_rule, 'model_copy') and callable(original_file_rule.model_copy): # Pydantic + modified_file_rule = original_file_rule.model_copy(update=update_dict) + elif dataclasses.is_dataclass(original_file_rule): # Dataclass + modified_file_rule = dataclasses.replace(original_file_rule, **update_dict) else: - logger.error( - f"Asset '{asset_name_for_log}': Unsupported image data type {image_data.dtype} " - f"for GLOSS map {original_temp_path}. Cannot invert. Adding original rule." - ) - new_files_to_process.append(file_rule) + logger.error(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Original FileRule is neither Pydantic nor dataclass. Cannot modify. Skipping update for this rule.") continue - # Save New Temporary (Roughness) Map - # Using original_temp_path.suffix ensures we keep the format (e.g., .png, .exr) - # Ensure file_rule.map_type exists before using sanitize_filename - map_type_for_filename = file_rule.map_type if hasattr(file_rule, 'map_type') else "unknownmaptype" - new_temp_filename = f"rough_from_gloss_{sanitize_filename(map_type_for_filename)}_{file_rule.id.hex}{original_temp_path.suffix}" - new_temp_path = context.engine_temp_dir / new_temp_filename + new_files_to_process[map_key_index] = modified_file_rule # Replace using map_key_index - save_success = ipu.save_image(new_temp_path, inverted_image_data) - - if save_success: - logger.info( - f"Asset '{asset_name_for_log}': Converted GLOSS map {original_temp_path} " - f"to ROUGHNESS map {new_temp_path}." - ) - - # Assuming FileRule has model_copy method - modified_file_rule = file_rule.model_copy(deep=True) if hasattr(file_rule, 'model_copy') else file_rule - modified_file_rule.map_type = "ROUGHNESS" # Ensure map_type can be set - - # Update context.processed_maps_details for the original file_rule.id.hex - context.processed_maps_details[map_detail_key]['temp_processed_file'] = str(new_temp_path) - context.processed_maps_details[map_detail_key]['original_map_type_before_conversion'] = "GLOSS" - context.processed_maps_details[map_detail_key]['notes'] = "Converted from GLOSS by GlossToRoughConversionStage" - - new_files_to_process.append(modified_file_rule) - processed_a_gloss_map = True - else: - logger.error( - f"Asset '{asset_name_for_log}': Failed to save inverted ROUGHNESS map to {new_temp_path} " - f"for original GLOSS map (ID: {map_detail_key}). Adding original rule." - ) - new_files_to_process.append(file_rule) - else: # Not a gloss map - new_files_to_process.append(file_rule) + # Update context.processed_maps_details for this map_key_index + map_details['temp_processed_file'] = str(new_temp_path) + map_details['original_map_type_before_conversion'] = processing_map_type + map_details['processing_map_type'] = "MAP_ROUGH" + map_details['map_type'] = "Roughness" + map_details['status'] = "Converted_To_Rough" + map_details['notes'] = map_details.get('notes', '') + "; Converted from GLOSS by GlossToRoughConversionStage" + if 'base_pot_resolution_name' in map_details: + map_details['processed_resolution_name'] = map_details['base_pot_resolution_name'] + processed_a_gloss_map = True + else: + logger.error( + f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Failed to save inverted ROUGHNESS map to {new_temp_path}. " + f"Original GLOSS FileRule remains." + ) + context.files_to_process = new_files_to_process if processed_a_gloss_map: logger.info( - f"Asset '{asset_name_for_log}': Gloss to Roughness conversion stage successfully processed one or more maps and updated file list." + f"Asset '{asset_name_for_log}': Gloss to Roughness conversion stage finished. Processed one or more maps and updated file list and map details." ) else: - logger.debug( - f"Asset '{asset_name_for_log}': No gloss maps were successfully converted in GlossToRoughConversionStage. " - f"File list for next stage contains original non-gloss maps and any gloss maps that failed conversion." + logger.info( + f"Asset '{asset_name_for_log}': No gloss maps were converted in GlossToRoughConversionStage. " + f"File list for next stage contains original non-gloss maps and any gloss maps that failed or were ineligible for conversion." ) return context \ No newline at end of file diff --git a/processing/pipeline/stages/individual_map_processing.py b/processing/pipeline/stages/individual_map_processing.py index 71614e6..acbe8bd 100644 --- a/processing/pipeline/stages/individual_map_processing.py +++ b/processing/pipeline/stages/individual_map_processing.py @@ -48,9 +48,9 @@ class IndividualMapProcessingStage(ProcessingStage): context.status_flags['individual_map_processing_failed'] = True # Mark all file_rules as failed for fr_idx, file_rule_to_fail in enumerate(context.files_to_process): - temp_id_for_fail = f"fr_fail_{fr_idx}" # Temporary ID for status update + # Use fr_idx as the key for status update for these early failures map_type_for_fail = file_rule_to_fail.item_type_override or file_rule_to_fail.item_type or "UnknownMapType" - self._update_file_rule_status(context, temp_id_for_fail, 'Failed', map_type=map_type_for_fail, details="SourceRule.input_path missing") + self._update_file_rule_status(context, fr_idx, 'Failed', map_type=map_type_for_fail, details="SourceRule.input_path missing") return context # The workspace_path in the context should be the directory where files are extracted/available. @@ -59,9 +59,9 @@ class IndividualMapProcessingStage(ProcessingStage): logger.error(f"Asset '{asset_name_for_log}': Workspace path '{source_base_path}' is not a valid directory.") context.status_flags['individual_map_processing_failed'] = True for fr_idx, file_rule_to_fail in enumerate(context.files_to_process): - temp_id_for_fail = f"fr_fail_{fr_idx}" # Use a temporary unique ID for this status update + # Use fr_idx as the key for status update map_type_for_fail = file_rule_to_fail.item_type_override or file_rule_to_fail.item_type or "UnknownMapType" - self._update_file_rule_status(context, temp_id_for_fail, 'Failed', map_type=map_type_for_fail, details="Workspace path invalid") + self._update_file_rule_status(context, fr_idx, 'Failed', map_type=map_type_for_fail, details="Workspace path invalid") return context # Fetch config settings once before the loop @@ -70,8 +70,17 @@ class IndividualMapProcessingStage(ProcessingStage): output_filename_pattern = getattr(context.config_obj, "output_filename_pattern", "[assetname]_[maptype]_[resolution].[ext]") for file_rule_idx, file_rule in enumerate(context.files_to_process): - # Generate a unique ID for this file_rule processing instance for processed_maps_details - current_map_id_hex = f"map_{file_rule_idx}_{uuid.uuid4().hex[:8]}" + # file_rule_idx will be the key for processed_maps_details. + # processing_instance_tag is for unique temp files and detailed logging for this specific run. + processing_instance_tag = f"map_{file_rule_idx}_{uuid.uuid4().hex[:8]}" + current_map_key = file_rule_idx # Key for processed_maps_details + + if not file_rule.file_path: # Ensure file_path exists, critical for later stages if they rely on it from FileRule + logger.error(f"Asset '{asset_name_for_log}', FileRule at index {file_rule_idx} has an empty or None file_path. Skipping this rule.") + self._update_file_rule_status(context, current_map_key, 'Failed', + processing_tag=processing_instance_tag, + details="FileRule has no file_path") + continue initial_current_map_type = file_rule.item_type_override or file_rule.item_type or "UnknownMapType" @@ -130,89 +139,98 @@ class IndividualMapProcessingStage(ProcessingStage): # --- END NEW SUFFIXING LOGIC --- # --- START: Filename-friendly map type derivation --- - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: --- Starting Filename-Friendly Map Type Logic for: {current_map_type} ---") + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: --- Starting Filename-Friendly Map Type Logic for: {current_map_type} ---") filename_friendly_map_type = current_map_type # Fallback # 1. Access FILE_TYPE_DEFINITIONS file_type_definitions = None - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Attempting to access context.config_obj.FILE_TYPE_DEFINITIONS.") + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Attempting to access context.config_obj.FILE_TYPE_DEFINITIONS.") try: file_type_definitions = context.config_obj.FILE_TYPE_DEFINITIONS if not file_type_definitions: # Check if it's None or empty - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: FILE_TYPE_DEFINITIONS is present but empty or None.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS is present but empty or None.") else: sample_defs_log = {k: file_type_definitions[k] for k in list(file_type_definitions.keys())[:2]} # Log first 2 for brevity - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Accessed FILE_TYPE_DEFINITIONS. Sample: {sample_defs_log}, Total keys: {len(file_type_definitions)}.") + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Accessed FILE_TYPE_DEFINITIONS. Sample: {sample_defs_log}, Total keys: {len(file_type_definitions)}.") except AttributeError: - logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Could not access context.config_obj.FILE_TYPE_DEFINITIONS via direct attribute.") + logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Could not access context.config_obj.FILE_TYPE_DEFINITIONS via direct attribute.") - base_map_key = None + base_map_key_val = None # Renamed from base_map_key to avoid conflict with current_map_key suffix_part = "" if file_type_definitions and isinstance(file_type_definitions, dict) and len(file_type_definitions) > 0: - base_map_key = None + base_map_key_val = None suffix_part = "" sorted_known_base_keys = sorted(list(file_type_definitions.keys()), key=len, reverse=True) - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Sorted known base keys for parsing: {sorted_known_base_keys}") + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Sorted known base keys for parsing: {sorted_known_base_keys}") for known_key in sorted_known_base_keys: - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Checking if '{current_map_type}' starts with '{known_key}'") + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Checking if '{current_map_type}' starts with '{known_key}'") if current_map_type.startswith(known_key): - base_map_key = known_key + base_map_key_val = known_key suffix_part = current_map_type[len(known_key):] - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Match found! current_map_type: '{current_map_type}', base_map_key: '{base_map_key}', suffix_part: '{suffix_part}'") + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Match found! current_map_type: '{current_map_type}', base_map_key_val: '{base_map_key_val}', suffix_part: '{suffix_part}'") break - if base_map_key is None: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Could not parse base_map_key from '{current_map_type}' using known keys. Fallback: filename_friendly_map_type = '{filename_friendly_map_type}'.") + if base_map_key_val is None: + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Could not parse base_map_key_val from '{current_map_type}' using known keys. Fallback: filename_friendly_map_type = '{filename_friendly_map_type}'.") else: - definition = file_type_definitions.get(base_map_key) - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Definition for '{base_map_key}': {definition}") + definition = file_type_definitions.get(base_map_key_val) + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Definition for '{base_map_key_val}': {definition}") if definition and isinstance(definition, dict): standard_type_alias = definition.get("standard_type") - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Standard type alias for '{base_map_key}': '{standard_type_alias}'") + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Standard type alias for '{base_map_key_val}': '{standard_type_alias}'") if standard_type_alias and isinstance(standard_type_alias, str) and standard_type_alias.strip(): filename_friendly_map_type = standard_type_alias.strip() + suffix_part - logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Successfully transformed map type: '{current_map_type}' -> '{filename_friendly_map_type}' (standard_type_alias: '{standard_type_alias}', suffix_part: '{suffix_part}').") + logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Successfully transformed map type: '{current_map_type}' -> '{filename_friendly_map_type}' (standard_type_alias: '{standard_type_alias}', suffix_part: '{suffix_part}').") else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Standard type alias for '{base_map_key}' is missing, empty, or not a string (value: '{standard_type_alias}'). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Standard type alias for '{base_map_key_val}' is missing, empty, or not a string (value: '{standard_type_alias}'). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: No definition or invalid definition for '{base_map_key}' (value: {definition}). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: No definition or invalid definition for '{base_map_key_val}' (value: {definition}). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") elif file_type_definitions is None: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: FILE_TYPE_DEFINITIONS not available for lookup (was None). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS not available for lookup (was None). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") elif not isinstance(file_type_definitions, dict): - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: FILE_TYPE_DEFINITIONS is not a dictionary (type: {type(file_type_definitions)}). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS is not a dictionary (type: {type(file_type_definitions)}). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: FILE_TYPE_DEFINITIONS is an empty dictionary. Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS is an empty dictionary. Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.") - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Final filename_friendly_map_type: '{filename_friendly_map_type}'") + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Final filename_friendly_map_type: '{filename_friendly_map_type}'") # --- END: Filename-friendly map type derivation --- if not current_map_type or not current_map_type.startswith("MAP_") or current_map_type == "MAP_GEN_COMPOSITE": logger.debug(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}': Skipping, item_type '{current_map_type}' (initial: '{initial_current_map_type}') not targeted for individual processing.") continue - logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Type: {current_map_type}, Initial Type: {initial_current_map_type}, ID: {current_map_id_hex}): Starting individual processing.") + logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Type: {current_map_type}, Initial Type: {initial_current_map_type}, Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Starting individual processing.") # A. Find Source File (using file_rule.file_path as the pattern relative to source_base_path) - # The _find_source_file might need adjustment if file_rule.file_path is absolute or needs complex globbing. - # For now, assume file_rule.file_path is a relative pattern or exact name. - source_file_path = self._find_source_file(source_base_path, file_rule.file_path, asset_name_for_log, current_map_id_hex) + source_file_path = self._find_source_file(source_base_path, file_rule.file_path, asset_name_for_log, processing_instance_tag) if not source_file_path: - logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Source file not found with path/pattern '{file_rule.file_path}' in '{source_base_path}'.") - self._update_file_rule_status(context, current_map_id_hex, 'Failed', map_type=filename_friendly_map_type, details="Source file not found") + logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Source file not found with path/pattern '{file_rule.file_path}' in '{source_base_path}'.") + self._update_file_rule_status(context, current_map_key, 'Failed', + map_type=filename_friendly_map_type, + processing_map_type=current_map_type, + source_file_rule_index=file_rule_idx, + processing_tag=processing_instance_tag, + details="Source file not found") continue # B. Load and Transform Image image_data: Optional[np.ndarray] = ipu.load_image(str(source_file_path)) if image_data is None: - logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Failed to load image from '{source_file_path}'.") - self._update_file_rule_status(context, current_map_id_hex, 'Failed', map_type=filename_friendly_map_type, source_file=str(source_file_path), details="Image load failed") + logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Failed to load image from '{source_file_path}'.") + self._update_file_rule_status(context, current_map_key, 'Failed', + map_type=filename_friendly_map_type, + processing_map_type=current_map_type, + source_file_rule_index=file_rule_idx, + processing_tag=processing_instance_tag, + source_file=str(source_file_path), + details="Image load failed") continue original_height, original_width = image_data.shape[:2] - logger.debug(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Loaded image '{source_file_path}' with dimensions {original_width}x{original_height}.") + logger.debug(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Loaded image '{source_file_path}' with dimensions {original_width}x{original_height}.") # 1. Initial Power-of-Two (POT) Downscaling pot_width = ipu.get_nearest_power_of_two_downscale(original_width) @@ -286,7 +304,7 @@ class IndividualMapProcessingStage(ProcessingStage): base_pot_width, base_pot_height = 1, 1 - logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Original dims: ({original_width},{original_height}), Initial POT Scaled Dims: ({base_pot_width},{base_pot_height}).") + logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Original dims: ({original_width},{original_height}), Initial POT Scaled Dims: ({base_pot_width},{base_pot_height}).") # Calculate and store aspect ratio change string if original_width > 0 and original_height > 0 and base_pot_width > 0 and base_pot_height > 0: @@ -297,19 +315,26 @@ class IndividualMapProcessingStage(ProcessingStage): if aspect_change_str: # This will overwrite if multiple maps are processed; specified by requirements. context.asset_metadata['aspect_ratio_change_string'] = aspect_change_str - logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type {current_map_type}: Calculated aspect ratio change string: '{aspect_change_str}' (Original: {original_width}x{original_height}, Base POT: {base_pot_width}x{base_pot_height}). Stored in asset_metadata.") + logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type {current_map_type}: Calculated aspect ratio change string: '{aspect_change_str}' (Original: {original_width}x{original_height}, Base POT: {base_pot_width}x{base_pot_height}). Stored in asset_metadata.") else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type {current_map_type}: Failed to calculate aspect ratio change string.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type {current_map_type}: Failed to calculate aspect ratio change string.") else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type {current_map_type}: Skipping aspect ratio change string calculation due to invalid dimensions (Original: {original_width}x{original_height}, Base POT: {base_pot_width}x{base_pot_height}).") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type {current_map_type}: Skipping aspect ratio change string calculation due to invalid dimensions (Original: {original_width}x{original_height}, Base POT: {base_pot_width}x{base_pot_height}).") base_pot_image_data = image_data.copy() if (base_pot_width, base_pot_height) != (original_width, original_height): interpolation = cv2.INTER_AREA # Good for downscaling base_pot_image_data = ipu.resize_image(base_pot_image_data, base_pot_width, base_pot_height, interpolation=interpolation) if base_pot_image_data is None: - logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Failed to resize image to base POT dimensions.") - self._update_file_rule_status(context, current_map_id_hex, 'Failed', map_type=filename_friendly_map_type, source_file=str(source_file_path), original_dimensions=(original_width, original_height), details="Base POT resize failed") + logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Failed to resize image to base POT dimensions.") + self._update_file_rule_status(context, current_map_key, 'Failed', + map_type=filename_friendly_map_type, + processing_map_type=current_map_type, + source_file_rule_index=file_rule_idx, + processing_tag=processing_instance_tag, + source_file=str(source_file_path), + original_dimensions=(original_width, original_height), + details="Base POT resize failed") continue # Color Profile Management (after initial POT resize, before multi-res saving) @@ -323,14 +348,14 @@ class IndividualMapProcessingStage(ProcessingStage): custom_transform_settings = file_rule.channel_merge_instructions['transform'] if isinstance(custom_transform_settings, dict): transform_settings.update(custom_transform_settings) - logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Loaded transform settings for color/output from file_rule.") + logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Loaded transform settings for color/output from file_rule.") if transform_settings['color_profile_management'] and transform_settings['target_color_profile'] == "RGB": if len(base_pot_image_data.shape) == 3 and base_pot_image_data.shape[2] == 3: # BGR to RGB - logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Converting BGR to RGB for base POT image.") + logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Converting BGR to RGB for base POT image.") base_pot_image_data = ipu.convert_bgr_to_rgb(base_pot_image_data) elif len(base_pot_image_data.shape) == 3 and base_pot_image_data.shape[2] == 4: # BGRA to RGBA - logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Converting BGRA to RGBA for base POT image.") + logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Converting BGRA to RGBA for base POT image.") base_pot_image_data = ipu.convert_bgra_to_rgba(base_pot_image_data) # Ensure engine_temp_dir exists before saving base POT @@ -340,11 +365,17 @@ class IndividualMapProcessingStage(ProcessingStage): logger.info(f"Asset '{asset_name_for_log}': Created engine_temp_dir at '{context.engine_temp_dir}'") except OSError as e: logger.error(f"Asset '{asset_name_for_log}': Failed to create engine_temp_dir '{context.engine_temp_dir}': {e}") - self._update_file_rule_status(context, current_map_id_hex, 'Failed', map_type=filename_friendly_map_type, source_file=str(source_file_path), details="Failed to create temp directory for base POT") + self._update_file_rule_status(context, current_map_key, 'Failed', + map_type=filename_friendly_map_type, + processing_map_type=current_map_type, + source_file_rule_index=file_rule_idx, + processing_tag=processing_instance_tag, + source_file=str(source_file_path), + details="Failed to create temp directory for base POT") continue temp_filename_suffix = Path(source_file_path).suffix - base_pot_temp_filename = f"{current_map_id_hex}_basePOT{temp_filename_suffix}" + base_pot_temp_filename = f"{processing_instance_tag}_basePOT{temp_filename_suffix}" # Use processing_instance_tag base_pot_temp_path = context.engine_temp_dir / base_pot_temp_filename # Determine save parameters for base POT image (can be different from variants if needed) @@ -354,18 +385,29 @@ class IndividualMapProcessingStage(ProcessingStage): # For now, using simple save. if not ipu.save_image(str(base_pot_temp_path), base_pot_image_data, params=base_save_params): - logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Failed to save base POT image to '{base_pot_temp_path}'.") - self._update_file_rule_status(context, current_map_id_hex, 'Failed', map_type=filename_friendly_map_type, source_file=str(source_file_path), original_dimensions=(original_width, original_height), base_pot_dimensions=(base_pot_width, base_pot_height), details="Base POT image save failed") + logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Failed to save base POT image to '{base_pot_temp_path}'.") + self._update_file_rule_status(context, current_map_key, 'Failed', + map_type=filename_friendly_map_type, + processing_map_type=current_map_type, + source_file_rule_index=file_rule_idx, + processing_tag=processing_instance_tag, + source_file=str(source_file_path), + original_dimensions=(original_width, original_height), + base_pot_dimensions=(base_pot_width, base_pot_height), + details="Base POT image save failed") continue - logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Successfully saved base POT image to '{base_pot_temp_path}' with dims ({base_pot_width}x{base_pot_height}).") + logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Successfully saved base POT image to '{base_pot_temp_path}' with dims ({base_pot_width}x{base_pot_height}).") # Initialize/update the status for this map in processed_maps_details self._update_file_rule_status( context, - current_map_id_hex, + current_map_key, # Use file_rule_idx as key 'BasePOTSaved', # Intermediate status, will be updated after variant check map_type=filename_friendly_map_type, + processing_map_type=current_map_type, + source_file_rule_index=file_rule_idx, + processing_tag=processing_instance_tag, # Store the tag source_file=str(source_file_path), original_dimensions=(original_width, original_height), base_pot_dimensions=(base_pot_width, base_pot_height), @@ -375,20 +417,20 @@ class IndividualMapProcessingStage(ProcessingStage): # 2. Multiple Resolution Output (Variants) processed_at_least_one_resolution_variant = False # Resolution variants are attempted for all map types individually processed. - # The filter at the beginning of the loop (around line 72) ensures only relevant maps reach this stage. + # The filter at the beginning of the loop ensures only relevant maps reach this stage. generate_variants_for_this_map_type = True if generate_variants_for_this_map_type: # This will now always be true if code execution reaches here - logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Map type '{current_map_type}' is eligible for individual processing. Attempting to generate resolution variants.") + logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Map type '{current_map_type}' is eligible for individual processing. Attempting to generate resolution variants.") # Sort resolutions from largest to smallest sorted_resolutions = sorted(image_resolutions.items(), key=lambda item: item[1], reverse=True) - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Sorted resolutions for variant processing: {sorted_resolutions}") + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Sorted resolutions for variant processing: {sorted_resolutions}") for res_key, res_max_dim in sorted_resolutions: current_w, current_h = base_pot_image_data.shape[1], base_pot_image_data.shape[0] if current_w <= 0 or current_h <=0: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Base POT image has zero dimension ({current_w}x{current_h}). Skipping this resolution variant.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Base POT image has zero dimension ({current_w}x{current_h}). Skipping this resolution variant.") continue if max(current_w, current_h) >= res_max_dim: @@ -401,24 +443,24 @@ class IndividualMapProcessingStage(ProcessingStage): target_h_res = res_max_dim target_w_res = max(1, round(target_h_res * (current_w / current_h))) else: - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Base POT image ({current_w}x{current_h}) is smaller than target max dim {res_max_dim}. Skipping this resolution variant.") + logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Base POT image ({current_w}x{current_h}) is smaller than target max dim {res_max_dim}. Skipping this resolution variant.") continue target_w_res = min(target_w_res, current_w) target_h_res = min(target_h_res, current_h) if target_w_res <=0 or target_h_res <=0: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Calculated target variant dims are zero or negative ({target_w_res}x{target_h_res}). Skipping.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Calculated target variant dims are zero or negative ({target_w_res}x{target_h_res}). Skipping.") continue - logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Processing variant for {res_max_dim}. Base POT Dims: ({current_w}x{current_h}), Target Dims for {res_key}: ({target_w_res}x{target_h_res}).") + logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Processing variant for {res_max_dim}. Base POT Dims: ({current_w}x{current_h}), Target Dims for {res_key}: ({target_w_res}x{target_h_res}).") output_image_data_for_res = base_pot_image_data if (target_w_res, target_h_res) != (current_w, current_h): interpolation_res = cv2.INTER_AREA output_image_data_for_res = ipu.resize_image(base_pot_image_data, target_w_res, target_h_res, interpolation=interpolation_res) if output_image_data_for_res is None: - logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Failed to resize image for resolution variant {res_key}.") + logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Failed to resize image for resolution variant {res_key}.") continue assetname_placeholder = context.asset_rule.asset_name if context.asset_rule else "UnknownAsset" @@ -431,7 +473,7 @@ class IndividualMapProcessingStage(ProcessingStage): .replace("[maptype]", sanitize_filename(filename_friendly_map_type)) \ .replace("[resolution]", sanitize_filename(resolution_placeholder)) \ .replace("[ext]", output_ext_variant) - temp_output_filename_variant = f"{current_map_id_hex}_variant_{temp_output_filename_variant}" # Distinguish variant temp files + temp_output_filename_variant = f"{processing_instance_tag}_variant_{temp_output_filename_variant}" # Use processing_instance_tag temp_output_path_variant = context.engine_temp_dir / temp_output_filename_variant save_params_variant = [] @@ -446,26 +488,26 @@ class IndividualMapProcessingStage(ProcessingStage): save_success_variant = ipu.save_image(str(temp_output_path_variant), output_image_data_for_res, params=save_params_variant) if not save_success_variant: - logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Failed to save temporary variant image to '{temp_output_path_variant}'.") + logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Failed to save temporary variant image to '{temp_output_path_variant}'.") continue - logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Successfully saved temporary variant map to '{temp_output_path_variant}' with dims ({target_w_res}x{target_h_res}).") + logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Successfully saved temporary variant map to '{temp_output_path_variant}' with dims ({target_w_res}x{target_h_res}).") processed_at_least_one_resolution_variant = True - if 'variants' not in context.processed_maps_details[current_map_id_hex]: - context.processed_maps_details[current_map_id_hex]['variants'] = [] + if 'variants' not in context.processed_maps_details[current_map_key]: # Use current_map_key (file_rule_idx) + context.processed_maps_details[current_map_key]['variants'] = [] - context.processed_maps_details[current_map_id_hex]['variants'].append({ + context.processed_maps_details[current_map_key]['variants'].append({ # Use current_map_key (file_rule_idx) 'resolution_key': res_key, - 'temp_path': str(temp_output_path_variant), # Changed 'path' to 'temp_path' + 'temp_path': str(temp_output_path_variant), 'dimensions': (target_w_res, target_h_res), - 'resolution_name': f"{target_w_res}x{target_h_res}" # Retain for potential use + 'resolution_name': f"{target_w_res}x{target_h_res}" }) if 'processed_files' not in context.asset_metadata: context.asset_metadata['processed_files'] = [] context.asset_metadata['processed_files'].append({ - 'processed_map_key': current_map_id_hex, + 'processed_map_key': current_map_key, # Use current_map_key (file_rule_idx) 'resolution_key': res_key, 'path': str(temp_output_path_variant), 'type': 'temporary_map_variant', @@ -479,11 +521,11 @@ class IndividualMapProcessingStage(ProcessingStage): source_of_stats_image = "unknown" if processed_at_least_one_resolution_variant and \ - current_map_id_hex in context.processed_maps_details and \ - 'variants' in context.processed_maps_details[current_map_id_hex] and \ - context.processed_maps_details[current_map_id_hex]['variants']: + current_map_key in context.processed_maps_details and \ + 'variants' in context.processed_maps_details[current_map_key] and \ + context.processed_maps_details[current_map_key]['variants']: - variants_list = context.processed_maps_details[current_map_id_hex]['variants'] + variants_list = context.processed_maps_details[current_map_key]['variants'] valid_variants_for_stats = [ v for v in variants_list if isinstance(v.get('dimensions'), tuple) and len(v['dimensions']) == 2 and v['dimensions'][0] > 0 and v['dimensions'][1] > 0 @@ -494,25 +536,25 @@ class IndividualMapProcessingStage(ProcessingStage): if smallest_variant and 'temp_path' in smallest_variant and smallest_variant.get('dimensions'): smallest_res_w, smallest_res_h = smallest_variant['dimensions'] - logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Identified smallest variant for stats: {smallest_variant.get('resolution_key', 'N/A')} ({smallest_res_w}x{smallest_res_h}) at {smallest_variant['temp_path']}") + logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Identified smallest variant for stats: {smallest_variant.get('resolution_key', 'N/A')} ({smallest_res_w}x{smallest_res_h}) at {smallest_variant['temp_path']}") lowest_res_image_data_for_stats = ipu.load_image(smallest_variant['temp_path']) image_to_stat_path_for_log = smallest_variant['temp_path'] source_of_stats_image = f"variant {smallest_variant.get('resolution_key', 'N/A')}" if lowest_res_image_data_for_stats is None: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Failed to load smallest variant image '{smallest_variant['temp_path']}' for stats.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Failed to load smallest variant image '{smallest_variant['temp_path']}' for stats.") else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Could not determine smallest variant for stats from valid variants list (details missing).") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Could not determine smallest variant for stats from valid variants list (details missing).") else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: No valid variants found to determine the smallest one for stats.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: No valid variants found to determine the smallest one for stats.") if lowest_res_image_data_for_stats is None: if base_pot_image_data is not None: - logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Using base POT image for stats (dimensions: {base_pot_width}x{base_pot_height}). Smallest variant not available/loaded or no variants generated.") + logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Using base POT image for stats (dimensions: {base_pot_width}x{base_pot_height}). Smallest variant not available/loaded or no variants generated.") lowest_res_image_data_for_stats = base_pot_image_data image_to_stat_path_for_log = f"In-memory base POT image (dims: {base_pot_width}x{base_pot_height})" source_of_stats_image = "base POT" else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Base POT image data is also None. Cannot calculate stats.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Base POT image data is also None. Cannot calculate stats.") if lowest_res_image_data_for_stats is not None: stats_dict = ipu.calculate_image_stats(lowest_res_image_data_for_stats) @@ -520,43 +562,59 @@ class IndividualMapProcessingStage(ProcessingStage): if 'image_stats_lowest_res' not in context.asset_metadata: context.asset_metadata['image_stats_lowest_res'] = {} - context.asset_metadata['image_stats_lowest_res'][current_map_type] = stats_dict - logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type '{current_map_type}': Calculated and stored image stats from '{source_of_stats_image}' (source ref: '{image_to_stat_path_for_log}').") + context.asset_metadata['image_stats_lowest_res'][current_map_type] = stats_dict # Keyed by map_type + logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': Calculated and stored image stats from '{source_of_stats_image}' (source ref: '{image_to_stat_path_for_log}').") elif stats_dict and "error" in stats_dict: - logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type '{current_map_type}': Error calculating image stats from '{source_of_stats_image}': {stats_dict['error']}.") + logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': Error calculating image stats from '{source_of_stats_image}': {stats_dict['error']}.") else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type '{current_map_type}': Failed to calculate image stats from '{source_of_stats_image}' (result was None or empty).") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': Failed to calculate image stats from '{source_of_stats_image}' (result was None or empty).") else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type '{current_map_type}': No image data available (from variant or base POT) to calculate stats.") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': No image data available (from variant or base POT) to calculate stats.") # Final status update based on whether variants were generated (and expected) if generate_variants_for_this_map_type: if processed_at_least_one_resolution_variant: - self._update_file_rule_status(context, current_map_id_hex, 'Processed_With_Variants', map_type=filename_friendly_map_type, details="Successfully processed with multiple resolution variants.") + self._update_file_rule_status(context, current_map_key, 'Processed_With_Variants', + map_type=filename_friendly_map_type, + processing_map_type=current_map_type, + source_file_rule_index=file_rule_idx, + processing_tag=processing_instance_tag, + details="Successfully processed with multiple resolution variants.") else: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Variants were expected for map type '{current_map_type}', but none were generated (e.g., base POT too small for any variant tier).") - self._update_file_rule_status(context, current_map_id_hex, 'Processed_No_Variants', map_type=filename_friendly_map_type, details="Variants expected but none generated (e.g., base POT too small).") + logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Variants were expected for map type '{current_map_type}', but none were generated (e.g., base POT too small for any variant tier).") + self._update_file_rule_status(context, current_map_key, 'Processed_No_Variants', + map_type=filename_friendly_map_type, + processing_map_type=current_map_type, + source_file_rule_index=file_rule_idx, + processing_tag=processing_instance_tag, + details="Variants expected but none generated (e.g., base POT too small).") else: # No variants were expected for this map type - self._update_file_rule_status(context, current_map_id_hex, 'Processed_No_Variants', map_type=filename_friendly_map_type, details="Processed to base POT; variants not applicable for this map type.") + self._update_file_rule_status(context, current_map_key, 'Processed_No_Variants', + map_type=filename_friendly_map_type, + processing_map_type=current_map_type, + source_file_rule_index=file_rule_idx, + processing_tag=processing_instance_tag, + details="Processed to base POT; variants not applicable for this map type.") logger.info(f"Asset '{asset_name_for_log}': Finished individual map processing stage.") return context - def _find_source_file(self, base_path: Path, pattern: str, asset_name_for_log: str, current_map_id_hex: str) -> Optional[Path]: # asset_id -> asset_name_for_log, file_rule_id_hex -> current_map_id_hex + def _find_source_file(self, base_path: Path, pattern: str, asset_name_for_log: str, processing_instance_tag: str) -> Optional[Path]: """ Finds a single source file matching the pattern within the base_path. + Logs use processing_instance_tag for specific run tracing. """ - if not pattern: # pattern is now file_rule.file_path - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Empty file_path provided in FileRule.") + if not pattern: + logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Empty file_path provided in FileRule.") return None # If pattern is an absolute path, use it directly potential_abs_path = Path(pattern) if potential_abs_path.is_absolute() and potential_abs_path.exists(): - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: file_path '{pattern}' is absolute and exists. Using it directly.") + logger.debug(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: file_path '{pattern}' is absolute and exists. Using it directly.") return potential_abs_path elif potential_abs_path.is_absolute(): - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: file_path '{pattern}' is absolute but does not exist.") + logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: file_path '{pattern}' is absolute but does not exist.") # Fall through to try resolving against base_path if it's just a name/relative pattern # Treat pattern as relative to base_path @@ -565,46 +623,49 @@ class IndividualMapProcessingStage(ProcessingStage): # First, check if pattern is an exact relative path exact_match_path = base_path / pattern if exact_match_path.exists() and exact_match_path.is_file(): - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Found exact match for '{pattern}' at '{exact_match_path}'.") + logger.debug(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Found exact match for '{pattern}' at '{exact_match_path}'.") return exact_match_path # If not an exact match, try as a glob pattern (recursive) matched_files_rglob = list(base_path.rglob(pattern)) if matched_files_rglob: if len(matched_files_rglob) > 1: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Multiple files ({len(matched_files_rglob)}) found for pattern '{pattern}' in '{base_path}' (recursive). Using first: {matched_files_rglob[0]}. Files: {matched_files_rglob}") + logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Multiple files ({len(matched_files_rglob)}) found for pattern '{pattern}' in '{base_path}' (recursive). Using first: {matched_files_rglob[0]}. Files: {matched_files_rglob}") return matched_files_rglob[0] # Try non-recursive glob if rglob fails matched_files_glob = list(base_path.glob(pattern)) if matched_files_glob: if len(matched_files_glob) > 1: - logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Multiple files ({len(matched_files_glob)}) found for pattern '{pattern}' in '{base_path}' (non-recursive). Using first: {matched_files_glob[0]}. Files: {matched_files_glob}") + logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Multiple files ({len(matched_files_glob)}) found for pattern '{pattern}' in '{base_path}' (non-recursive). Using first: {matched_files_glob[0]}. Files: {matched_files_glob}") return matched_files_glob[0] - logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: No files found matching pattern '{pattern}' in '{base_path}' (exact, recursive, or non-recursive).") + logger.debug(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: No files found matching pattern '{pattern}' in '{base_path}' (exact, recursive, or non-recursive).") return None except Exception as e: - logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Error searching for file with pattern '{pattern}' in '{base_path}': {e}") + logger.error(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Error searching for file with pattern '{pattern}' in '{base_path}': {e}") return None - def _update_file_rule_status(self, context: AssetProcessingContext, map_id_hex: str, status: str, **kwargs): # file_rule_id_hex -> map_id_hex - """Helper to update processed_maps_details for a map.""" + def _update_file_rule_status(self, context: AssetProcessingContext, map_key_index: int, status: str, **kwargs): # Renamed map_id_hex to map_key_index + """Helper to update processed_maps_details for a map, keyed by file_rule_idx.""" asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset" - if map_id_hex not in context.processed_maps_details: - context.processed_maps_details[map_id_hex] = {} + if map_key_index not in context.processed_maps_details: + context.processed_maps_details[map_key_index] = {} - context.processed_maps_details[map_id_hex]['status'] = status + context.processed_maps_details[map_key_index]['status'] = status for key, value in kwargs.items(): - context.processed_maps_details[map_id_hex][key] = value + # Ensure source_file_rule_id_hex is not added if it was somehow passed (it shouldn't be) + if key == 'source_file_rule_id_hex': + continue + context.processed_maps_details[map_key_index][key] = value - if 'map_type' not in context.processed_maps_details[map_id_hex] and 'map_type' in kwargs: - context.processed_maps_details[map_id_hex]['map_type'] = kwargs['map_type'] + if 'map_type' not in context.processed_maps_details[map_key_index] and 'map_type' in kwargs: + context.processed_maps_details[map_key_index]['map_type'] = kwargs['map_type'] # Add formatted resolution names if 'original_dimensions' in kwargs and isinstance(kwargs['original_dimensions'], tuple) and len(kwargs['original_dimensions']) == 2: orig_w, orig_h = kwargs['original_dimensions'] - context.processed_maps_details[map_id_hex]['original_resolution_name'] = f"{orig_w}x{orig_h}" + context.processed_maps_details[map_key_index]['original_resolution_name'] = f"{orig_w}x{orig_h}" # Determine the correct dimensions to use for 'processed_resolution_name' # This name refers to the base POT scaled image dimensions before variant generation. @@ -619,21 +680,21 @@ class IndividualMapProcessingStage(ProcessingStage): if dims_to_log_as_base_processed: proc_w, proc_h = dims_to_log_as_base_processed resolution_name_str = f"{proc_w}x{proc_h}" - context.processed_maps_details[map_id_hex]['base_pot_resolution_name'] = resolution_name_str + context.processed_maps_details[map_key_index]['base_pot_resolution_name'] = resolution_name_str # Ensure 'processed_resolution_name' is also set for OutputOrganizationStage compatibility - context.processed_maps_details[map_id_hex]['processed_resolution_name'] = resolution_name_str + context.processed_maps_details[map_key_index]['processed_resolution_name'] = resolution_name_str elif 'processed_dimensions' in kwargs or 'base_pot_dimensions' in kwargs: details_for_warning = kwargs.get('processed_dimensions', kwargs.get('base_pot_dimensions')) - logger.warning(f"Asset '{asset_name_for_log}', Map ID {map_id_hex}: 'processed_dimensions' or 'base_pot_dimensions' key present but its value is not a valid 2-element tuple: {details_for_warning}") + logger.warning(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: 'processed_dimensions' or 'base_pot_dimensions' key present but its value is not a valid 2-element tuple: {details_for_warning}") # If temp_processed_file was passed, ensure it's in the details if 'temp_processed_file' in kwargs: - context.processed_maps_details[map_id_hex]['temp_processed_file'] = kwargs['temp_processed_file'] + context.processed_maps_details[map_key_index]['temp_processed_file'] = kwargs['temp_processed_file'] # Log all details being stored for clarity, including the newly added resolution names - log_details = context.processed_maps_details[map_id_hex].copy() + log_details = context.processed_maps_details[map_key_index].copy() # Avoid logging full image data if it accidentally gets into kwargs if 'image_data' in log_details: del log_details['image_data'] if 'base_pot_image_data' in log_details: del log_details['base_pot_image_data'] - logger.debug(f"Asset '{asset_name_for_log}', Map ID {map_id_hex}: Status updated to '{status}'. Details: {log_details}") \ No newline at end of file + logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: Status updated to '{status}'. Details: {log_details}") \ No newline at end of file diff --git a/processing/pipeline/stages/map_merging.py b/processing/pipeline/stages/map_merging.py index 791f7b9..5dacc73 100644 --- a/processing/pipeline/stages/map_merging.py +++ b/processing/pipeline/stages/map_merging.py @@ -125,72 +125,120 @@ class MapMergingStage(ProcessingStage): required_input_map_types = set(inputs_map_type_to_channel.values()) for required_map_type in required_input_map_types: - found_processed_map = None - processed_map_key = None - for p_key, p_details in context.processed_maps_details.items(): - processed_map_type_in_details = p_details.get('map_type') - # Check for direct match or match with "MAP_" prefix - if (processed_map_type_in_details == required_map_type or \ - processed_map_type_in_details == f"MAP_{required_map_type}") and \ - p_details.get('status') == 'Processed': - found_processed_map = p_details - processed_map_key = p_key # The UUID hex key from individual processing + found_processed_map_details = None + # The key `p_key_idx` is the file_rule_idx from the IndividualMapProcessingStage + for p_key_idx, p_details in context.processed_maps_details.items(): # p_key_idx is an int + processed_map_identifier = p_details.get('processing_map_type', p_details.get('map_type')) + + # Comprehensive list of valid statuses for an input map to be used in merging + valid_input_statuses = ['BasePOTSaved', 'Processed_With_Variants', 'Processed_No_Variants', 'Converted_To_Rough'] + + is_match = False + if processed_map_identifier == required_map_type: + is_match = True + elif required_map_type.startswith("MAP_") and processed_map_identifier == required_map_type.split("MAP_")[-1]: + is_match = True + elif not required_map_type.startswith("MAP_") and processed_map_identifier == f"MAP_{required_map_type}": + is_match = True + + if is_match and p_details.get('status') in valid_input_statuses: + found_processed_map_details = p_details + # The key `p_key_idx` (which is the FileRule index) is implicitly associated with these details. break - if not found_processed_map: - logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Required input map_type '{required_map_type}' for output '{output_map_type}' not found or not processed in processed_maps_details.") - # Option: Use default value for the entire map if one could be constructed for this map_type - # For now, we fail the merge if a required map is missing. - all_inputs_valid = False - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Required input map_type '{required_map_type}' missing."} - break # Break from finding inputs for this merge rule + if not found_processed_map_details: + can_be_fully_defaulted = True + channels_requiring_this_map = [ + ch_key for ch_key, map_type_val in inputs_map_type_to_channel.items() + if map_type_val == required_map_type + ] - temp_file_path = Path(found_processed_map['temp_processed_file']) - if not temp_file_path.exists(): - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Temp file {temp_file_path} for input map_type '{required_map_type}' does not exist.") - all_inputs_valid = False - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Temp file for input '{required_map_type}' missing."} - break - - try: - image_data = ipu.load_image(temp_file_path) - if image_data is None: raise ValueError("Loaded image is None") - except Exception as e: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error loading image {temp_file_path} for input map_type '{required_map_type}': {e}") - all_inputs_valid = False - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Error loading input '{required_map_type}'."} - break - - loaded_input_maps[required_map_type] = image_data - input_map_paths[required_map_type] = str(temp_file_path) - - current_dims = (image_data.shape[1], image_data.shape[0]) - if target_dims is None: - target_dims = current_dims - elif current_dims != target_dims: - logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{required_map_type}' dims {current_dims} differ from target {target_dims}. Resizing.") - try: - image_data = ipu.resize_image(image_data, target_dims[0], target_dims[1]) - if image_data is None: raise ValueError("Resize returned None") - loaded_input_maps[required_map_type] = image_data - except Exception as e: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to resize '{required_map_type}': {e}") + if not channels_requiring_this_map: + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Internal logic error. Required map_type '{required_map_type}' is not actually used by any output channel. Configuration: {inputs_map_type_to_channel}") all_inputs_valid = False - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Failed to resize input '{required_map_type}'."} + context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Internal error: required map_type '{required_map_type}' not in use."} break + + for channel_char_needing_default in channels_requiring_this_map: + if default_values.get(channel_char_needing_default) is None: + can_be_fully_defaulted = False + break + + if can_be_fully_defaulted: + logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Required input map_type '{required_map_type}' for output '{output_map_type}' not found or not in usable state. Will attempt to use default values for its channels: {channels_requiring_this_map}.") + else: + logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Required input map_type '{required_map_type}' for output '{output_map_type}' not found/unusable, AND not all its required channels ({channels_requiring_this_map}) have defaults. Failing merge op.") + all_inputs_valid = False + context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Input '{required_map_type}' missing and defaults incomplete."} + break + + if found_processed_map_details: + temp_file_path_str = found_processed_map_details.get('temp_processed_file') + if not temp_file_path_str: + # Log with p_key_idx if available, or just the map type if not (though it should be if found_processed_map_details is set) + log_key_info = f"(Associated Key Index: {p_key_idx})" if 'p_key_idx' in locals() and found_processed_map_details else "" # Use locals() to check if p_key_idx is defined in this scope + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: 'temp_processed_file' missing in details for found map_type '{required_map_type}' {log_key_info}.") + all_inputs_valid = False + context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Temp file path missing for input '{required_map_type}'."} + break + + temp_file_path = Path(temp_file_path_str) + if not temp_file_path.exists(): + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Temp file {temp_file_path} for input map_type '{required_map_type}' does not exist.") + all_inputs_valid = False + context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Temp file for input '{required_map_type}' missing."} + break + + try: + image_data = ipu.load_image(str(temp_file_path)) + if image_data is None: raise ValueError("Loaded image is None") + except Exception as e: + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error loading image {temp_file_path} for input map_type '{required_map_type}': {e}") + all_inputs_valid = False + context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Error loading input '{required_map_type}'."} + break + + loaded_input_maps[required_map_type] = image_data + input_map_paths[required_map_type] = str(temp_file_path) + + current_dims = (image_data.shape[1], image_data.shape[0]) + if target_dims is None: + target_dims = current_dims + elif current_dims != target_dims: + logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{required_map_type}' dims {current_dims} differ from target {target_dims}. Resizing.") + try: + image_data_resized = ipu.resize_image(image_data, target_dims[0], target_dims[1]) + if image_data_resized is None: raise ValueError("Resize returned None") + loaded_input_maps[required_map_type] = image_data_resized + except Exception as e: + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to resize '{required_map_type}': {e}") + all_inputs_valid = False + context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Failed to resize input '{required_map_type}'."} + break if not all_inputs_valid: logger.warning(f"Asset {asset_name_for_log}: Skipping merge for Op ID {merge_op_id} ('{output_map_type}') due to invalid inputs.") continue - if not loaded_input_maps or target_dims is None: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: No input maps loaded or target_dims not set for '{output_map_type}'. This shouldn't happen if all_inputs_valid was true.") - context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'Internal error: input maps not loaded or target_dims missing.'} - continue + if not loaded_input_maps and not any(default_values.get(ch) is not None for ch in inputs_map_type_to_channel.keys()): + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: No input maps loaded and no defaults available for any channel for '{output_map_type}'. Cannot proceed.") + context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'No input maps loaded and no defaults available.'} + continue - # Determine output channels (e.g., 3 for RGB, 1 for Grayscale) - # This depends on the keys in inputs_map_type_to_channel (R,G,B,A) - output_channel_keys = sorted(list(inputs_map_type_to_channel.keys())) # e.g. ['B', 'G', 'R'] + if target_dims is None: + default_res_key = context.config_obj.get("default_output_resolution_key_for_merge", "1K") + image_resolutions_cfg = getattr(context.config_obj, "image_resolutions", {}) + default_max_dim = image_resolutions_cfg.get(default_res_key) + + if default_max_dim: + target_dims = (default_max_dim, default_max_dim) + logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Target dimensions not set by inputs (all defaulted). Using configured default resolution '{default_res_key}': {target_dims}.") + else: + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Target dimensions could not be determined for '{output_map_type}' (all inputs defaulted and no default output resolution configured).") + context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'Target dimensions undetermined for fully defaulted merge.'} + continue + + output_channel_keys = sorted(list(inputs_map_type_to_channel.keys())) num_output_channels = len(output_channel_keys) if num_output_channels == 0: @@ -199,79 +247,86 @@ class MapMergingStage(ProcessingStage): continue try: - if num_output_channels == 1: # Grayscale output - merged_image = np.zeros((target_dims[1], target_dims[0]), dtype=np.uint8) - else: # Color output - merged_image = np.zeros((target_dims[1], target_dims[0], num_output_channels), dtype=np.uint8) + output_dtype = np.uint8 + + if num_output_channels == 1: + merged_image = np.zeros((target_dims[1], target_dims[0]), dtype=output_dtype) + else: + merged_image = np.zeros((target_dims[1], target_dims[0], num_output_channels), dtype=output_dtype) except Exception as e: logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error creating empty merged image for '{output_map_type}': {e}") context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f'Error creating output canvas: {e}'} continue merge_op_failed_detail = False - for i, out_channel_char in enumerate(output_channel_keys): # e.g. R, G, B + for i, out_channel_char in enumerate(output_channel_keys): input_map_type_for_this_channel = inputs_map_type_to_channel[out_channel_char] source_image = loaded_input_maps.get(input_map_type_for_this_channel) source_data_this_channel = None if source_image is not None: - if source_image.ndim == 2: # Grayscale source + if source_image.dtype != np.uint8: + logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{input_map_type_for_this_channel}' has dtype {source_image.dtype}, expected uint8. Attempting conversion.") + source_image = ipu.convert_to_uint8(source_image) + if source_image is None: + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to convert input '{input_map_type_for_this_channel}' to uint8.") + merge_op_failed_detail = True; break + + + if source_image.ndim == 2: source_data_this_channel = source_image - elif source_image.ndim == 3 or source_image.ndim == 4: # Color source (3-channel BGR or 4-channel BGRA), assumed loaded by ipu.load_image - # Standard BGR(A) channel indexing: B=0, G=1, R=2, A=3 (if present) - # This map helps get NRM's Red data for 'R' output, NRM's Green for 'G' output etc. - # based on the semantic meaning of out_channel_char. + elif source_image.ndim == 3: semantic_to_bgr_idx = {'R': 2, 'G': 1, 'B': 0, 'A': 3} - if input_map_type_for_this_channel == "NRM": - idx_to_extract = semantic_to_bgr_idx.get(out_channel_char) - - if idx_to_extract is not None and idx_to_extract < source_image.shape[2]: - source_data_this_channel = source_image[:, :, idx_to_extract] - logger.debug(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: For output '{out_channel_char}', using NRM's semantic '{out_channel_char}' channel (BGR(A) index {idx_to_extract}).") - else: - # Fallback if out_channel_char isn't R,G,B,A or NRM doesn't have the channel (e.g. 3-channel NRM and 'A' requested) - logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Could not map output '{out_channel_char}' to a specific BGR(A) channel of NRM (shape {source_image.shape}). Defaulting to NRM's channel 0 (Blue).") - source_data_this_channel = source_image[:, :, 0] + idx_to_extract = semantic_to_bgr_idx.get(out_channel_char.upper()) + + if idx_to_extract is not None and idx_to_extract < source_image.shape[2]: + source_data_this_channel = source_image[:, :, idx_to_extract] + logger.debug(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: For output '{out_channel_char}', using source '{input_map_type_for_this_channel}' semantic '{out_channel_char}' (BGR(A) index {idx_to_extract}).") else: - # For other multi-channel sources (e.g., ROUGH as RGB, or other color maps not "NRM") - # Default to taking the first channel (Blue in BGR). - # This covers "Roughness map's greyscale data" if ROUGH is RGB (by taking one of its channels as a proxy). + logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Could not map output '{out_channel_char}' to a specific BGR(A) channel of '{input_map_type_for_this_channel}' (shape {source_image.shape}). Defaulting to its channel 0 (Blue).") source_data_this_channel = source_image[:, :, 0] - logger.debug(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: For output '{out_channel_char}', source {input_map_type_for_this_channel} (shape {source_image.shape}) is multi-channel but not NRM. Using its channel 0 (Blue).") - else: # Source map was not found, use default + else: + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Source image '{input_map_type_for_this_channel}' has unexpected dimensions: {source_image.ndim} (shape {source_image.shape}).") + merge_op_failed_detail = True; break + + else: default_val_for_channel = default_values.get(out_channel_char) if default_val_for_channel is not None: - # Convert 0-1 float default to 0-255 uint8 - source_data_this_channel = np.full((target_dims[1], target_dims[0]), int(default_val_for_channel * 255), dtype=np.uint8) - logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Using default value {default_val_for_channel} for output channel '{out_channel_char}' as input map '{input_map_type_for_this_channel}' was missing.") + try: + scaled_default_val = int(float(default_val_for_channel) * 255) + source_data_this_channel = np.full((target_dims[1], target_dims[0]), scaled_default_val, dtype=np.uint8) + logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Using default value {default_val_for_channel} (scaled to {scaled_default_val}) for output channel '{out_channel_char}' as input map '{input_map_type_for_this_channel}' was missing.") + except ValueError: + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Default value '{default_val_for_channel}' for channel '{out_channel_char}' is not a valid float. Cannot scale.") + merge_op_failed_detail = True; break else: logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{input_map_type_for_this_channel}' for output channel '{out_channel_char}' is missing and no default value provided.") merge_op_failed_detail = True; break - if source_data_this_channel is None: # Should be caught by default value logic or earlier checks + if source_data_this_channel is None: logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to get source data for output channel '{out_channel_char}'.") merge_op_failed_detail = True; break try: - if merged_image.ndim == 2: # Single channel output + if merged_image.ndim == 2: merged_image = source_data_this_channel - else: # Multi-channel output + else: merged_image[:, :, i] = source_data_this_channel except Exception as e: - logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error assigning data to output channel '{out_channel_char}' (index {i}): {e}") + logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error assigning data to output channel '{out_channel_char}' (index {i}): {e}. Merged shape: {merged_image.shape}, Source data shape: {source_data_this_channel.shape}") merge_op_failed_detail = True; break if merge_op_failed_detail: context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'Error during channel assignment.'} continue - output_format = 'png' # Default, can be configured per rule later + output_format = 'png' temp_merged_filename = f"merged_{sanitize_filename(output_map_type)}_{merge_op_id}.{output_format}" temp_merged_path = context.engine_temp_dir / temp_merged_filename try: - save_success = ipu.save_image(temp_merged_path, merged_image) + save_success = ipu.save_image(str(temp_merged_path), merged_image) if not save_success: raise ValueError("Save image returned false") except Exception as e: logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error saving merged image {temp_merged_path}: {e}") diff --git a/processing/pipeline/stages/metadata_finalization_save.py b/processing/pipeline/stages/metadata_finalization_save.py index 8f3a555..f2adb70 100644 --- a/processing/pipeline/stages/metadata_finalization_save.py +++ b/processing/pipeline/stages/metadata_finalization_save.py @@ -66,7 +66,56 @@ class MetadataFinalizationAndSaveStage(ProcessingStage): context.asset_metadata['status'] = "Processed" # Add details of processed and merged maps - context.asset_metadata['processed_map_details'] = getattr(context, 'processed_maps_details', {}) + # Restructure processed_map_details before assigning + restructured_processed_maps = {} + # getattr(context, 'processed_maps_details', {}) is the source (plural 'maps') + original_processed_maps = getattr(context, 'processed_maps_details', {}) + + # Define keys to remove at the top level of each map entry + map_keys_to_remove = [ + "status", "source_file_path", "temp_processed_file", # Assuming "source_file_path" is the correct key + "original_resolution_name", "base_pot_resolution_name", "processed_resolution_name" + ] + # Define keys to remove from each variant + variant_keys_to_remove = ["temp_path", "dimensions"] + + for map_key, map_detail_original in original_processed_maps.items(): + # Create a new dictionary for the modified map entry + new_map_entry = {} + for key, value in map_detail_original.items(): + if key not in map_keys_to_remove: + new_map_entry[key] = value + + if "variants" in map_detail_original and isinstance(map_detail_original["variants"], dict): + new_variants_dict = {} + for variant_name, variant_data_original in map_detail_original["variants"].items(): + new_variant_entry = {} + for key, value in variant_data_original.items(): + if key not in variant_keys_to_remove: + new_variant_entry[key] = value + + # Add 'path_to_file' + # This path is expected to be set by OutputOrganizationStage in the context. + # It should be a Path object representing the path relative to the metadata directory, + # or an absolute Path that make_serializable can convert. + # Using 'final_output_path_for_metadata' as the key from context. + if 'final_output_path_for_metadata' in variant_data_original: + new_variant_entry['path_to_file'] = variant_data_original['final_output_path_for_metadata'] + else: + # Log a warning if the expected path is not found + logger.warning( + f"Asset '{asset_name_for_log}': 'final_output_path_for_metadata' " + f"missing for variant '{variant_name}' in map '{map_key}'. " + f"Metadata will be incomplete for this variant's path." + ) + new_variant_entry['path_to_file'] = "ERROR_PATH_NOT_FOUND" # Placeholder + new_variants_dict[variant_name] = new_variant_entry + new_map_entry["variants"] = new_variants_dict + + restructured_processed_maps[map_key] = new_map_entry + + # Assign the restructured details. Note: 'processed_map_details' (singular 'map') is the key in asset_metadata. + context.asset_metadata['processed_map_details'] = restructured_processed_maps context.asset_metadata['merged_map_details'] = getattr(context, 'merged_maps_details', {}) # (Optional) Add a list of all temporary files diff --git a/processing/pipeline/stages/output_organization.py b/processing/pipeline/stages/output_organization.py index 5087bd6..69fe625 100644 --- a/processing/pipeline/stages/output_organization.py +++ b/processing/pipeline/stages/output_organization.py @@ -210,11 +210,13 @@ class OutputOrganizationStage(ProcessingStage): logger.info(f"Asset '{asset_name_for_log}': Copied variant {temp_variant_path} to {final_variant_path} for map '{processed_map_key}'.") final_output_files.append(str(final_variant_path)) variant_detail['status'] = 'Organized' - - variant_detail['final_output_path'] = str(final_variant_path) - relative_final_variant_path_str = str(Path(relative_dir_path_str_variant) / Path(output_filename_variant)) - map_metadata_entry['variant_paths'][variant_resolution_key] = relative_final_variant_path_str - processed_any_variant_successfully = True + + variant_detail['final_output_path'] = str(final_variant_path) + # Store the Path object for metadata stage to make it relative later + variant_detail['final_output_path_for_metadata'] = final_variant_path + relative_final_variant_path_str = str(Path(relative_dir_path_str_variant) / Path(output_filename_variant)) + map_metadata_entry['variant_paths'][variant_resolution_key] = relative_final_variant_path_str + processed_any_variant_successfully = True except Exception as e: logger.error(f"Asset '{asset_name_for_log}': Failed to copy variant {temp_variant_path} for map key '{processed_map_key}' (res: {variant_resolution_key}). Error: {e}", exc_info=True)