diff --git a/configuration.py b/configuration.py index cbaec74..dec107d 100644 --- a/configuration.py +++ b/configuration.py @@ -379,6 +379,22 @@ class Configuration: """Gets the configured JPG quality level.""" return self._core_settings.get('JPG_QUALITY', 95) + @property + def invert_normal_green_globally(self) -> bool: + """Gets the global setting for inverting the green channel of normal maps.""" + # Default to False if the setting is missing in the core config + return self._core_settings.get('invert_normal_map_green_channel_globally', False) + + @property + def overwrite_existing(self) -> bool: + """Gets the setting for overwriting existing files from core settings.""" + return self._core_settings.get('overwrite_existing', False) + + @property + def png_compression_level(self) -> int: + """Gets the PNG compression level from core settings.""" + return self._core_settings.get('PNG_COMPRESSION', 6) # Default to 6 if not found + @property def resolution_threshold_for_jpg(self) -> int: """Gets the pixel dimension threshold for using JPG for 8-bit images.""" diff --git a/processing/pipeline/stages/individual_map_processing.py b/processing/pipeline/stages/individual_map_processing.py index a937834..6386a33 100644 --- a/processing/pipeline/stages/individual_map_processing.py +++ b/processing/pipeline/stages/individual_map_processing.py @@ -86,10 +86,27 @@ class IndividualMapProcessingStage(ProcessingStage): # Gloss-to-Rough if processing_map_type.startswith("MAP_GLOSS"): logger.info(f"{log_prefix}: Applying Gloss-to-Rough conversion.") - current_image_data = ipu.invert_image_colors(current_image_data) - updated_processing_map_type = processing_map_type.replace("GLOSS", "ROUGH") - logger.info(f"{log_prefix}: Map type updated: '{processing_map_type}' -> '{updated_processing_map_type}'") - transformation_notes.append("Gloss-to-Rough applied") + inversion_succeeded = False + # Replicate inversion logic from GlossToRoughConversionStage + if np.issubdtype(current_image_data.dtype, np.floating): + current_image_data = 1.0 - current_image_data + current_image_data = np.clip(current_image_data, 0.0, 1.0) + logger.debug(f"{log_prefix}: Inverted float image data for Gloss->Rough.") + inversion_succeeded = True + elif np.issubdtype(current_image_data.dtype, np.integer): + max_val = np.iinfo(current_image_data.dtype).max + current_image_data = max_val - current_image_data + logger.debug(f"{log_prefix}: Inverted integer image data (max_val: {max_val}) for Gloss->Rough.") + inversion_succeeded = True + else: + logger.error(f"{log_prefix}: Unsupported image data type {current_image_data.dtype} for GLOSS map. Cannot invert.") + transformation_notes.append("Gloss-to-Rough FAILED (unsupported dtype)") + + # Update type and notes based on success flag + if inversion_succeeded: + updated_processing_map_type = processing_map_type.replace("GLOSS", "ROUGH") + logger.info(f"{log_prefix}: Map type updated: '{processing_map_type}' -> '{updated_processing_map_type}'") + transformation_notes.append("Gloss-to-Rough applied") # Normal Green Invert # Use internal 'MAP_NRM' type for check @@ -119,10 +136,13 @@ class IndividualMapProcessingStage(ProcessingStage): respect_variant_map_types = getattr(config, "respect_variant_map_types", []) # Needed for suffixing logic initial_scaling_mode = getattr(config, "INITIAL_SCALING_MODE", "NONE") merge_dimension_mismatch_strategy = getattr(config, "MERGE_DIMENSION_MISMATCH_STRATEGY", "USE_LARGEST") - invert_normal_green = getattr(config.general_settings, "invert_normal_map_green_channel_globally", False) - output_base_dir = context.output_dir # Assuming output_dir is set in context + invert_normal_green = config.invert_normal_green_globally # Use the new property + output_base_dir = context.output_base_path # This is the FINAL base path asset_name = context.asset_rule.asset_name if context.asset_rule else "UnknownAsset" - output_filename_pattern_tokens = {'asset_name': asset_name, 'output_base_directory': str(output_base_dir)} + # For save_image_variants, the 'output_base_directory' should be the engine_temp_dir, + # as these are intermediate variant files before final organization. + temp_output_base_dir_for_variants = context.engine_temp_dir + output_filename_pattern_tokens = {'asset_name': asset_name, 'output_base_directory': temp_output_base_dir_for_variants} # --- Prepare Items to Process --- items_to_process: List[Union[Tuple[int, FileRule], Tuple[str, Dict]]] = [] @@ -541,12 +561,29 @@ class IndividualMapProcessingStage(ProcessingStage): "base_map_type": base_map_type, # Filename-friendly "source_bit_depth_info": source_bit_depth_info_for_save_util, "output_filename_pattern_tokens": output_filename_pattern_tokens, - "config_obj": config, # Pass the whole config object - "asset_name_for_log": asset_name_for_log, # Pass asset name for logging within save util - "processing_instance_tag": processing_instance_tag # Pass tag for logging within save util + # "config_obj": config, # Removed: save_image_variants doesn't expect this directly + # "asset_name_for_log": asset_name_for_log, # Removed: save_image_variants doesn't expect this + # "processing_instance_tag": processing_instance_tag # Removed: save_image_variants doesn't expect this } - saved_files_details_list = save_image_variants(**save_args) + # Pass only the expected arguments to save_image_variants + # We need to extract the required args from config and pass them individually + save_args_filtered = { + "source_image_data": image_to_save, + "base_map_type": base_map_type, + "source_bit_depth_info": source_bit_depth_info_for_save_util, + "image_resolutions": config.image_resolutions, + "file_type_defs": config.FILE_TYPE_DEFINITIONS, + "output_format_8bit": config.get_8bit_output_format(), + "output_format_16bit_primary": config.get_16bit_output_formats()[0], + "output_format_16bit_fallback": config.get_16bit_output_formats()[1], + "png_compression_level": config.png_compression_level, + "jpg_quality": config.jpg_quality, + "output_filename_pattern_tokens": output_filename_pattern_tokens, + "output_filename_pattern": config.output_filename_pattern, + } + + saved_files_details_list = save_image_variants(**save_args_filtered) if saved_files_details_list: logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Unified Save Utility completed successfully. Saved {len(saved_files_details_list)} variants.") diff --git a/processing/pipeline/stages/map_merging.py b/processing/pipeline/stages/map_merging.py index 696b05e..1c8b180 100644 --- a/processing/pipeline/stages/map_merging.py +++ b/processing/pipeline/stages/map_merging.py @@ -47,15 +47,6 @@ class MapMergingStage(ProcessingStage): # The core merge rules are in context.config_obj.map_merge_rules # Each rule in there defines an output_map_type and its inputs. - logger.error(f"Asset {asset_name_for_log}, Potential Merge for {current_map_type}: Merge rule processing needs rework. FileRule lacks 'merge_settings' and 'id'. Skipping this rule.") - context.merged_maps_details[merge_rule_id_hex] = { - 'map_type': current_map_type, - 'status': 'Failed', - 'reason': 'Merge rule processing logic in MapMergingStage needs refactor due to FileRule changes.' - } - continue - - # For now, let's assume no merge rules are processed until the logic is fixed. num_merge_rules_attempted = 0 # If context.config_obj.map_merge_rules exists, iterate it here. diff --git a/processing/pipeline/stages/output_organization.py b/processing/pipeline/stages/output_organization.py index 69fe625..c612251 100644 --- a/processing/pipeline/stages/output_organization.py +++ b/processing/pipeline/stages/output_organization.py @@ -34,15 +34,7 @@ class OutputOrganizationStage(ProcessingStage): return context final_output_files: List[str] = [] - overwrite_existing = False - # Correctly access general_settings and overwrite_existing from config_obj - if hasattr(context.config_obj, 'general_settings'): - if isinstance(context.config_obj.general_settings, dict): - overwrite_existing = context.config_obj.general_settings.get('overwrite_existing', False) - elif hasattr(context.config_obj.general_settings, 'overwrite_existing'): # If general_settings is an object - overwrite_existing = getattr(context.config_obj.general_settings, 'overwrite_existing', False) - else: - logger.warning(f"Asset '{asset_name_for_log}': config_obj.general_settings not found, defaulting overwrite_existing to False.") + overwrite_existing = context.config_obj.overwrite_existing output_dir_pattern = getattr(context.config_obj, 'output_directory_pattern', "[supplier]/[assetname]") output_filename_pattern_config = getattr(context.config_obj, 'output_filename_pattern', "[assetname]_[maptype]_[resolution].[ext]") @@ -53,15 +45,104 @@ class OutputOrganizationStage(ProcessingStage): logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(context.processed_maps_details)} processed individual map entries.") for processed_map_key, details in context.processed_maps_details.items(): map_status = details.get('status') - base_map_type = details.get('map_type', 'unknown_map_type') # Original map type + base_map_type = details.get('map_type', 'unknown_map_type') # Final filename-friendly type - if map_status in ['Processed', 'Processed_No_Variants']: - if not details.get('temp_processed_file'): - logger.debug(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (status '{map_status}') due to missing 'temp_processed_file'.") + # --- Handle maps processed by the Unified Save Utility --- + if map_status == 'Processed_Via_Save_Utility': + saved_files_info = details.get('saved_files_info') + if not saved_files_info or not isinstance(saved_files_info, list): + logger.warning(f"Asset '{asset_name_for_log}': Map key '{processed_map_key}' (status '{map_status}') has missing or invalid 'saved_files_info'. Skipping organization.") + details['status'] = 'Organization Failed (Missing saved_files_info)' + continue + + logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(saved_files_info)} variants for map key '{processed_map_key}' (map type: {base_map_type}) from Save Utility.") + + map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {}) + map_metadata_entry['map_type'] = base_map_type + map_metadata_entry.setdefault('variant_paths', {}) # Initialize if not present + + processed_any_variant_successfully = False + failed_any_variant = False + + for variant_index, variant_detail in enumerate(saved_files_info): + # Extract info from the save utility's output structure + temp_variant_path_str = variant_detail.get('path') # Key is 'path' + if not temp_variant_path_str: + logger.warning(f"Asset '{asset_name_for_log}': Variant {variant_index} for map '{processed_map_key}' is missing 'path' in saved_files_info. Skipping.") + # Optionally update variant_detail status if it's mutable and tracked, otherwise just skip + continue + + temp_variant_path = Path(temp_variant_path_str) + if not temp_variant_path.is_file(): + logger.warning(f"Asset '{asset_name_for_log}': Temporary variant file '{temp_variant_path}' for map '{processed_map_key}' not found. Skipping.") + continue + + variant_resolution_key = variant_detail.get('resolution_key', f"varRes{variant_index}") + variant_ext = variant_detail.get('format', temp_variant_path.suffix.lstrip('.')) # Use 'format' key + + token_data_variant = { + "assetname": asset_name_for_log, + "supplier": context.effective_supplier or "DefaultSupplier", + "maptype": base_map_type, + "resolution": variant_resolution_key, + "ext": variant_ext, + "incrementingvalue": getattr(context, 'incrementing_value', None), + "sha5": getattr(context, 'sha5_value', None) + } + token_data_variant_cleaned = {k: v for k, v in token_data_variant.items() if v is not None} + output_filename_variant = generate_path_from_pattern(output_filename_pattern_config, token_data_variant_cleaned) + + try: + relative_dir_path_str_variant = generate_path_from_pattern( + pattern_string=output_dir_pattern, + token_data=token_data_variant_cleaned + ) + final_variant_path = Path(context.output_base_path) / Path(relative_dir_path_str_variant) / Path(output_filename_variant) + final_variant_path.parent.mkdir(parents=True, exist_ok=True) + + if final_variant_path.exists() and not overwrite_existing: + logger.info(f"Asset '{asset_name_for_log}': Output variant file {final_variant_path} for map '{processed_map_key}' (res: {variant_resolution_key}) exists and overwrite is disabled. Skipping copy.") + # Optionally update variant_detail status if needed + else: + shutil.copy2(temp_variant_path, final_variant_path) + logger.info(f"Asset '{asset_name_for_log}': Copied variant {temp_variant_path} to {final_variant_path} for map '{processed_map_key}'.") + final_output_files.append(str(final_variant_path)) + # Optionally update variant_detail status if needed + + # Store relative path in metadata + relative_final_variant_path_str = str(Path(relative_dir_path_str_variant) / Path(output_filename_variant)) + map_metadata_entry['variant_paths'][variant_resolution_key] = relative_final_variant_path_str + processed_any_variant_successfully = True + + except Exception as e: + logger.error(f"Asset '{asset_name_for_log}': Failed to copy variant {temp_variant_path} for map key '{processed_map_key}' (res: {variant_resolution_key}). Error: {e}", exc_info=True) + context.status_flags['output_organization_error'] = True + context.asset_metadata['status'] = "Failed (Output Organization Error - Variant)" + # Optionally update variant_detail status if needed + failed_any_variant = True + + # Update parent map detail status based on variant outcomes + if failed_any_variant: + details['status'] = 'Organization Failed (Save Utility Variants)' + elif processed_any_variant_successfully: + details['status'] = 'Organized (Save Utility Variants)' + else: # No variants were successfully copied (e.g., all skipped due to existing file or missing temp file) + details['status'] = 'Organization Skipped (No Save Utility Variants Copied/Needed)' + + # --- Handle older/other processing statuses (like single file processing) --- + elif map_status in ['Processed', 'Processed_No_Variants', 'Converted_To_Rough']: # Add other single-file statuses if needed + temp_file_path_str = details.get('temp_processed_file') + if not temp_file_path_str: + logger.warning(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (status '{map_status}') due to missing 'temp_processed_file'.") details['status'] = 'Organization Skipped (Missing Temp File)' continue - - temp_file_path = Path(details['temp_processed_file']) + + temp_file_path = Path(temp_file_path_str) + if not temp_file_path.is_file(): + logger.warning(f"Asset '{asset_name_for_log}': Temporary file '{temp_file_path}' for map '{processed_map_key}' not found. Skipping.") + details['status'] = 'Organization Skipped (Temp File Not Found)' + continue + resolution_str = details.get('processed_resolution_name', details.get('original_resolution_name', 'resX')) token_data = { @@ -74,7 +155,7 @@ class OutputOrganizationStage(ProcessingStage): "sha5": getattr(context, 'sha5_value', None) } token_data_cleaned = {k: v for k, v in token_data.items() if v is not None} - + output_filename = generate_path_from_pattern(output_filename_pattern_config, token_data_cleaned) try: @@ -87,18 +168,21 @@ class OutputOrganizationStage(ProcessingStage): if final_path.exists() and not overwrite_existing: logger.info(f"Asset '{asset_name_for_log}': Output file {final_path} for map '{processed_map_key}' exists and overwrite is disabled. Skipping copy.") + details['status'] = 'Organized (Exists, Skipped Copy)' else: shutil.copy2(temp_file_path, final_path) logger.info(f"Asset '{asset_name_for_log}': Copied {temp_file_path} to {final_path} for map '{processed_map_key}'.") final_output_files.append(str(final_path)) - + details['status'] = 'Organized' + details['final_output_path'] = str(final_path) - details['status'] = 'Organized' # Update asset_metadata for metadata.json map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {}) map_metadata_entry['map_type'] = base_map_type map_metadata_entry['path'] = str(Path(relative_dir_path_str) / Path(output_filename)) # Store relative path + if 'variant_paths' in map_metadata_entry: # Clean up variant paths if present from previous runs + del map_metadata_entry['variant_paths'] except Exception as e: logger.error(f"Asset '{asset_name_for_log}': Failed to copy {temp_file_path} for map key '{processed_map_key}'. Error: {e}", exc_info=True) @@ -106,79 +190,36 @@ class OutputOrganizationStage(ProcessingStage): context.asset_metadata['status'] = "Failed (Output Organization Error)" details['status'] = 'Organization Failed' + # --- Handle legacy 'Processed_With_Variants' status (if still needed, otherwise remove) --- + # This block is kept for potential backward compatibility but might be redundant + # if 'Processed_Via_Save_Utility' is the new standard for variants. elif map_status == 'Processed_With_Variants': - variants = details.get('variants') - if not variants: # No variants list, or it's empty - logger.warning(f"Asset '{asset_name_for_log}': Map key '{processed_map_key}' (status '{map_status}') has no 'variants' list or it is empty. Attempting fallback to base file.") - if not details.get('temp_processed_file'): - logger.error(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (fallback) as 'temp_processed_file' is also missing.") - details['status'] = 'Organization Failed (No Variants, No Temp File)' - continue # Skip to next map key + variants = details.get('variants') # Expects old structure: list of dicts with 'temp_path' + if not variants: + logger.warning(f"Asset '{asset_name_for_log}': Map key '{processed_map_key}' (status '{map_status}') has no 'variants' list. Skipping.") + details['status'] = 'Organization Failed (Legacy Variants Missing)' + continue - # Fallback: Process the base temp_processed_file - temp_file_path = Path(details['temp_processed_file']) - resolution_str = details.get('processed_resolution_name', details.get('original_resolution_name', 'baseRes')) + logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(variants)} legacy variants for map key '{processed_map_key}' (map type: {base_map_type}).") - token_data = { - "assetname": asset_name_for_log, - "supplier": context.effective_supplier or "DefaultSupplier", - "maptype": base_map_type, - "resolution": resolution_str, - "ext": temp_file_path.suffix.lstrip('.'), - "incrementingvalue": getattr(context, 'incrementing_value', None), - "sha5": getattr(context, 'sha5_value', None) - } - token_data_cleaned = {k: v for k, v in token_data.items() if v is not None} - output_filename = generate_path_from_pattern(output_filename_pattern_config, token_data_cleaned) - - try: - relative_dir_path_str = generate_path_from_pattern( - pattern_string=output_dir_pattern, - token_data=token_data_cleaned - ) - final_path = Path(context.output_base_path) / Path(relative_dir_path_str) / Path(output_filename) - final_path.parent.mkdir(parents=True, exist_ok=True) - - if final_path.exists() and not overwrite_existing: - logger.info(f"Asset '{asset_name_for_log}': Output file {final_path} for map '{processed_map_key}' (fallback) exists and overwrite is disabled. Skipping copy.") - else: - shutil.copy2(temp_file_path, final_path) - logger.info(f"Asset '{asset_name_for_log}': Copied {temp_file_path} to {final_path} for map '{processed_map_key}' (fallback).") - final_output_files.append(str(final_path)) - - details['final_output_path'] = str(final_path) - details['status'] = 'Organized (Base File Fallback)' - - map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {}) - map_metadata_entry['map_type'] = base_map_type - map_metadata_entry['path'] = str(Path(relative_dir_path_str) / Path(output_filename)) - if 'variant_paths' in map_metadata_entry: # Clean up if it was somehow set - del map_metadata_entry['variant_paths'] - except Exception as e: - logger.error(f"Asset '{asset_name_for_log}': Failed to copy {temp_file_path} (fallback) for map key '{processed_map_key}'. Error: {e}", exc_info=True) - context.status_flags['output_organization_error'] = True - context.asset_metadata['status'] = "Failed (Output Organization Error - Fallback)" - details['status'] = 'Organization Failed (Fallback)' - continue # Finished with this map key due to fallback - - # If we are here, 'variants' list exists and is not empty. Proceed with variant processing. - logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(variants)} variants for map key '{processed_map_key}' (map type: {base_map_type}).") - map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {}) map_metadata_entry['map_type'] = base_map_type - map_metadata_entry.setdefault('variant_paths', {}) # Initialize if not present + map_metadata_entry.setdefault('variant_paths', {}) processed_any_variant_successfully = False failed_any_variant = False for variant_index, variant_detail in enumerate(variants): - temp_variant_path_str = variant_detail.get('temp_path') + temp_variant_path_str = variant_detail.get('temp_path') # Uses 'temp_path' if not temp_variant_path_str: - logger.warning(f"Asset '{asset_name_for_log}': Variant {variant_index} for map '{processed_map_key}' is missing 'temp_path'. Skipping.") - variant_detail['status'] = 'Organization Skipped (Missing Temp Path)' + logger.warning(f"Asset '{asset_name_for_log}': Legacy Variant {variant_index} for map '{processed_map_key}' is missing 'temp_path'. Skipping.") continue - + temp_variant_path = Path(temp_variant_path_str) + if not temp_variant_path.is_file(): + logger.warning(f"Asset '{asset_name_for_log}': Legacy temporary variant file '{temp_variant_path}' for map '{processed_map_key}' not found. Skipping.") + continue + variant_resolution_key = variant_detail.get('resolution_key', f"varRes{variant_index}") variant_ext = temp_variant_path.suffix.lstrip('.') @@ -193,7 +234,7 @@ class OutputOrganizationStage(ProcessingStage): } token_data_variant_cleaned = {k: v for k, v in token_data_variant.items() if v is not None} output_filename_variant = generate_path_from_pattern(output_filename_pattern_config, token_data_variant_cleaned) - + try: relative_dir_path_str_variant = generate_path_from_pattern( pattern_string=output_dir_pattern, @@ -203,50 +244,32 @@ class OutputOrganizationStage(ProcessingStage): final_variant_path.parent.mkdir(parents=True, exist_ok=True) if final_variant_path.exists() and not overwrite_existing: - logger.info(f"Asset '{asset_name_for_log}': Output variant file {final_variant_path} for map '{processed_map_key}' (res: {variant_resolution_key}) exists and overwrite is disabled. Skipping copy.") - variant_detail['status'] = 'Organized (Exists, Skipped Copy)' + logger.info(f"Asset '{asset_name_for_log}': Output legacy variant file {final_variant_path} exists and overwrite is disabled. Skipping copy.") else: shutil.copy2(temp_variant_path, final_variant_path) - logger.info(f"Asset '{asset_name_for_log}': Copied variant {temp_variant_path} to {final_variant_path} for map '{processed_map_key}'.") + logger.info(f"Asset '{asset_name_for_log}': Copied legacy variant {temp_variant_path} to {final_variant_path}.") final_output_files.append(str(final_variant_path)) - variant_detail['status'] = 'Organized' - - variant_detail['final_output_path'] = str(final_variant_path) - # Store the Path object for metadata stage to make it relative later - variant_detail['final_output_path_for_metadata'] = final_variant_path + relative_final_variant_path_str = str(Path(relative_dir_path_str_variant) / Path(output_filename_variant)) map_metadata_entry['variant_paths'][variant_resolution_key] = relative_final_variant_path_str processed_any_variant_successfully = True except Exception as e: - logger.error(f"Asset '{asset_name_for_log}': Failed to copy variant {temp_variant_path} for map key '{processed_map_key}' (res: {variant_resolution_key}). Error: {e}", exc_info=True) + logger.error(f"Asset '{asset_name_for_log}': Failed to copy legacy variant {temp_variant_path}. Error: {e}", exc_info=True) context.status_flags['output_organization_error'] = True - context.asset_metadata['status'] = "Failed (Output Organization Error - Variant)" - variant_detail['status'] = 'Organization Failed' + context.asset_metadata['status'] = "Failed (Output Organization Error - Legacy Variant)" failed_any_variant = True - - # Update parent map detail status based on variant outcomes + if failed_any_variant: - details['status'] = 'Organization Failed (Variants)' + details['status'] = 'Organization Failed (Legacy Variants)' elif processed_any_variant_successfully: - # Check if all processable variants were organized - all_attempted_organized = True - for v_detail in variants: - if v_detail.get('temp_path') and not v_detail.get('status', '').startswith('Organized'): - all_attempted_organized = False - break - if all_attempted_organized: - details['status'] = 'Organized (All Attempted Variants)' - else: - details['status'] = 'Partially Organized (Variants)' - elif not any(v.get('temp_path') for v in variants): # No variants had temp_paths to begin with - details['status'] = 'Processed_With_Variants (No Valid Variants to Organize)' - else: # Variants list existed, items had temp_paths, but none were successfully organized (e.g., all skipped due to existing file and no overwrite) - details['status'] = 'Organization Skipped (No Variants Copied/Needed)' + details['status'] = 'Organized (Legacy Variants)' + else: + details['status'] = 'Organization Skipped (No Legacy Variants Copied/Needed)' - - else: # Other statuses like 'Skipped', 'Failed', 'Organization Failed' etc. - logger.debug(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (status: '{map_status}') for organization as it's not 'Processed', 'Processed_No_Variants', or 'Processed_With_Variants'.") + # --- Handle other statuses (Skipped, Failed, etc.) --- + else: # Catches statuses not explicitly handled above + logger.debug(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (status: '{map_status}') for organization as it's not a recognized final processed state or variant state.") continue else: logger.debug(f"Asset '{asset_name_for_log}': No processed individual maps to organize.") diff --git a/processing/utils/image_saving_utils.py b/processing/utils/image_saving_utils.py index 7a51d14..66591a8 100644 --- a/processing/utils/image_saving_utils.py +++ b/processing/utils/image_saving_utils.py @@ -75,7 +75,10 @@ def save_image_variants( source_max_dim = max(source_h, source_w) # 1. Use provided configuration inputs (already available as function arguments) - logger.info(f"Saving variants for map type: {base_map_type}") + logger.info(f"SaveImageVariants: Starting for map type: {base_map_type}. Source shape: {source_image_data.shape}, Source bit depths: {source_bit_depth_info}") + logger.debug(f"SaveImageVariants: Resolutions: {image_resolutions}, File Type Defs: {file_type_defs.keys()}, Output Formats: 8bit={output_format_8bit}, 16bit_pri={output_format_16bit_primary}, 16bit_fall={output_format_16bit_fallback}") + logger.debug(f"SaveImageVariants: PNG Comp: {png_compression_level}, JPG Qual: {jpg_quality}") + logger.debug(f"SaveImageVariants: Output Tokens: {output_filename_pattern_tokens}, Output Pattern: {output_filename_pattern}") # 2. Determine Target Bit Depth target_bit_depth = 8 # Default @@ -111,46 +114,54 @@ def save_image_variants( logger.error(f"Unsupported target bit depth: {target_bit_depth}. Defaulting to 8-bit format.") output_ext = output_format_8bit.lstrip('.').lower() - logger.info(f"Target bit depth: {target_bit_depth}, Output format: {output_ext}") + logger.info(f"SaveImageVariants: Determined target bit depth: {target_bit_depth}, Output format: {output_ext} for map type {base_map_type}") # 4. Generate and Save Resolution Variants # Sort resolutions by max dimension descending sorted_resolutions = sorted(image_resolutions.items(), key=lambda item: item[1], reverse=True) for res_key, res_max_dim in sorted_resolutions: - logger.info(f"Processing resolution variant: {res_key} ({res_max_dim} max dim)") + logger.info(f"SaveImageVariants: Processing variant {res_key} ({res_max_dim}px) for {base_map_type}") - # Calculate target dimensions, ensuring no upscaling - if source_max_dim <= res_max_dim: - # If source is smaller or equal, use source dimensions + # --- Prevent Upscaling --- + # Skip this resolution variant if its target dimension is larger than the source image's largest dimension. + if res_max_dim > source_max_dim: + logger.info(f"SaveImageVariants: Skipping variant {res_key} ({res_max_dim}px) for {base_map_type} because target resolution is larger than source ({source_max_dim}px).") + continue # Skip to the next resolution + + # Calculate target dimensions for valid variants (equal or smaller than source) + if source_max_dim == res_max_dim: + # Use source dimensions if target is equal target_w_res, target_h_res = source_w, source_h - if source_max_dim < res_max_dim: - logger.info(f"Source image ({source_w}x{source_h}) is smaller than target resolution {res_key} ({res_max_dim}). Saving at source resolution.") - else: + logger.info(f"SaveImageVariants: Using source resolution ({source_w}x{source_h}) for {res_key} variant of {base_map_type} as target matches source.") + else: # Downscale (source_max_dim > res_max_dim) # Downscale, maintaining aspect ratio aspect_ratio = source_w / source_h - if source_w > source_h: + if source_w >= source_h: # Use >= to handle square images correctly target_w_res = res_max_dim - target_h_res = int(res_max_dim / aspect_ratio) + target_h_res = max(1, int(res_max_dim / aspect_ratio)) # Ensure height is at least 1 else: target_h_res = res_max_dim - target_w_res = int(res_max_dim * aspect_ratio) - logger.info(f"Resizing source image ({source_w}x{source_h}) to {target_w_res}x{target_h_res} for {res_key} variant.") + target_w_res = max(1, int(res_max_dim * aspect_ratio)) # Ensure width is at least 1 + logger.info(f"SaveImageVariants: Calculated downscale for {base_map_type} {res_key}: from ({source_w}x{source_h}) to ({target_w_res}x{target_h_res})") - # Resize source_image_data - # Use INTER_AREA for downscaling, INTER_LINEAR or INTER_CUBIC for upscaling (though we avoid upscaling here) - interpolation_method = cv2.INTER_AREA # Good for downscaling - # If we were allowing upscaling, we might add logic like: - # if target_w_res > source_w or target_h_res > source_h: - # interpolation_method = cv2.INTER_LINEAR # Or INTER_CUBIC - - try: - variant_data = ipu.resize_image(source_image_data, (target_w_res, target_h_res), interpolation=interpolation_method) - logger.debug(f"Resized variant data shape: {variant_data.shape}") - except Exception as e: - logger.error(f"Error resizing image for {res_key} variant: {e}") - continue # Skip this variant if resizing fails + # Resize source_image_data (only if necessary) + if (target_w_res, target_h_res) == (source_w, source_h): + # No resize needed if dimensions match + variant_data = source_image_data.copy() # Copy to avoid modifying original if needed later + logger.debug(f"SaveImageVariants: No resize needed for {base_map_type} {res_key}, using copy of source data.") + else: + # Perform resize only if dimensions differ (i.e., downscaling) + interpolation_method = cv2.INTER_AREA # Good for downscaling + try: + variant_data = ipu.resize_image(source_image_data, target_w_res, target_h_res, interpolation=interpolation_method) + if variant_data is None: # Check if resize failed + raise ValueError("ipu.resize_image returned None") + logger.debug(f"SaveImageVariants: Resized variant data shape for {base_map_type} {res_key}: {variant_data.shape}") + except Exception as e: + logger.error(f"SaveImageVariants: Error resizing image for {base_map_type} {res_key} variant: {e}") + continue # Skip this variant if resizing fails # Filename Construction current_tokens = output_filename_pattern_tokens.copy() @@ -172,14 +183,14 @@ def save_image_variants( continue # Skip this variant output_path = output_base_directory / filename - logger.info(f"Constructed output path: {output_path}") + logger.info(f"SaveImageVariants: Constructed output path for {base_map_type} {res_key}: {output_path}") # Ensure parent directory exists output_path.parent.mkdir(parents=True, exist_ok=True) - logger.debug(f"Ensured directory exists: {output_path.parent}") + logger.debug(f"SaveImageVariants: Ensured directory exists for {base_map_type} {res_key}: {output_path.parent}") except Exception as e: - logger.error(f"Error constructing filepath for {res_key} variant: {e}") + logger.error(f"SaveImageVariants: Error constructing filepath for {base_map_type} {res_key} variant: {e}") continue # Skip this variant if path construction fails @@ -188,37 +199,40 @@ def save_image_variants( if output_ext == 'jpg': save_params_cv2.append(cv2.IMWRITE_JPEG_QUALITY) save_params_cv2.append(jpg_quality) - logger.debug(f"Using JPG quality: {jpg_quality}") + logger.debug(f"SaveImageVariants: Using JPG quality: {jpg_quality} for {base_map_type} {res_key}") elif output_ext == 'png': save_params_cv2.append(cv2.IMWRITE_PNG_COMPRESSION) save_params_cv2.append(png_compression_level) - logger.debug(f"Using PNG compression level: {png_compression_level}") + logger.debug(f"SaveImageVariants: Using PNG compression level: {png_compression_level} for {base_map_type} {res_key}") # Add other format specific parameters if needed (e.g., TIFF compression) - # Bit Depth Conversion (just before saving) - image_data_for_save = variant_data - try: - if target_bit_depth == 8: - image_data_for_save = ipu.convert_to_uint8(variant_data) - logger.debug("Converted variant data to uint8.") - elif target_bit_depth == 16: - # ipu.convert_to_uint16 might handle different input types (float, uint8) - # Assuming variant_data might be float after resizing, convert to uint16 - image_data_for_save = ipu.convert_to_uint16(variant_data) - logger.debug("Converted variant data to uint16.") - # Add other bit depth conversions if needed - except Exception as e: - logger.error(f"Error converting image data to target bit depth {target_bit_depth} for {res_key} variant: {e}") - continue # Skip this variant if conversion fails + # Bit Depth Conversion is handled by ipu.save_image via output_dtype_target + image_data_for_save = variant_data # Use the resized variant data directly + + # Determine the target dtype for ipu.save_image + output_dtype_for_save: Optional[np.dtype] = None + if target_bit_depth == 8: + output_dtype_for_save = np.uint8 + elif target_bit_depth == 16: + output_dtype_for_save = np.uint16 + # Add other target bit depths like float16/float32 if necessary + # elif target_bit_depth == 32: # Assuming float32 for EXR etc. + # output_dtype_for_save = np.float32 # Saving try: # ipu.save_image is expected to handle the actual cv2.imwrite call - success = ipu.save_image(str(output_path), image_data_for_save, params=save_params_cv2) + logger.debug(f"SaveImageVariants: Attempting to save {base_map_type} {res_key} to {output_path} with params {save_params_cv2}, target_dtype: {output_dtype_for_save}") + success = ipu.save_image( + str(output_path), + image_data_for_save, + output_dtype_target=output_dtype_for_save, # Pass the target dtype + params=save_params_cv2 + ) if success: - logger.info(f"Successfully saved {res_key} variant to {output_path}") + logger.info(f"SaveImageVariants: Successfully saved {base_map_type} {res_key} variant to {output_path}") # Collect details for the returned list saved_file_details.append({ 'path': str(output_path), @@ -228,10 +242,10 @@ def save_image_variants( 'dimensions': (target_w_res, target_h_res) }) else: - logger.error(f"Failed to save {res_key} variant to {output_path}") + logger.error(f"SaveImageVariants: Failed to save {base_map_type} {res_key} variant to {output_path} (ipu.save_image returned False)") except Exception as e: - logger.error(f"Error saving image for {res_key} variant to {output_path}: {e}") + logger.error(f"SaveImageVariants: Error during ipu.save_image for {base_map_type} {res_key} variant to {output_path}: {e}", exc_info=True) # Continue to next variant even if one fails