Rusfort/Asset-Frameworker
1
0
Fork 0
Code Issues 24 Pull Requests Actions Packages Projects 3 Releases Wiki Activity

BugFixes

Browse Source
This commit is contained in:
Rusfort 2025-05-12 16:49:57 +02:00
parent 06552216d5
commit ab4db1b8bd
5 changed files with 261 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
configuration.py
View File

@ -379,6 +379,22 @@ class Configuration:
"""Gets the configured JPG quality level.""" """Gets the configured JPG quality level."""
return self._core_settings.get('JPG_QUALITY', 95) return self._core_settings.get('JPG_QUALITY', 95)
@property
def invert_normal_green_globally(self) -> bool:
"""Gets the global setting for inverting the green channel of normal maps."""
# Default to False if the setting is missing in the core config
return self._core_settings.get('invert_normal_map_green_channel_globally', False)
@property
def overwrite_existing(self) -> bool:
"""Gets the setting for overwriting existing files from core settings."""
return self._core_settings.get('overwrite_existing', False)
@property
def png_compression_level(self) -> int:
"""Gets the PNG compression level from core settings."""
return self._core_settings.get('PNG_COMPRESSION', 6) # Default to 6 if not found
@property @property
def resolution_threshold_for_jpg(self) -> int: def resolution_threshold_for_jpg(self) -> int:
"""Gets the pixel dimension threshold for using JPG for 8-bit images.""" """Gets the pixel dimension threshold for using JPG for 8-bit images."""

59
processing/pipeline/stages/individual_map_processing.py
View File

@ -86,10 +86,27 @@ class IndividualMapProcessingStage(ProcessingStage):
# Gloss-to-Rough # Gloss-to-Rough
if processing_map_type.startswith("MAP_GLOSS"): if processing_map_type.startswith("MAP_GLOSS"):
logger.info(f"{log_prefix}: Applying Gloss-to-Rough conversion.") logger.info(f"{log_prefix}: Applying Gloss-to-Rough conversion.")
current_image_data = ipu.invert_image_colors(current_image_data) inversion_succeeded = False
updated_processing_map_type = processing_map_type.replace("GLOSS", "ROUGH") # Replicate inversion logic from GlossToRoughConversionStage
logger.info(f"{log_prefix}: Map type updated: '{processing_map_type}' -> '{updated_processing_map_type}'") if np.issubdtype(current_image_data.dtype, np.floating):
transformation_notes.append("Gloss-to-Rough applied") current_image_data = 1.0 - current_image_data
current_image_data = np.clip(current_image_data, 0.0, 1.0)
logger.debug(f"{log_prefix}: Inverted float image data for Gloss->Rough.")
inversion_succeeded = True
elif np.issubdtype(current_image_data.dtype, np.integer):
max_val = np.iinfo(current_image_data.dtype).max
current_image_data = max_val - current_image_data
logger.debug(f"{log_prefix}: Inverted integer image data (max_val: {max_val}) for Gloss->Rough.")
inversion_succeeded = True
else:
logger.error(f"{log_prefix}: Unsupported image data type {current_image_data.dtype} for GLOSS map. Cannot invert.")
transformation_notes.append("Gloss-to-Rough FAILED (unsupported dtype)")
# Update type and notes based on success flag
if inversion_succeeded:
updated_processing_map_type = processing_map_type.replace("GLOSS", "ROUGH")
logger.info(f"{log_prefix}: Map type updated: '{processing_map_type}' -> '{updated_processing_map_type}'")
transformation_notes.append("Gloss-to-Rough applied")
# Normal Green Invert # Normal Green Invert
# Use internal 'MAP_NRM' type for check # Use internal 'MAP_NRM' type for check
@ -119,10 +136,13 @@ class IndividualMapProcessingStage(ProcessingStage):
respect_variant_map_types = getattr(config, "respect_variant_map_types", []) # Needed for suffixing logic respect_variant_map_types = getattr(config, "respect_variant_map_types", []) # Needed for suffixing logic
initial_scaling_mode = getattr(config, "INITIAL_SCALING_MODE", "NONE") initial_scaling_mode = getattr(config, "INITIAL_SCALING_MODE", "NONE")
merge_dimension_mismatch_strategy = getattr(config, "MERGE_DIMENSION_MISMATCH_STRATEGY", "USE_LARGEST") merge_dimension_mismatch_strategy = getattr(config, "MERGE_DIMENSION_MISMATCH_STRATEGY", "USE_LARGEST")
invert_normal_green = getattr(config.general_settings, "invert_normal_map_green_channel_globally", False) invert_normal_green = config.invert_normal_green_globally # Use the new property
output_base_dir = context.output_dir # Assuming output_dir is set in context output_base_dir = context.output_base_path # This is the FINAL base path
asset_name = context.asset_rule.asset_name if context.asset_rule else "UnknownAsset" asset_name = context.asset_rule.asset_name if context.asset_rule else "UnknownAsset"
output_filename_pattern_tokens = {'asset_name': asset_name, 'output_base_directory': str(output_base_dir)} # For save_image_variants, the 'output_base_directory' should be the engine_temp_dir,
# as these are intermediate variant files before final organization.
temp_output_base_dir_for_variants = context.engine_temp_dir
output_filename_pattern_tokens = {'asset_name': asset_name, 'output_base_directory': temp_output_base_dir_for_variants}
# --- Prepare Items to Process --- # --- Prepare Items to Process ---
items_to_process: List[Union[Tuple[int, FileRule], Tuple[str, Dict]]] = [] items_to_process: List[Union[Tuple[int, FileRule], Tuple[str, Dict]]] = []
@ -541,12 +561,29 @@ class IndividualMapProcessingStage(ProcessingStage):
"base_map_type": base_map_type, # Filename-friendly "base_map_type": base_map_type, # Filename-friendly
"source_bit_depth_info": source_bit_depth_info_for_save_util, "source_bit_depth_info": source_bit_depth_info_for_save_util,
"output_filename_pattern_tokens": output_filename_pattern_tokens, "output_filename_pattern_tokens": output_filename_pattern_tokens,
"config_obj": config, # Pass the whole config object # "config_obj": config, # Removed: save_image_variants doesn't expect this directly
"asset_name_for_log": asset_name_for_log, # Pass asset name for logging within save util # "asset_name_for_log": asset_name_for_log, # Removed: save_image_variants doesn't expect this
"processing_instance_tag": processing_instance_tag # Pass tag for logging within save util # "processing_instance_tag": processing_instance_tag # Removed: save_image_variants doesn't expect this
} }
saved_files_details_list = save_image_variants(**save_args) # Pass only the expected arguments to save_image_variants
# We need to extract the required args from config and pass them individually
save_args_filtered = {
"source_image_data": image_to_save,
"base_map_type": base_map_type,
"source_bit_depth_info": source_bit_depth_info_for_save_util,
"image_resolutions": config.image_resolutions,
"file_type_defs": config.FILE_TYPE_DEFINITIONS,
"output_format_8bit": config.get_8bit_output_format(),
"output_format_16bit_primary": config.get_16bit_output_formats()[0],
"output_format_16bit_fallback": config.get_16bit_output_formats()[1],
"png_compression_level": config.png_compression_level,
"jpg_quality": config.jpg_quality,
"output_filename_pattern_tokens": output_filename_pattern_tokens,
"output_filename_pattern": config.output_filename_pattern,
}
saved_files_details_list = save_image_variants(**save_args_filtered)
if saved_files_details_list: if saved_files_details_list:
logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Unified Save Utility completed successfully. Saved {len(saved_files_details_list)} variants.") logger.info(f"Asset '{asset_name_for_log}', Key {item_key}, Proc. Tag {processing_instance_tag}: Unified Save Utility completed successfully. Saved {len(saved_files_details_list)} variants.")

9
processing/pipeline/stages/map_merging.py
View File

@ -47,15 +47,6 @@ class MapMergingStage(ProcessingStage):
# The core merge rules are in context.config_obj.map_merge_rules # The core merge rules are in context.config_obj.map_merge_rules
# Each rule in there defines an output_map_type and its inputs. # Each rule in there defines an output_map_type and its inputs.
logger.error(f"Asset {asset_name_for_log}, Potential Merge for {current_map_type}: Merge rule processing needs rework. FileRule lacks 'merge_settings' and 'id'. Skipping this rule.")
context.merged_maps_details[merge_rule_id_hex] = {
'map_type': current_map_type,
'status': 'Failed',
'reason': 'Merge rule processing logic in MapMergingStage needs refactor due to FileRule changes.'
}
continue
# For now, let's assume no merge rules are processed until the logic is fixed. # For now, let's assume no merge rules are processed until the logic is fixed.
num_merge_rules_attempted = 0 num_merge_rules_attempted = 0
# If context.config_obj.map_merge_rules exists, iterate it here. # If context.config_obj.map_merge_rules exists, iterate it here.

243
processing/pipeline/stages/output_organization.py
View File

@ -34,15 +34,7 @@ class OutputOrganizationStage(ProcessingStage):
return context return context
final_output_files: List[str] = [] final_output_files: List[str] = []
overwrite_existing = False overwrite_existing = context.config_obj.overwrite_existing
# Correctly access general_settings and overwrite_existing from config_obj
if hasattr(context.config_obj, 'general_settings'):
if isinstance(context.config_obj.general_settings, dict):
overwrite_existing = context.config_obj.general_settings.get('overwrite_existing', False)
elif hasattr(context.config_obj.general_settings, 'overwrite_existing'): # If general_settings is an object
overwrite_existing = getattr(context.config_obj.general_settings, 'overwrite_existing', False)
else:
logger.warning(f"Asset '{asset_name_for_log}': config_obj.general_settings not found, defaulting overwrite_existing to False.")
output_dir_pattern = getattr(context.config_obj, 'output_directory_pattern', "[supplier]/[assetname]") output_dir_pattern = getattr(context.config_obj, 'output_directory_pattern', "[supplier]/[assetname]")
output_filename_pattern_config = getattr(context.config_obj, 'output_filename_pattern', "[assetname]_[maptype]_[resolution].[ext]") output_filename_pattern_config = getattr(context.config_obj, 'output_filename_pattern', "[assetname]_[maptype]_[resolution].[ext]")
@ -53,15 +45,104 @@ class OutputOrganizationStage(ProcessingStage):
logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(context.processed_maps_details)} processed individual map entries.") logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(context.processed_maps_details)} processed individual map entries.")
for processed_map_key, details in context.processed_maps_details.items(): for processed_map_key, details in context.processed_maps_details.items():
map_status = details.get('status') map_status = details.get('status')
base_map_type = details.get('map_type', 'unknown_map_type') # Original map type base_map_type = details.get('map_type', 'unknown_map_type') # Final filename-friendly type
if map_status in ['Processed', 'Processed_No_Variants']: # --- Handle maps processed by the Unified Save Utility ---
if not details.get('temp_processed_file'): if map_status == 'Processed_Via_Save_Utility':
logger.debug(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (status '{map_status}') due to missing 'temp_processed_file'.") saved_files_info = details.get('saved_files_info')
if not saved_files_info or not isinstance(saved_files_info, list):
logger.warning(f"Asset '{asset_name_for_log}': Map key '{processed_map_key}' (status '{map_status}') has missing or invalid 'saved_files_info'. Skipping organization.")
details['status'] = 'Organization Failed (Missing saved_files_info)'
continue
logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(saved_files_info)} variants for map key '{processed_map_key}' (map type: {base_map_type}) from Save Utility.")
map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {})
map_metadata_entry['map_type'] = base_map_type
map_metadata_entry.setdefault('variant_paths', {}) # Initialize if not present
processed_any_variant_successfully = False
failed_any_variant = False
for variant_index, variant_detail in enumerate(saved_files_info):
# Extract info from the save utility's output structure
temp_variant_path_str = variant_detail.get('path') # Key is 'path'
if not temp_variant_path_str:
logger.warning(f"Asset '{asset_name_for_log}': Variant {variant_index} for map '{processed_map_key}' is missing 'path' in saved_files_info. Skipping.")
# Optionally update variant_detail status if it's mutable and tracked, otherwise just skip
continue
temp_variant_path = Path(temp_variant_path_str)
if not temp_variant_path.is_file():
logger.warning(f"Asset '{asset_name_for_log}': Temporary variant file '{temp_variant_path}' for map '{processed_map_key}' not found. Skipping.")
continue
variant_resolution_key = variant_detail.get('resolution_key', f"varRes{variant_index}")
variant_ext = variant_detail.get('format', temp_variant_path.suffix.lstrip('.')) # Use 'format' key
token_data_variant = {
"assetname": asset_name_for_log,
"supplier": context.effective_supplier or "DefaultSupplier",
"maptype": base_map_type,
"resolution": variant_resolution_key,
"ext": variant_ext,
"incrementingvalue": getattr(context, 'incrementing_value', None),
"sha5": getattr(context, 'sha5_value', None)
}
token_data_variant_cleaned = {k: v for k, v in token_data_variant.items() if v is not None}
output_filename_variant = generate_path_from_pattern(output_filename_pattern_config, token_data_variant_cleaned)
try:
relative_dir_path_str_variant = generate_path_from_pattern(
pattern_string=output_dir_pattern,
token_data=token_data_variant_cleaned
)
final_variant_path = Path(context.output_base_path) / Path(relative_dir_path_str_variant) / Path(output_filename_variant)
final_variant_path.parent.mkdir(parents=True, exist_ok=True)
if final_variant_path.exists() and not overwrite_existing:
logger.info(f"Asset '{asset_name_for_log}': Output variant file {final_variant_path} for map '{processed_map_key}' (res: {variant_resolution_key}) exists and overwrite is disabled. Skipping copy.")
# Optionally update variant_detail status if needed
else:
shutil.copy2(temp_variant_path, final_variant_path)
logger.info(f"Asset '{asset_name_for_log}': Copied variant {temp_variant_path} to {final_variant_path} for map '{processed_map_key}'.")
final_output_files.append(str(final_variant_path))
# Optionally update variant_detail status if needed
# Store relative path in metadata
relative_final_variant_path_str = str(Path(relative_dir_path_str_variant) / Path(output_filename_variant))
map_metadata_entry['variant_paths'][variant_resolution_key] = relative_final_variant_path_str
processed_any_variant_successfully = True
except Exception as e:
logger.error(f"Asset '{asset_name_for_log}': Failed to copy variant {temp_variant_path} for map key '{processed_map_key}' (res: {variant_resolution_key}). Error: {e}", exc_info=True)
context.status_flags['output_organization_error'] = True
context.asset_metadata['status'] = "Failed (Output Organization Error - Variant)"
# Optionally update variant_detail status if needed
failed_any_variant = True
# Update parent map detail status based on variant outcomes
if failed_any_variant:
details['status'] = 'Organization Failed (Save Utility Variants)'
elif processed_any_variant_successfully:
details['status'] = 'Organized (Save Utility Variants)'
else: # No variants were successfully copied (e.g., all skipped due to existing file or missing temp file)
details['status'] = 'Organization Skipped (No Save Utility Variants Copied/Needed)'
# --- Handle older/other processing statuses (like single file processing) ---
elif map_status in ['Processed', 'Processed_No_Variants', 'Converted_To_Rough']: # Add other single-file statuses if needed
temp_file_path_str = details.get('temp_processed_file')
if not temp_file_path_str:
logger.warning(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (status '{map_status}') due to missing 'temp_processed_file'.")
details['status'] = 'Organization Skipped (Missing Temp File)' details['status'] = 'Organization Skipped (Missing Temp File)'
continue continue
temp_file_path = Path(details['temp_processed_file']) temp_file_path = Path(temp_file_path_str)
if not temp_file_path.is_file():
logger.warning(f"Asset '{asset_name_for_log}': Temporary file '{temp_file_path}' for map '{processed_map_key}' not found. Skipping.")
details['status'] = 'Organization Skipped (Temp File Not Found)'
continue
resolution_str = details.get('processed_resolution_name', details.get('original_resolution_name', 'resX')) resolution_str = details.get('processed_resolution_name', details.get('original_resolution_name', 'resX'))
token_data = { token_data = {
@ -74,7 +155,7 @@ class OutputOrganizationStage(ProcessingStage):
"sha5": getattr(context, 'sha5_value', None) "sha5": getattr(context, 'sha5_value', None)
} }
token_data_cleaned = {k: v for k, v in token_data.items() if v is not None} token_data_cleaned = {k: v for k, v in token_data.items() if v is not None}
output_filename = generate_path_from_pattern(output_filename_pattern_config, token_data_cleaned) output_filename = generate_path_from_pattern(output_filename_pattern_config, token_data_cleaned)
try: try:
@ -87,18 +168,21 @@ class OutputOrganizationStage(ProcessingStage):
if final_path.exists() and not overwrite_existing: if final_path.exists() and not overwrite_existing:
logger.info(f"Asset '{asset_name_for_log}': Output file {final_path} for map '{processed_map_key}' exists and overwrite is disabled. Skipping copy.") logger.info(f"Asset '{asset_name_for_log}': Output file {final_path} for map '{processed_map_key}' exists and overwrite is disabled. Skipping copy.")
details['status'] = 'Organized (Exists, Skipped Copy)'
else: else:
shutil.copy2(temp_file_path, final_path) shutil.copy2(temp_file_path, final_path)
logger.info(f"Asset '{asset_name_for_log}': Copied {temp_file_path} to {final_path} for map '{processed_map_key}'.") logger.info(f"Asset '{asset_name_for_log}': Copied {temp_file_path} to {final_path} for map '{processed_map_key}'.")
final_output_files.append(str(final_path)) final_output_files.append(str(final_path))
details['status'] = 'Organized'
details['final_output_path'] = str(final_path) details['final_output_path'] = str(final_path)
details['status'] = 'Organized'
# Update asset_metadata for metadata.json # Update asset_metadata for metadata.json
map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {}) map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {})
map_metadata_entry['map_type'] = base_map_type map_metadata_entry['map_type'] = base_map_type
map_metadata_entry['path'] = str(Path(relative_dir_path_str) / Path(output_filename)) # Store relative path map_metadata_entry['path'] = str(Path(relative_dir_path_str) / Path(output_filename)) # Store relative path
if 'variant_paths' in map_metadata_entry: # Clean up variant paths if present from previous runs
del map_metadata_entry['variant_paths']
except Exception as e: except Exception as e:
logger.error(f"Asset '{asset_name_for_log}': Failed to copy {temp_file_path} for map key '{processed_map_key}'. Error: {e}", exc_info=True) logger.error(f"Asset '{asset_name_for_log}': Failed to copy {temp_file_path} for map key '{processed_map_key}'. Error: {e}", exc_info=True)
@ -106,79 +190,36 @@ class OutputOrganizationStage(ProcessingStage):
context.asset_metadata['status'] = "Failed (Output Organization Error)" context.asset_metadata['status'] = "Failed (Output Organization Error)"
details['status'] = 'Organization Failed' details['status'] = 'Organization Failed'
# --- Handle legacy 'Processed_With_Variants' status (if still needed, otherwise remove) ---
# This block is kept for potential backward compatibility but might be redundant
# if 'Processed_Via_Save_Utility' is the new standard for variants.
elif map_status == 'Processed_With_Variants': elif map_status == 'Processed_With_Variants':
variants = details.get('variants') variants = details.get('variants') # Expects old structure: list of dicts with 'temp_path'
if not variants: # No variants list, or it's empty if not variants:
logger.warning(f"Asset '{asset_name_for_log}': Map key '{processed_map_key}' (status '{map_status}') has no 'variants' list or it is empty. Attempting fallback to base file.") logger.warning(f"Asset '{asset_name_for_log}': Map key '{processed_map_key}' (status '{map_status}') has no 'variants' list. Skipping.")
if not details.get('temp_processed_file'): details['status'] = 'Organization Failed (Legacy Variants Missing)'
logger.error(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (fallback) as 'temp_processed_file' is also missing.") continue
details['status'] = 'Organization Failed (No Variants, No Temp File)'
continue # Skip to next map key
# Fallback: Process the base temp_processed_file logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(variants)} legacy variants for map key '{processed_map_key}' (map type: {base_map_type}).")
temp_file_path = Path(details['temp_processed_file'])
resolution_str = details.get('processed_resolution_name', details.get('original_resolution_name', 'baseRes'))
token_data = {
"assetname": asset_name_for_log,
"supplier": context.effective_supplier or "DefaultSupplier",
"maptype": base_map_type,
"resolution": resolution_str,
"ext": temp_file_path.suffix.lstrip('.'),
"incrementingvalue": getattr(context, 'incrementing_value', None),
"sha5": getattr(context, 'sha5_value', None)
}
token_data_cleaned = {k: v for k, v in token_data.items() if v is not None}
output_filename = generate_path_from_pattern(output_filename_pattern_config, token_data_cleaned)
try:
relative_dir_path_str = generate_path_from_pattern(
pattern_string=output_dir_pattern,
token_data=token_data_cleaned
)
final_path = Path(context.output_base_path) / Path(relative_dir_path_str) / Path(output_filename)
final_path.parent.mkdir(parents=True, exist_ok=True)
if final_path.exists() and not overwrite_existing:
logger.info(f"Asset '{asset_name_for_log}': Output file {final_path} for map '{processed_map_key}' (fallback) exists and overwrite is disabled. Skipping copy.")
else:
shutil.copy2(temp_file_path, final_path)
logger.info(f"Asset '{asset_name_for_log}': Copied {temp_file_path} to {final_path} for map '{processed_map_key}' (fallback).")
final_output_files.append(str(final_path))
details['final_output_path'] = str(final_path)
details['status'] = 'Organized (Base File Fallback)'
map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {})
map_metadata_entry['map_type'] = base_map_type
map_metadata_entry['path'] = str(Path(relative_dir_path_str) / Path(output_filename))
if 'variant_paths' in map_metadata_entry: # Clean up if it was somehow set
del map_metadata_entry['variant_paths']
except Exception as e:
logger.error(f"Asset '{asset_name_for_log}': Failed to copy {temp_file_path} (fallback) for map key '{processed_map_key}'. Error: {e}", exc_info=True)
context.status_flags['output_organization_error'] = True
context.asset_metadata['status'] = "Failed (Output Organization Error - Fallback)"
details['status'] = 'Organization Failed (Fallback)'
continue # Finished with this map key due to fallback
# If we are here, 'variants' list exists and is not empty. Proceed with variant processing.
logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(variants)} variants for map key '{processed_map_key}' (map type: {base_map_type}).")
map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {}) map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {})
map_metadata_entry['map_type'] = base_map_type map_metadata_entry['map_type'] = base_map_type
map_metadata_entry.setdefault('variant_paths', {}) # Initialize if not present map_metadata_entry.setdefault('variant_paths', {})
processed_any_variant_successfully = False processed_any_variant_successfully = False
failed_any_variant = False failed_any_variant = False
for variant_index, variant_detail in enumerate(variants): for variant_index, variant_detail in enumerate(variants):
temp_variant_path_str = variant_detail.get('temp_path') temp_variant_path_str = variant_detail.get('temp_path') # Uses 'temp_path'
if not temp_variant_path_str: if not temp_variant_path_str:
logger.warning(f"Asset '{asset_name_for_log}': Variant {variant_index} for map '{processed_map_key}' is missing 'temp_path'. Skipping.") logger.warning(f"Asset '{asset_name_for_log}': Legacy Variant {variant_index} for map '{processed_map_key}' is missing 'temp_path'. Skipping.")
variant_detail['status'] = 'Organization Skipped (Missing Temp Path)'
continue continue
temp_variant_path = Path(temp_variant_path_str) temp_variant_path = Path(temp_variant_path_str)
if not temp_variant_path.is_file():
logger.warning(f"Asset '{asset_name_for_log}': Legacy temporary variant file '{temp_variant_path}' for map '{processed_map_key}' not found. Skipping.")
continue
variant_resolution_key = variant_detail.get('resolution_key', f"varRes{variant_index}") variant_resolution_key = variant_detail.get('resolution_key', f"varRes{variant_index}")
variant_ext = temp_variant_path.suffix.lstrip('.') variant_ext = temp_variant_path.suffix.lstrip('.')
@ -193,7 +234,7 @@ class OutputOrganizationStage(ProcessingStage):
} }
token_data_variant_cleaned = {k: v for k, v in token_data_variant.items() if v is not None} token_data_variant_cleaned = {k: v for k, v in token_data_variant.items() if v is not None}
output_filename_variant = generate_path_from_pattern(output_filename_pattern_config, token_data_variant_cleaned) output_filename_variant = generate_path_from_pattern(output_filename_pattern_config, token_data_variant_cleaned)
try: try:
relative_dir_path_str_variant = generate_path_from_pattern( relative_dir_path_str_variant = generate_path_from_pattern(
pattern_string=output_dir_pattern, pattern_string=output_dir_pattern,
@ -203,50 +244,32 @@ class OutputOrganizationStage(ProcessingStage):
final_variant_path.parent.mkdir(parents=True, exist_ok=True) final_variant_path.parent.mkdir(parents=True, exist_ok=True)
if final_variant_path.exists() and not overwrite_existing: if final_variant_path.exists() and not overwrite_existing:
logger.info(f"Asset '{asset_name_for_log}': Output variant file {final_variant_path} for map '{processed_map_key}' (res: {variant_resolution_key}) exists and overwrite is disabled. Skipping copy.") logger.info(f"Asset '{asset_name_for_log}': Output legacy variant file {final_variant_path} exists and overwrite is disabled. Skipping copy.")
variant_detail['status'] = 'Organized (Exists, Skipped Copy)'
else: else:
shutil.copy2(temp_variant_path, final_variant_path) shutil.copy2(temp_variant_path, final_variant_path)
logger.info(f"Asset '{asset_name_for_log}': Copied variant {temp_variant_path} to {final_variant_path} for map '{processed_map_key}'.") logger.info(f"Asset '{asset_name_for_log}': Copied legacy variant {temp_variant_path} to {final_variant_path}.")
final_output_files.append(str(final_variant_path)) final_output_files.append(str(final_variant_path))
variant_detail['status'] = 'Organized'
variant_detail['final_output_path'] = str(final_variant_path)
# Store the Path object for metadata stage to make it relative later
variant_detail['final_output_path_for_metadata'] = final_variant_path
relative_final_variant_path_str = str(Path(relative_dir_path_str_variant) / Path(output_filename_variant)) relative_final_variant_path_str = str(Path(relative_dir_path_str_variant) / Path(output_filename_variant))
map_metadata_entry['variant_paths'][variant_resolution_key] = relative_final_variant_path_str map_metadata_entry['variant_paths'][variant_resolution_key] = relative_final_variant_path_str
processed_any_variant_successfully = True processed_any_variant_successfully = True
except Exception as e: except Exception as e:
logger.error(f"Asset '{asset_name_for_log}': Failed to copy variant {temp_variant_path} for map key '{processed_map_key}' (res: {variant_resolution_key}). Error: {e}", exc_info=True) logger.error(f"Asset '{asset_name_for_log}': Failed to copy legacy variant {temp_variant_path}. Error: {e}", exc_info=True)
context.status_flags['output_organization_error'] = True context.status_flags['output_organization_error'] = True
context.asset_metadata['status'] = "Failed (Output Organization Error - Variant)" context.asset_metadata['status'] = "Failed (Output Organization Error - Legacy Variant)"
variant_detail['status'] = 'Organization Failed'
failed_any_variant = True failed_any_variant = True
# Update parent map detail status based on variant outcomes
if failed_any_variant: if failed_any_variant:
details['status'] = 'Organization Failed (Variants)' details['status'] = 'Organization Failed (Legacy Variants)'
elif processed_any_variant_successfully: elif processed_any_variant_successfully:
# Check if all processable variants were organized details['status'] = 'Organized (Legacy Variants)'
all_attempted_organized = True else:
for v_detail in variants: details['status'] = 'Organization Skipped (No Legacy Variants Copied/Needed)'
if v_detail.get('temp_path') and not v_detail.get('status', '').startswith('Organized'):
all_attempted_organized = False
break
if all_attempted_organized:
details['status'] = 'Organized (All Attempted Variants)'
else:
details['status'] = 'Partially Organized (Variants)'
elif not any(v.get('temp_path') for v in variants): # No variants had temp_paths to begin with
details['status'] = 'Processed_With_Variants (No Valid Variants to Organize)'
else: # Variants list existed, items had temp_paths, but none were successfully organized (e.g., all skipped due to existing file and no overwrite)
details['status'] = 'Organization Skipped (No Variants Copied/Needed)'
# --- Handle other statuses (Skipped, Failed, etc.) ---
else: # Other statuses like 'Skipped', 'Failed', 'Organization Failed' etc. else: # Catches statuses not explicitly handled above
logger.debug(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (status: '{map_status}') for organization as it's not 'Processed', 'Processed_No_Variants', or 'Processed_With_Variants'.") logger.debug(f"Asset '{asset_name_for_log}': Skipping map key '{processed_map_key}' (status: '{map_status}') for organization as it's not a recognized final processed state or variant state.")
continue continue
else: else:
logger.debug(f"Asset '{asset_name_for_log}': No processed individual maps to organize.") logger.debug(f"Asset '{asset_name_for_log}': No processed individual maps to organize.")

114
processing/utils/image_saving_utils.py
View File

@ -75,7 +75,10 @@ def save_image_variants(
source_max_dim = max(source_h, source_w) source_max_dim = max(source_h, source_w)
# 1. Use provided configuration inputs (already available as function arguments) # 1. Use provided configuration inputs (already available as function arguments)
logger.info(f"Saving variants for map type: {base_map_type}") logger.info(f"SaveImageVariants: Starting for map type: {base_map_type}. Source shape: {source_image_data.shape}, Source bit depths: {source_bit_depth_info}")
logger.debug(f"SaveImageVariants: Resolutions: {image_resolutions}, File Type Defs: {file_type_defs.keys()}, Output Formats: 8bit={output_format_8bit}, 16bit_pri={output_format_16bit_primary}, 16bit_fall={output_format_16bit_fallback}")
logger.debug(f"SaveImageVariants: PNG Comp: {png_compression_level}, JPG Qual: {jpg_quality}")
logger.debug(f"SaveImageVariants: Output Tokens: {output_filename_pattern_tokens}, Output Pattern: {output_filename_pattern}")
# 2. Determine Target Bit Depth # 2. Determine Target Bit Depth
target_bit_depth = 8 # Default target_bit_depth = 8 # Default
@ -111,46 +114,54 @@ def save_image_variants(
logger.error(f"Unsupported target bit depth: {target_bit_depth}. Defaulting to 8-bit format.") logger.error(f"Unsupported target bit depth: {target_bit_depth}. Defaulting to 8-bit format.")
output_ext = output_format_8bit.lstrip('.').lower() output_ext = output_format_8bit.lstrip('.').lower()
logger.info(f"Target bit depth: {target_bit_depth}, Output format: {output_ext}") logger.info(f"SaveImageVariants: Determined target bit depth: {target_bit_depth}, Output format: {output_ext} for map type {base_map_type}")
# 4. Generate and Save Resolution Variants # 4. Generate and Save Resolution Variants
# Sort resolutions by max dimension descending # Sort resolutions by max dimension descending
sorted_resolutions = sorted(image_resolutions.items(), key=lambda item: item[1], reverse=True) sorted_resolutions = sorted(image_resolutions.items(), key=lambda item: item[1], reverse=True)
for res_key, res_max_dim in sorted_resolutions: for res_key, res_max_dim in sorted_resolutions:
logger.info(f"Processing resolution variant: {res_key} ({res_max_dim} max dim)") logger.info(f"SaveImageVariants: Processing variant {res_key} ({res_max_dim}px) for {base_map_type}")
# Calculate target dimensions, ensuring no upscaling # --- Prevent Upscaling ---
if source_max_dim <= res_max_dim: # Skip this resolution variant if its target dimension is larger than the source image's largest dimension.
# If source is smaller or equal, use source dimensions if res_max_dim > source_max_dim:
logger.info(f"SaveImageVariants: Skipping variant {res_key} ({res_max_dim}px) for {base_map_type} because target resolution is larger than source ({source_max_dim}px).")
continue # Skip to the next resolution
# Calculate target dimensions for valid variants (equal or smaller than source)
if source_max_dim == res_max_dim:
# Use source dimensions if target is equal
target_w_res, target_h_res = source_w, source_h target_w_res, target_h_res = source_w, source_h
if source_max_dim < res_max_dim: logger.info(f"SaveImageVariants: Using source resolution ({source_w}x{source_h}) for {res_key} variant of {base_map_type} as target matches source.")
logger.info(f"Source image ({source_w}x{source_h}) is smaller than target resolution {res_key} ({res_max_dim}). Saving at source resolution.") else: # Downscale (source_max_dim > res_max_dim)
else:
# Downscale, maintaining aspect ratio # Downscale, maintaining aspect ratio
aspect_ratio = source_w / source_h aspect_ratio = source_w / source_h
if source_w > source_h: if source_w >= source_h: # Use >= to handle square images correctly
target_w_res = res_max_dim target_w_res = res_max_dim
target_h_res = int(res_max_dim / aspect_ratio) target_h_res = max(1, int(res_max_dim / aspect_ratio)) # Ensure height is at least 1
else: else:
target_h_res = res_max_dim target_h_res = res_max_dim
target_w_res = int(res_max_dim * aspect_ratio) target_w_res = max(1, int(res_max_dim * aspect_ratio)) # Ensure width is at least 1
logger.info(f"Resizing source image ({source_w}x{source_h}) to {target_w_res}x{target_h_res} for {res_key} variant.") logger.info(f"SaveImageVariants: Calculated downscale for {base_map_type} {res_key}: from ({source_w}x{source_h}) to ({target_w_res}x{target_h_res})")
# Resize source_image_data # Resize source_image_data (only if necessary)
# Use INTER_AREA for downscaling, INTER_LINEAR or INTER_CUBIC for upscaling (though we avoid upscaling here) if (target_w_res, target_h_res) == (source_w, source_h):
interpolation_method = cv2.INTER_AREA # Good for downscaling # No resize needed if dimensions match
# If we were allowing upscaling, we might add logic like: variant_data = source_image_data.copy() # Copy to avoid modifying original if needed later
# if target_w_res > source_w or target_h_res > source_h: logger.debug(f"SaveImageVariants: No resize needed for {base_map_type} {res_key}, using copy of source data.")
# interpolation_method = cv2.INTER_LINEAR # Or INTER_CUBIC else:
# Perform resize only if dimensions differ (i.e., downscaling)
try: interpolation_method = cv2.INTER_AREA # Good for downscaling
variant_data = ipu.resize_image(source_image_data, (target_w_res, target_h_res), interpolation=interpolation_method) try:
logger.debug(f"Resized variant data shape: {variant_data.shape}") variant_data = ipu.resize_image(source_image_data, target_w_res, target_h_res, interpolation=interpolation_method)
except Exception as e: if variant_data is None: # Check if resize failed
logger.error(f"Error resizing image for {res_key} variant: {e}") raise ValueError("ipu.resize_image returned None")
continue # Skip this variant if resizing fails logger.debug(f"SaveImageVariants: Resized variant data shape for {base_map_type} {res_key}: {variant_data.shape}")
except Exception as e:
logger.error(f"SaveImageVariants: Error resizing image for {base_map_type} {res_key} variant: {e}")
continue # Skip this variant if resizing fails
# Filename Construction # Filename Construction
current_tokens = output_filename_pattern_tokens.copy() current_tokens = output_filename_pattern_tokens.copy()
@ -172,14 +183,14 @@ def save_image_variants(
continue # Skip this variant continue # Skip this variant
output_path = output_base_directory / filename output_path = output_base_directory / filename
logger.info(f"Constructed output path: {output_path}") logger.info(f"SaveImageVariants: Constructed output path for {base_map_type} {res_key}: {output_path}")
# Ensure parent directory exists # Ensure parent directory exists
output_path.parent.mkdir(parents=True, exist_ok=True) output_path.parent.mkdir(parents=True, exist_ok=True)
logger.debug(f"Ensured directory exists: {output_path.parent}") logger.debug(f"SaveImageVariants: Ensured directory exists for {base_map_type} {res_key}: {output_path.parent}")
except Exception as e: except Exception as e:
logger.error(f"Error constructing filepath for {res_key} variant: {e}") logger.error(f"SaveImageVariants: Error constructing filepath for {base_map_type} {res_key} variant: {e}")
continue # Skip this variant if path construction fails continue # Skip this variant if path construction fails
@ -188,37 +199,40 @@ def save_image_variants(
if output_ext == 'jpg': if output_ext == 'jpg':
save_params_cv2.append(cv2.IMWRITE_JPEG_QUALITY) save_params_cv2.append(cv2.IMWRITE_JPEG_QUALITY)
save_params_cv2.append(jpg_quality) save_params_cv2.append(jpg_quality)
logger.debug(f"Using JPG quality: {jpg_quality}") logger.debug(f"SaveImageVariants: Using JPG quality: {jpg_quality} for {base_map_type} {res_key}")
elif output_ext == 'png': elif output_ext == 'png':
save_params_cv2.append(cv2.IMWRITE_PNG_COMPRESSION) save_params_cv2.append(cv2.IMWRITE_PNG_COMPRESSION)
save_params_cv2.append(png_compression_level) save_params_cv2.append(png_compression_level)
logger.debug(f"Using PNG compression level: {png_compression_level}") logger.debug(f"SaveImageVariants: Using PNG compression level: {png_compression_level} for {base_map_type} {res_key}")
# Add other format specific parameters if needed (e.g., TIFF compression) # Add other format specific parameters if needed (e.g., TIFF compression)
# Bit Depth Conversion (just before saving) # Bit Depth Conversion is handled by ipu.save_image via output_dtype_target
image_data_for_save = variant_data image_data_for_save = variant_data # Use the resized variant data directly
try:
if target_bit_depth == 8: # Determine the target dtype for ipu.save_image
image_data_for_save = ipu.convert_to_uint8(variant_data) output_dtype_for_save: Optional[np.dtype] = None
logger.debug("Converted variant data to uint8.") if target_bit_depth == 8:
elif target_bit_depth == 16: output_dtype_for_save = np.uint8
# ipu.convert_to_uint16 might handle different input types (float, uint8) elif target_bit_depth == 16:
# Assuming variant_data might be float after resizing, convert to uint16 output_dtype_for_save = np.uint16
image_data_for_save = ipu.convert_to_uint16(variant_data) # Add other target bit depths like float16/float32 if necessary
logger.debug("Converted variant data to uint16.") # elif target_bit_depth == 32: # Assuming float32 for EXR etc.
# Add other bit depth conversions if needed # output_dtype_for_save = np.float32
except Exception as e:
logger.error(f"Error converting image data to target bit depth {target_bit_depth} for {res_key} variant: {e}")
continue # Skip this variant if conversion fails
# Saving # Saving
try: try:
# ipu.save_image is expected to handle the actual cv2.imwrite call # ipu.save_image is expected to handle the actual cv2.imwrite call
success = ipu.save_image(str(output_path), image_data_for_save, params=save_params_cv2) logger.debug(f"SaveImageVariants: Attempting to save {base_map_type} {res_key} to {output_path} with params {save_params_cv2}, target_dtype: {output_dtype_for_save}")
success = ipu.save_image(
str(output_path),
image_data_for_save,
output_dtype_target=output_dtype_for_save, # Pass the target dtype
params=save_params_cv2
)
if success: if success:
logger.info(f"Successfully saved {res_key} variant to {output_path}") logger.info(f"SaveImageVariants: Successfully saved {base_map_type} {res_key} variant to {output_path}")
# Collect details for the returned list # Collect details for the returned list
saved_file_details.append({ saved_file_details.append({
'path': str(output_path), 'path': str(output_path),
@ -228,10 +242,10 @@ def save_image_variants(
'dimensions': (target_w_res, target_h_res) 'dimensions': (target_w_res, target_h_res)
}) })
else: else:
logger.error(f"Failed to save {res_key} variant to {output_path}") logger.error(f"SaveImageVariants: Failed to save {base_map_type} {res_key} variant to {output_path} (ipu.save_image returned False)")
except Exception as e: except Exception as e:
logger.error(f"Error saving image for {res_key} variant to {output_path}: {e}") logger.error(f"SaveImageVariants: Error during ipu.save_image for {base_map_type} {res_key} variant to {output_path}: {e}", exc_info=True)
# Continue to next variant even if one fails # Continue to next variant even if one fails