Rusfort/Asset-Frameworker
1
0
Fork 0
Code Issues 24 Pull Requests Actions Packages Projects 3 Releases Wiki Activity

More Refactor Fixes, Issuetracker updated

Browse Source
This commit is contained in:
Rusfort 2025-05-09 21:48:45 +02:00
parent beb8640085
commit 5bf53f036c
5 changed files with 509 additions and 318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

226
processing/pipeline/stages/gloss_to_rough_conversion.py
View File

@ -2,6 +2,7 @@ import logging
from pathlib import Path
import numpy as np
from typing import List
import dataclasses
from .base_stage import ProcessingStage
from ..asset_context import AssetProcessingContext
@ -35,135 +36,158 @@ class GlossToRoughConversionStage(ProcessingStage):
logger.debug(f"Asset '{asset_name_for_log}': Skipping GlossToRoughConversionStage due to skip_asset flag.")
return context
if not context.files_to_process or not context.processed_maps_details:
if not context.processed_maps_details: # files_to_process might be empty if only gloss maps existed and all are converted
logger.debug(
f"Asset '{asset_name_for_log}': No files to process or processed_maps_details empty "
f"in GlossToRoughConversionStage. Skipping."
f"Asset '{asset_name_for_log}': processed_maps_details is empty in GlossToRoughConversionStage. Skipping."
)
return context
new_files_to_process: List[FileRule] = []
# Start with a copy of the current file rules. We will modify this list.
new_files_to_process: List[FileRule] = list(context.files_to_process) if context.files_to_process else []
processed_a_gloss_map = False
successful_conversion_statuses = ['BasePOTSaved', 'Processed_With_Variants', 'Processed_No_Variants']
logger.info(f"Asset '{asset_name_for_log}': Starting Gloss to Roughness Conversion Stage.")
logger.info(f"Asset '{asset_name_for_log}': Starting Gloss to Roughness Conversion Stage. Examining {len(context.processed_maps_details)} processed map entries.")
for idx, file_rule in enumerate(context.files_to_process):
# Assuming FileRule has 'map_type' and 'id' (with a .hex attribute) and 'source_file_path'
# These might need to be checked with hasattr if they are optional or could be missing
if hasattr(file_rule, 'map_type') and file_rule.map_type == "GLOSS":
if not hasattr(file_rule, 'id') or not hasattr(file_rule.id, 'hex'):
logger.warning(f"Asset '{asset_name_for_log}': GLOSS FileRule missing 'id.hex'. Skipping conversion for this rule: {file_rule}")
new_files_to_process.append(file_rule)
continue
map_detail_key = file_rule.id.hex
# Iterate using the index (map_key_index) as the key, which is now standard.
for map_key_index, map_details in context.processed_maps_details.items():
processing_map_type = map_details.get('processing_map_type', '')
map_status = map_details.get('status')
original_temp_path_str = map_details.get('temp_processed_file')
# source_file_rule_idx from details should align with map_key_index.
# We primarily use map_key_index for accessing FileRule from context.files_to_process.
source_file_rule_idx_from_details = map_details.get('source_file_rule_index')
processing_tag = map_details.get('processing_tag')
source_file_path_for_log = file_rule.source_file_path if hasattr(file_rule, 'source_file_path') else "Unknown source path"
if map_key_index != source_file_rule_idx_from_details:
logger.warning(
f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: Mismatch between map key index and 'source_file_rule_index' ({source_file_rule_idx_from_details}) in details. "
f"Using map_key_index ({map_key_index}) for FileRule lookup. This might indicate a data consistency issue from previous stage."
)
if map_detail_key not in context.processed_maps_details:
logger.warning(
f"Asset '{asset_name_for_log}': GLOSS map '{source_file_path_for_log}' "
f"(ID: {map_detail_key}) found in files_to_process but not in processed_maps_details. "
f"Adding original rule and skipping conversion for this map."
)
new_files_to_process.append(file_rule)
continue
if not processing_tag:
logger.warning(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: 'processing_tag' is missing in map_details. Using a fallback for temp filename. This is unexpected.")
processing_tag = f"mki_{map_key_index}_fallback_tag"
map_details = context.processed_maps_details[map_detail_key]
if map_details.get('status') != 'Processed' or 'temp_processed_file' not in map_details:
logger.warning(
f"Asset '{asset_name_for_log}': GLOSS map '{source_file_path_for_log}' "
f"(ID: {map_detail_key}) not successfully processed by previous stage or temp file missing. "
f"Status: {map_details.get('status')}. Adding original rule and skipping conversion."
)
new_files_to_process.append(file_rule)
continue
if not processing_map_type.startswith("MAP_GLOSS"):
# logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: Type '{processing_map_type}' is not GLOSS. Skipping.")
continue
original_temp_path_str = map_details['temp_processed_file']
original_temp_path = Path(original_temp_path_str)
logger.info(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Identified potential GLOSS map (Type: {processing_map_type}).")
if not original_temp_path.exists():
logger.error(
f"Asset '{asset_name_for_log}': Temporary file {original_temp_path_str} for GLOSS map "
f"(ID: {map_detail_key}) does not exist. Adding original rule and skipping conversion."
)
new_files_to_process.append(file_rule)
continue
if map_status not in successful_conversion_statuses:
logger.warning(
f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): Status '{map_status}' is not one of {successful_conversion_statuses}. "
f"Skipping conversion for this map."
)
continue
logger.debug(f"Asset '{asset_name_for_log}': Processing GLOSS map {original_temp_path} for conversion.")
image_data = ipu.load_image(original_temp_path)
if not original_temp_path_str:
logger.warning(
f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): 'temp_processed_file' missing in details. "
f"Skipping conversion."
)
continue
if image_data is None:
logger.error(
f"Asset '{asset_name_for_log}': Failed to load image data from {original_temp_path} "
f"for GLOSS map (ID: {map_detail_key}). Adding original rule and skipping conversion."
)
new_files_to_process.append(file_rule)
continue
original_temp_path = Path(original_temp_path_str)
if not original_temp_path.exists():
logger.error(
f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): Temporary file {original_temp_path_str} "
f"does not exist. Skipping conversion."
)
continue
# Perform Inversion
inverted_image_data: np.ndarray
if np.issubdtype(image_data.dtype, np.floating):
inverted_image_data = 1.0 - image_data
inverted_image_data = np.clip(inverted_image_data, 0.0, 1.0) # Ensure range for floats
logger.debug(f"Asset '{asset_name_for_log}': Inverted float image data for {original_temp_path}.")
elif np.issubdtype(image_data.dtype, np.integer):
max_val = np.iinfo(image_data.dtype).max
inverted_image_data = max_val - image_data
logger.debug(f"Asset '{asset_name_for_log}': Inverted integer image data (max_val: {max_val}) for {original_temp_path}.")
# Use map_key_index directly to access the FileRule
# Ensure map_key_index is a valid index for context.files_to_process
if not isinstance(map_key_index, int) or map_key_index < 0 or map_key_index >= len(context.files_to_process):
logger.error(
f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}) (GLOSS): Invalid map_key_index ({map_key_index}) for accessing files_to_process (len: {len(context.files_to_process)}). "
f"Skipping conversion."
)
continue
original_file_rule = context.files_to_process[map_key_index]
source_file_path_for_log = original_file_rule.file_path if hasattr(original_file_rule, 'file_path') else "Unknown source path"
logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Processing GLOSS map from '{original_temp_path_str}' (Original FileRule path: '{source_file_path_for_log}') for conversion.")
image_data = ipu.load_image(str(original_temp_path))
if image_data is None:
logger.error(
f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Failed to load image data from {original_temp_path_str}. "
f"Skipping conversion."
)
continue
# Perform Inversion
inverted_image_data: np.ndarray
if np.issubdtype(image_data.dtype, np.floating):
inverted_image_data = 1.0 - image_data
inverted_image_data = np.clip(inverted_image_data, 0.0, 1.0)
logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Inverted float image data.")
elif np.issubdtype(image_data.dtype, np.integer):
max_val = np.iinfo(image_data.dtype).max
inverted_image_data = max_val - image_data
logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Inverted integer image data (max_val: {max_val}).")
else:
logger.error(
f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Unsupported image data type {image_data.dtype} "
f"for GLOSS map. Cannot invert. Skipping conversion."
)
continue
# Save New Temporary (Roughness) Map
new_temp_filename = f"rough_from_gloss_{processing_tag}{original_temp_path.suffix}"
new_temp_path = context.engine_temp_dir / new_temp_filename
save_success = ipu.save_image(str(new_temp_path), inverted_image_data)
if save_success:
logger.info(
f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Converted GLOSS map {original_temp_path_str} "
f"to ROUGHNESS map {new_temp_path}."
)
update_dict = {'item_type': "MAP_ROUGH", 'item_type_override': "MAP_ROUGH"}
modified_file_rule: Optional[FileRule] = None
if hasattr(original_file_rule, 'model_copy') and callable(original_file_rule.model_copy): # Pydantic
modified_file_rule = original_file_rule.model_copy(update=update_dict)
elif dataclasses.is_dataclass(original_file_rule): # Dataclass
modified_file_rule = dataclasses.replace(original_file_rule, **update_dict)
else:
logger.error(
f"Asset '{asset_name_for_log}': Unsupported image data type {image_data.dtype} "
f"for GLOSS map {original_temp_path}. Cannot invert. Adding original rule."
)
new_files_to_process.append(file_rule)
logger.error(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Original FileRule is neither Pydantic nor dataclass. Cannot modify. Skipping update for this rule.")
continue
# Save New Temporary (Roughness) Map
# Using original_temp_path.suffix ensures we keep the format (e.g., .png, .exr)
# Ensure file_rule.map_type exists before using sanitize_filename
map_type_for_filename = file_rule.map_type if hasattr(file_rule, 'map_type') else "unknownmaptype"
new_temp_filename = f"rough_from_gloss_{sanitize_filename(map_type_for_filename)}_{file_rule.id.hex}{original_temp_path.suffix}"
new_temp_path = context.engine_temp_dir / new_temp_filename
new_files_to_process[map_key_index] = modified_file_rule # Replace using map_key_index
save_success = ipu.save_image(new_temp_path, inverted_image_data)
# Update context.processed_maps_details for this map_key_index
map_details['temp_processed_file'] = str(new_temp_path)
map_details['original_map_type_before_conversion'] = processing_map_type
map_details['processing_map_type'] = "MAP_ROUGH"
map_details['map_type'] = "Roughness"
map_details['status'] = "Converted_To_Rough"
map_details['notes'] = map_details.get('notes', '') + "; Converted from GLOSS by GlossToRoughConversionStage"
if 'base_pot_resolution_name' in map_details:
map_details['processed_resolution_name'] = map_details['base_pot_resolution_name']
if save_success:
logger.info(
f"Asset '{asset_name_for_log}': Converted GLOSS map {original_temp_path} "
f"to ROUGHNESS map {new_temp_path}."
)
# Assuming FileRule has model_copy method
modified_file_rule = file_rule.model_copy(deep=True) if hasattr(file_rule, 'model_copy') else file_rule
modified_file_rule.map_type = "ROUGHNESS" # Ensure map_type can be set
# Update context.processed_maps_details for the original file_rule.id.hex
context.processed_maps_details[map_detail_key]['temp_processed_file'] = str(new_temp_path)
context.processed_maps_details[map_detail_key]['original_map_type_before_conversion'] = "GLOSS"
context.processed_maps_details[map_detail_key]['notes'] = "Converted from GLOSS by GlossToRoughConversionStage"
new_files_to_process.append(modified_file_rule)
processed_a_gloss_map = True
else:
logger.error(
f"Asset '{asset_name_for_log}': Failed to save inverted ROUGHNESS map to {new_temp_path} "
f"for original GLOSS map (ID: {map_detail_key}). Adding original rule."
)
new_files_to_process.append(file_rule)
else: # Not a gloss map
new_files_to_process.append(file_rule)
processed_a_gloss_map = True
else:
logger.error(
f"Asset '{asset_name_for_log}', Map Key Index {map_key_index} (Tag: {processing_tag}): Failed to save inverted ROUGHNESS map to {new_temp_path}. "
f"Original GLOSS FileRule remains."
)
context.files_to_process = new_files_to_process
if processed_a_gloss_map:
logger.info(
f"Asset '{asset_name_for_log}': Gloss to Roughness conversion stage successfully processed one or more maps and updated file list."
f"Asset '{asset_name_for_log}': Gloss to Roughness conversion stage finished. Processed one or more maps and updated file list and map details."
)
else:
logger.debug(
f"Asset '{asset_name_for_log}': No gloss maps were successfully converted in GlossToRoughConversionStage. "
f"File list for next stage contains original non-gloss maps and any gloss maps that failed conversion."
logger.info(
f"Asset '{asset_name_for_log}': No gloss maps were converted in GlossToRoughConversionStage. "
f"File list for next stage contains original non-gloss maps and any gloss maps that failed or were ineligible for conversion."
)
return context

289
processing/pipeline/stages/individual_map_processing.py
View File

@ -48,9 +48,9 @@ class IndividualMapProcessingStage(ProcessingStage):
context.status_flags['individual_map_processing_failed'] = True
# Mark all file_rules as failed
for fr_idx, file_rule_to_fail in enumerate(context.files_to_process):
temp_id_for_fail = f"fr_fail_{fr_idx}" # Temporary ID for status update
# Use fr_idx as the key for status update for these early failures
map_type_for_fail = file_rule_to_fail.item_type_override or file_rule_to_fail.item_type or "UnknownMapType"
self._update_file_rule_status(context, temp_id_for_fail, 'Failed', map_type=map_type_for_fail, details="SourceRule.input_path missing")
self._update_file_rule_status(context, fr_idx, 'Failed', map_type=map_type_for_fail, details="SourceRule.input_path missing")
return context
# The workspace_path in the context should be the directory where files are extracted/available.
@ -59,9 +59,9 @@ class IndividualMapProcessingStage(ProcessingStage):
logger.error(f"Asset '{asset_name_for_log}': Workspace path '{source_base_path}' is not a valid directory.")
context.status_flags['individual_map_processing_failed'] = True
for fr_idx, file_rule_to_fail in enumerate(context.files_to_process):
temp_id_for_fail = f"fr_fail_{fr_idx}" # Use a temporary unique ID for this status update
# Use fr_idx as the key for status update
map_type_for_fail = file_rule_to_fail.item_type_override or file_rule_to_fail.item_type or "UnknownMapType"
self._update_file_rule_status(context, temp_id_for_fail, 'Failed', map_type=map_type_for_fail, details="Workspace path invalid")
self._update_file_rule_status(context, fr_idx, 'Failed', map_type=map_type_for_fail, details="Workspace path invalid")
return context
# Fetch config settings once before the loop
@ -70,8 +70,17 @@ class IndividualMapProcessingStage(ProcessingStage):
output_filename_pattern = getattr(context.config_obj, "output_filename_pattern", "[assetname]_[maptype]_[resolution].[ext]")
for file_rule_idx, file_rule in enumerate(context.files_to_process):
# Generate a unique ID for this file_rule processing instance for processed_maps_details
current_map_id_hex = f"map_{file_rule_idx}_{uuid.uuid4().hex[:8]}"
# file_rule_idx will be the key for processed_maps_details.
# processing_instance_tag is for unique temp files and detailed logging for this specific run.
processing_instance_tag = f"map_{file_rule_idx}_{uuid.uuid4().hex[:8]}"
current_map_key = file_rule_idx # Key for processed_maps_details
if not file_rule.file_path: # Ensure file_path exists, critical for later stages if they rely on it from FileRule
logger.error(f"Asset '{asset_name_for_log}', FileRule at index {file_rule_idx} has an empty or None file_path. Skipping this rule.")
self._update_file_rule_status(context, current_map_key, 'Failed',
processing_tag=processing_instance_tag,
details="FileRule has no file_path")
continue
initial_current_map_type = file_rule.item_type_override or file_rule.item_type or "UnknownMapType"
@ -130,89 +139,98 @@ class IndividualMapProcessingStage(ProcessingStage):
# --- END NEW SUFFIXING LOGIC ---
# --- START: Filename-friendly map type derivation ---
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: --- Starting Filename-Friendly Map Type Logic for: {current_map_type} ---")
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: --- Starting Filename-Friendly Map Type Logic for: {current_map_type} ---")
filename_friendly_map_type = current_map_type # Fallback
# 1. Access FILE_TYPE_DEFINITIONS
file_type_definitions = None
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Attempting to access context.config_obj.FILE_TYPE_DEFINITIONS.")
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Attempting to access context.config_obj.FILE_TYPE_DEFINITIONS.")
try:
file_type_definitions = context.config_obj.FILE_TYPE_DEFINITIONS
if not file_type_definitions: # Check if it's None or empty
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: FILE_TYPE_DEFINITIONS is present but empty or None.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS is present but empty or None.")
else:
sample_defs_log = {k: file_type_definitions[k] for k in list(file_type_definitions.keys())[:2]} # Log first 2 for brevity
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Accessed FILE_TYPE_DEFINITIONS. Sample: {sample_defs_log}, Total keys: {len(file_type_definitions)}.")
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Accessed FILE_TYPE_DEFINITIONS. Sample: {sample_defs_log}, Total keys: {len(file_type_definitions)}.")
except AttributeError:
logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Could not access context.config_obj.FILE_TYPE_DEFINITIONS via direct attribute.")
logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Could not access context.config_obj.FILE_TYPE_DEFINITIONS via direct attribute.")
base_map_key = None
base_map_key_val = None # Renamed from base_map_key to avoid conflict with current_map_key
suffix_part = ""
if file_type_definitions and isinstance(file_type_definitions, dict) and len(file_type_definitions) > 0:
base_map_key = None
base_map_key_val = None
suffix_part = ""
sorted_known_base_keys = sorted(list(file_type_definitions.keys()), key=len, reverse=True)
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Sorted known base keys for parsing: {sorted_known_base_keys}")
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Sorted known base keys for parsing: {sorted_known_base_keys}")
for known_key in sorted_known_base_keys:
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Checking if '{current_map_type}' starts with '{known_key}'")
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Checking if '{current_map_type}' starts with '{known_key}'")
if current_map_type.startswith(known_key):
base_map_key = known_key
base_map_key_val = known_key
suffix_part = current_map_type[len(known_key):]
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Match found! current_map_type: '{current_map_type}', base_map_key: '{base_map_key}', suffix_part: '{suffix_part}'")
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Match found! current_map_type: '{current_map_type}', base_map_key_val: '{base_map_key_val}', suffix_part: '{suffix_part}'")
break
if base_map_key is None:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Could not parse base_map_key from '{current_map_type}' using known keys. Fallback: filename_friendly_map_type = '{filename_friendly_map_type}'.")
if base_map_key_val is None:
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Could not parse base_map_key_val from '{current_map_type}' using known keys. Fallback: filename_friendly_map_type = '{filename_friendly_map_type}'.")
else:
definition = file_type_definitions.get(base_map_key)
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Definition for '{base_map_key}': {definition}")
definition = file_type_definitions.get(base_map_key_val)
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Definition for '{base_map_key_val}': {definition}")
if definition and isinstance(definition, dict):
standard_type_alias = definition.get("standard_type")
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Standard type alias for '{base_map_key}': '{standard_type_alias}'")
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Standard type alias for '{base_map_key_val}': '{standard_type_alias}'")
if standard_type_alias and isinstance(standard_type_alias, str) and standard_type_alias.strip():
filename_friendly_map_type = standard_type_alias.strip() + suffix_part
logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Successfully transformed map type: '{current_map_type}' -> '{filename_friendly_map_type}' (standard_type_alias: '{standard_type_alias}', suffix_part: '{suffix_part}').")
logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Successfully transformed map type: '{current_map_type}' -> '{filename_friendly_map_type}' (standard_type_alias: '{standard_type_alias}', suffix_part: '{suffix_part}').")
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Standard type alias for '{base_map_key}' is missing, empty, or not a string (value: '{standard_type_alias}'). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Standard type alias for '{base_map_key_val}' is missing, empty, or not a string (value: '{standard_type_alias}'). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.")
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: No definition or invalid definition for '{base_map_key}' (value: {definition}). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: No definition or invalid definition for '{base_map_key_val}' (value: {definition}). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.")
elif file_type_definitions is None:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: FILE_TYPE_DEFINITIONS not available for lookup (was None). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS not available for lookup (was None). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.")
elif not isinstance(file_type_definitions, dict):
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: FILE_TYPE_DEFINITIONS is not a dictionary (type: {type(file_type_definitions)}). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS is not a dictionary (type: {type(file_type_definitions)}). Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.")
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: FILE_TYPE_DEFINITIONS is an empty dictionary. Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: FILE_TYPE_DEFINITIONS is an empty dictionary. Using fallback. filename_friendly_map_type = '{filename_friendly_map_type}'.")
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Final filename_friendly_map_type: '{filename_friendly_map_type}'")
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Final filename_friendly_map_type: '{filename_friendly_map_type}'")
# --- END: Filename-friendly map type derivation ---
if not current_map_type or not current_map_type.startswith("MAP_") or current_map_type == "MAP_GEN_COMPOSITE":
logger.debug(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}': Skipping, item_type '{current_map_type}' (initial: '{initial_current_map_type}') not targeted for individual processing.")
continue
logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Type: {current_map_type}, Initial Type: {initial_current_map_type}, ID: {current_map_id_hex}): Starting individual processing.")
logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Type: {current_map_type}, Initial Type: {initial_current_map_type}, Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Starting individual processing.")
# A. Find Source File (using file_rule.file_path as the pattern relative to source_base_path)
# The _find_source_file might need adjustment if file_rule.file_path is absolute or needs complex globbing.
# For now, assume file_rule.file_path is a relative pattern or exact name.
source_file_path = self._find_source_file(source_base_path, file_rule.file_path, asset_name_for_log, current_map_id_hex)
source_file_path = self._find_source_file(source_base_path, file_rule.file_path, asset_name_for_log, processing_instance_tag)
if not source_file_path:
logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Source file not found with path/pattern '{file_rule.file_path}' in '{source_base_path}'.")
self._update_file_rule_status(context, current_map_id_hex, 'Failed', map_type=filename_friendly_map_type, details="Source file not found")
logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Source file not found with path/pattern '{file_rule.file_path}' in '{source_base_path}'.")
self._update_file_rule_status(context, current_map_key, 'Failed',
map_type=filename_friendly_map_type,
processing_map_type=current_map_type,
source_file_rule_index=file_rule_idx,
processing_tag=processing_instance_tag,
details="Source file not found")
continue
# B. Load and Transform Image
image_data: Optional[np.ndarray] = ipu.load_image(str(source_file_path))
if image_data is None:
logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Failed to load image from '{source_file_path}'.")
self._update_file_rule_status(context, current_map_id_hex, 'Failed', map_type=filename_friendly_map_type, source_file=str(source_file_path), details="Image load failed")
logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Failed to load image from '{source_file_path}'.")
self._update_file_rule_status(context, current_map_key, 'Failed',
map_type=filename_friendly_map_type,
processing_map_type=current_map_type,
source_file_rule_index=file_rule_idx,
processing_tag=processing_instance_tag,
source_file=str(source_file_path),
details="Image load failed")
continue
original_height, original_width = image_data.shape[:2]
logger.debug(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Loaded image '{source_file_path}' with dimensions {original_width}x{original_height}.")
logger.debug(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Loaded image '{source_file_path}' with dimensions {original_width}x{original_height}.")
# 1. Initial Power-of-Two (POT) Downscaling
pot_width = ipu.get_nearest_power_of_two_downscale(original_width)
@ -286,7 +304,7 @@ class IndividualMapProcessingStage(ProcessingStage):
base_pot_width, base_pot_height = 1, 1
logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Original dims: ({original_width},{original_height}), Initial POT Scaled Dims: ({base_pot_width},{base_pot_height}).")
logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Original dims: ({original_width},{original_height}), Initial POT Scaled Dims: ({base_pot_width},{base_pot_height}).")
# Calculate and store aspect ratio change string
if original_width > 0 and original_height > 0 and base_pot_width > 0 and base_pot_height > 0:
@ -297,19 +315,26 @@ class IndividualMapProcessingStage(ProcessingStage):
if aspect_change_str:
# This will overwrite if multiple maps are processed; specified by requirements.
context.asset_metadata['aspect_ratio_change_string'] = aspect_change_str
logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type {current_map_type}: Calculated aspect ratio change string: '{aspect_change_str}' (Original: {original_width}x{original_height}, Base POT: {base_pot_width}x{base_pot_height}). Stored in asset_metadata.")
logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type {current_map_type}: Calculated aspect ratio change string: '{aspect_change_str}' (Original: {original_width}x{original_height}, Base POT: {base_pot_width}x{base_pot_height}). Stored in asset_metadata.")
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type {current_map_type}: Failed to calculate aspect ratio change string.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type {current_map_type}: Failed to calculate aspect ratio change string.")
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type {current_map_type}: Skipping aspect ratio change string calculation due to invalid dimensions (Original: {original_width}x{original_height}, Base POT: {base_pot_width}x{base_pot_height}).")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type {current_map_type}: Skipping aspect ratio change string calculation due to invalid dimensions (Original: {original_width}x{original_height}, Base POT: {base_pot_width}x{base_pot_height}).")
base_pot_image_data = image_data.copy()
if (base_pot_width, base_pot_height) != (original_width, original_height):
interpolation = cv2.INTER_AREA # Good for downscaling
base_pot_image_data = ipu.resize_image(base_pot_image_data, base_pot_width, base_pot_height, interpolation=interpolation)
if base_pot_image_data is None:
logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Failed to resize image to base POT dimensions.")
self._update_file_rule_status(context, current_map_id_hex, 'Failed', map_type=filename_friendly_map_type, source_file=str(source_file_path), original_dimensions=(original_width, original_height), details="Base POT resize failed")
logger.error(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Failed to resize image to base POT dimensions.")
self._update_file_rule_status(context, current_map_key, 'Failed',
map_type=filename_friendly_map_type,
processing_map_type=current_map_type,
source_file_rule_index=file_rule_idx,
processing_tag=processing_instance_tag,
source_file=str(source_file_path),
original_dimensions=(original_width, original_height),
details="Base POT resize failed")
continue
# Color Profile Management (after initial POT resize, before multi-res saving)
@ -323,14 +348,14 @@ class IndividualMapProcessingStage(ProcessingStage):
custom_transform_settings = file_rule.channel_merge_instructions['transform']
if isinstance(custom_transform_settings, dict):
transform_settings.update(custom_transform_settings)
logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Loaded transform settings for color/output from file_rule.")
logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Loaded transform settings for color/output from file_rule.")
if transform_settings['color_profile_management'] and transform_settings['target_color_profile'] == "RGB":
if len(base_pot_image_data.shape) == 3 and base_pot_image_data.shape[2] == 3: # BGR to RGB
logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Converting BGR to RGB for base POT image.")
logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Converting BGR to RGB for base POT image.")
base_pot_image_data = ipu.convert_bgr_to_rgb(base_pot_image_data)
elif len(base_pot_image_data.shape) == 3 and base_pot_image_data.shape[2] == 4: # BGRA to RGBA
logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (ID: {current_map_id_hex}): Converting BGRA to RGBA for base POT image.")
logger.info(f"Asset '{asset_name_for_log}', FileRule path '{file_rule.file_path}' (Key: {current_map_key}, Proc. Tag: {processing_instance_tag}): Converting BGRA to RGBA for base POT image.")
base_pot_image_data = ipu.convert_bgra_to_rgba(base_pot_image_data)
# Ensure engine_temp_dir exists before saving base POT
@ -340,11 +365,17 @@ class IndividualMapProcessingStage(ProcessingStage):
logger.info(f"Asset '{asset_name_for_log}': Created engine_temp_dir at '{context.engine_temp_dir}'")
except OSError as e:
logger.error(f"Asset '{asset_name_for_log}': Failed to create engine_temp_dir '{context.engine_temp_dir}': {e}")
self._update_file_rule_status(context, current_map_id_hex, 'Failed', map_type=filename_friendly_map_type, source_file=str(source_file_path), details="Failed to create temp directory for base POT")
self._update_file_rule_status(context, current_map_key, 'Failed',
map_type=filename_friendly_map_type,
processing_map_type=current_map_type,
source_file_rule_index=file_rule_idx,
processing_tag=processing_instance_tag,
source_file=str(source_file_path),
details="Failed to create temp directory for base POT")
continue
temp_filename_suffix = Path(source_file_path).suffix
base_pot_temp_filename = f"{current_map_id_hex}_basePOT{temp_filename_suffix}"
base_pot_temp_filename = f"{processing_instance_tag}_basePOT{temp_filename_suffix}" # Use processing_instance_tag
base_pot_temp_path = context.engine_temp_dir / base_pot_temp_filename
# Determine save parameters for base POT image (can be different from variants if needed)
@ -354,18 +385,29 @@ class IndividualMapProcessingStage(ProcessingStage):
# For now, using simple save.
if not ipu.save_image(str(base_pot_temp_path), base_pot_image_data, params=base_save_params):
logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Failed to save base POT image to '{base_pot_temp_path}'.")
self._update_file_rule_status(context, current_map_id_hex, 'Failed', map_type=filename_friendly_map_type, source_file=str(source_file_path), original_dimensions=(original_width, original_height), base_pot_dimensions=(base_pot_width, base_pot_height), details="Base POT image save failed")
logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Failed to save base POT image to '{base_pot_temp_path}'.")
self._update_file_rule_status(context, current_map_key, 'Failed',
map_type=filename_friendly_map_type,
processing_map_type=current_map_type,
source_file_rule_index=file_rule_idx,
processing_tag=processing_instance_tag,
source_file=str(source_file_path),
original_dimensions=(original_width, original_height),
base_pot_dimensions=(base_pot_width, base_pot_height),
details="Base POT image save failed")
continue
logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Successfully saved base POT image to '{base_pot_temp_path}' with dims ({base_pot_width}x{base_pot_height}).")
logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Successfully saved base POT image to '{base_pot_temp_path}' with dims ({base_pot_width}x{base_pot_height}).")
# Initialize/update the status for this map in processed_maps_details
self._update_file_rule_status(
context,
current_map_id_hex,
current_map_key, # Use file_rule_idx as key
'BasePOTSaved', # Intermediate status, will be updated after variant check
map_type=filename_friendly_map_type,
processing_map_type=current_map_type,
source_file_rule_index=file_rule_idx,
processing_tag=processing_instance_tag, # Store the tag
source_file=str(source_file_path),
original_dimensions=(original_width, original_height),
base_pot_dimensions=(base_pot_width, base_pot_height),
@ -375,20 +417,20 @@ class IndividualMapProcessingStage(ProcessingStage):
# 2. Multiple Resolution Output (Variants)
processed_at_least_one_resolution_variant = False
# Resolution variants are attempted for all map types individually processed.
# The filter at the beginning of the loop (around line 72) ensures only relevant maps reach this stage.
# The filter at the beginning of the loop ensures only relevant maps reach this stage.
generate_variants_for_this_map_type = True
if generate_variants_for_this_map_type: # This will now always be true if code execution reaches here
logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Map type '{current_map_type}' is eligible for individual processing. Attempting to generate resolution variants.")
logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Map type '{current_map_type}' is eligible for individual processing. Attempting to generate resolution variants.")
# Sort resolutions from largest to smallest
sorted_resolutions = sorted(image_resolutions.items(), key=lambda item: item[1], reverse=True)
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Sorted resolutions for variant processing: {sorted_resolutions}")
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Sorted resolutions for variant processing: {sorted_resolutions}")
for res_key, res_max_dim in sorted_resolutions:
current_w, current_h = base_pot_image_data.shape[1], base_pot_image_data.shape[0]
if current_w <= 0 or current_h <=0:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Base POT image has zero dimension ({current_w}x{current_h}). Skipping this resolution variant.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Base POT image has zero dimension ({current_w}x{current_h}). Skipping this resolution variant.")
continue
if max(current_w, current_h) >= res_max_dim:
@ -401,24 +443,24 @@ class IndividualMapProcessingStage(ProcessingStage):
target_h_res = res_max_dim
target_w_res = max(1, round(target_h_res * (current_w / current_h)))
else:
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Base POT image ({current_w}x{current_h}) is smaller than target max dim {res_max_dim}. Skipping this resolution variant.")
logger.debug(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Base POT image ({current_w}x{current_h}) is smaller than target max dim {res_max_dim}. Skipping this resolution variant.")
continue
target_w_res = min(target_w_res, current_w)
target_h_res = min(target_h_res, current_h)
if target_w_res <=0 or target_h_res <=0:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Calculated target variant dims are zero or negative ({target_w_res}x{target_h_res}). Skipping.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Calculated target variant dims are zero or negative ({target_w_res}x{target_h_res}). Skipping.")
continue
logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Processing variant for {res_max_dim}. Base POT Dims: ({current_w}x{current_h}), Target Dims for {res_key}: ({target_w_res}x{target_h_res}).")
logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Processing variant for {res_max_dim}. Base POT Dims: ({current_w}x{current_h}), Target Dims for {res_key}: ({target_w_res}x{target_h_res}).")
output_image_data_for_res = base_pot_image_data
if (target_w_res, target_h_res) != (current_w, current_h):
interpolation_res = cv2.INTER_AREA
output_image_data_for_res = ipu.resize_image(base_pot_image_data, target_w_res, target_h_res, interpolation=interpolation_res)
if output_image_data_for_res is None:
logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Failed to resize image for resolution variant {res_key}.")
logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Failed to resize image for resolution variant {res_key}.")
continue
assetname_placeholder = context.asset_rule.asset_name if context.asset_rule else "UnknownAsset"
@ -431,7 +473,7 @@ class IndividualMapProcessingStage(ProcessingStage):
.replace("[maptype]", sanitize_filename(filename_friendly_map_type)) \
.replace("[resolution]", sanitize_filename(resolution_placeholder)) \
.replace("[ext]", output_ext_variant)
temp_output_filename_variant = f"{current_map_id_hex}_variant_{temp_output_filename_variant}" # Distinguish variant temp files
temp_output_filename_variant = f"{processing_instance_tag}_variant_{temp_output_filename_variant}" # Use processing_instance_tag
temp_output_path_variant = context.engine_temp_dir / temp_output_filename_variant
save_params_variant = []
@ -446,26 +488,26 @@ class IndividualMapProcessingStage(ProcessingStage):
save_success_variant = ipu.save_image(str(temp_output_path_variant), output_image_data_for_res, params=save_params_variant)
if not save_success_variant:
logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Failed to save temporary variant image to '{temp_output_path_variant}'.")
logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Failed to save temporary variant image to '{temp_output_path_variant}'.")
continue
logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Res {res_key}: Successfully saved temporary variant map to '{temp_output_path_variant}' with dims ({target_w_res}x{target_h_res}).")
logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Res {res_key}: Successfully saved temporary variant map to '{temp_output_path_variant}' with dims ({target_w_res}x{target_h_res}).")
processed_at_least_one_resolution_variant = True
if 'variants' not in context.processed_maps_details[current_map_id_hex]:
context.processed_maps_details[current_map_id_hex]['variants'] = []
if 'variants' not in context.processed_maps_details[current_map_key]: # Use current_map_key (file_rule_idx)
context.processed_maps_details[current_map_key]['variants'] = []
context.processed_maps_details[current_map_id_hex]['variants'].append({
context.processed_maps_details[current_map_key]['variants'].append({ # Use current_map_key (file_rule_idx)
'resolution_key': res_key,
'temp_path': str(temp_output_path_variant), # Changed 'path' to 'temp_path'
'temp_path': str(temp_output_path_variant),
'dimensions': (target_w_res, target_h_res),
'resolution_name': f"{target_w_res}x{target_h_res}" # Retain for potential use
'resolution_name': f"{target_w_res}x{target_h_res}"
})
if 'processed_files' not in context.asset_metadata:
context.asset_metadata['processed_files'] = []
context.asset_metadata['processed_files'].append({
'processed_map_key': current_map_id_hex,
'processed_map_key': current_map_key, # Use current_map_key (file_rule_idx)
'resolution_key': res_key,
'path': str(temp_output_path_variant),
'type': 'temporary_map_variant',
@ -479,11 +521,11 @@ class IndividualMapProcessingStage(ProcessingStage):
source_of_stats_image = "unknown"
if processed_at_least_one_resolution_variant and \
current_map_id_hex in context.processed_maps_details and \
'variants' in context.processed_maps_details[current_map_id_hex] and \
context.processed_maps_details[current_map_id_hex]['variants']:
current_map_key in context.processed_maps_details and \
'variants' in context.processed_maps_details[current_map_key] and \
context.processed_maps_details[current_map_key]['variants']:
variants_list = context.processed_maps_details[current_map_id_hex]['variants']
variants_list = context.processed_maps_details[current_map_key]['variants']
valid_variants_for_stats = [
v for v in variants_list
if isinstance(v.get('dimensions'), tuple) and len(v['dimensions']) == 2 and v['dimensions'][0] > 0 and v['dimensions'][1] > 0
@ -494,25 +536,25 @@ class IndividualMapProcessingStage(ProcessingStage):
if smallest_variant and 'temp_path' in smallest_variant and smallest_variant.get('dimensions'):
smallest_res_w, smallest_res_h = smallest_variant['dimensions']
logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Identified smallest variant for stats: {smallest_variant.get('resolution_key', 'N/A')} ({smallest_res_w}x{smallest_res_h}) at {smallest_variant['temp_path']}")
logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Identified smallest variant for stats: {smallest_variant.get('resolution_key', 'N/A')} ({smallest_res_w}x{smallest_res_h}) at {smallest_variant['temp_path']}")
lowest_res_image_data_for_stats = ipu.load_image(smallest_variant['temp_path'])
image_to_stat_path_for_log = smallest_variant['temp_path']
source_of_stats_image = f"variant {smallest_variant.get('resolution_key', 'N/A')}"
if lowest_res_image_data_for_stats is None:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Failed to load smallest variant image '{smallest_variant['temp_path']}' for stats.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Failed to load smallest variant image '{smallest_variant['temp_path']}' for stats.")
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Could not determine smallest variant for stats from valid variants list (details missing).")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Could not determine smallest variant for stats from valid variants list (details missing).")
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: No valid variants found to determine the smallest one for stats.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: No valid variants found to determine the smallest one for stats.")
if lowest_res_image_data_for_stats is None:
if base_pot_image_data is not None:
logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Using base POT image for stats (dimensions: {base_pot_width}x{base_pot_height}). Smallest variant not available/loaded or no variants generated.")
logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Using base POT image for stats (dimensions: {base_pot_width}x{base_pot_height}). Smallest variant not available/loaded or no variants generated.")
lowest_res_image_data_for_stats = base_pot_image_data
image_to_stat_path_for_log = f"In-memory base POT image (dims: {base_pot_width}x{base_pot_height})"
source_of_stats_image = "base POT"
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Base POT image data is also None. Cannot calculate stats.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Base POT image data is also None. Cannot calculate stats.")
if lowest_res_image_data_for_stats is not None:
stats_dict = ipu.calculate_image_stats(lowest_res_image_data_for_stats)
@ -520,43 +562,59 @@ class IndividualMapProcessingStage(ProcessingStage):
if 'image_stats_lowest_res' not in context.asset_metadata:
context.asset_metadata['image_stats_lowest_res'] = {}
context.asset_metadata['image_stats_lowest_res'][current_map_type] = stats_dict
logger.info(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type '{current_map_type}': Calculated and stored image stats from '{source_of_stats_image}' (source ref: '{image_to_stat_path_for_log}').")
context.asset_metadata['image_stats_lowest_res'][current_map_type] = stats_dict # Keyed by map_type
logger.info(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': Calculated and stored image stats from '{source_of_stats_image}' (source ref: '{image_to_stat_path_for_log}').")
elif stats_dict and "error" in stats_dict:
logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type '{current_map_type}': Error calculating image stats from '{source_of_stats_image}': {stats_dict['error']}.")
logger.error(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': Error calculating image stats from '{source_of_stats_image}': {stats_dict['error']}.")
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type '{current_map_type}': Failed to calculate image stats from '{source_of_stats_image}' (result was None or empty).")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': Failed to calculate image stats from '{source_of_stats_image}' (result was None or empty).")
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}, Map Type '{current_map_type}': No image data available (from variant or base POT) to calculate stats.")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}, Map Type '{current_map_type}': No image data available (from variant or base POT) to calculate stats.")
# Final status update based on whether variants were generated (and expected)
if generate_variants_for_this_map_type:
if processed_at_least_one_resolution_variant:
self._update_file_rule_status(context, current_map_id_hex, 'Processed_With_Variants', map_type=filename_friendly_map_type, details="Successfully processed with multiple resolution variants.")
self._update_file_rule_status(context, current_map_key, 'Processed_With_Variants',
map_type=filename_friendly_map_type,
processing_map_type=current_map_type,
source_file_rule_index=file_rule_idx,
processing_tag=processing_instance_tag,
details="Successfully processed with multiple resolution variants.")
else:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Variants were expected for map type '{current_map_type}', but none were generated (e.g., base POT too small for any variant tier).")
self._update_file_rule_status(context, current_map_id_hex, 'Processed_No_Variants', map_type=filename_friendly_map_type, details="Variants expected but none generated (e.g., base POT too small).")
logger.warning(f"Asset '{asset_name_for_log}', Map Key {current_map_key}, Proc. Tag {processing_instance_tag}: Variants were expected for map type '{current_map_type}', but none were generated (e.g., base POT too small for any variant tier).")
self._update_file_rule_status(context, current_map_key, 'Processed_No_Variants',
map_type=filename_friendly_map_type,
processing_map_type=current_map_type,
source_file_rule_index=file_rule_idx,
processing_tag=processing_instance_tag,
details="Variants expected but none generated (e.g., base POT too small).")
else: # No variants were expected for this map type
self._update_file_rule_status(context, current_map_id_hex, 'Processed_No_Variants', map_type=filename_friendly_map_type, details="Processed to base POT; variants not applicable for this map type.")
self._update_file_rule_status(context, current_map_key, 'Processed_No_Variants',
map_type=filename_friendly_map_type,
processing_map_type=current_map_type,
source_file_rule_index=file_rule_idx,
processing_tag=processing_instance_tag,
details="Processed to base POT; variants not applicable for this map type.")
logger.info(f"Asset '{asset_name_for_log}': Finished individual map processing stage.")
return context
def _find_source_file(self, base_path: Path, pattern: str, asset_name_for_log: str, current_map_id_hex: str) -> Optional[Path]: # asset_id -> asset_name_for_log, file_rule_id_hex -> current_map_id_hex
def _find_source_file(self, base_path: Path, pattern: str, asset_name_for_log: str, processing_instance_tag: str) -> Optional[Path]:
"""
Finds a single source file matching the pattern within the base_path.
Logs use processing_instance_tag for specific run tracing.
"""
if not pattern: # pattern is now file_rule.file_path
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Empty file_path provided in FileRule.")
if not pattern:
logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Empty file_path provided in FileRule.")
return None
# If pattern is an absolute path, use it directly
potential_abs_path = Path(pattern)
if potential_abs_path.is_absolute() and potential_abs_path.exists():
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: file_path '{pattern}' is absolute and exists. Using it directly.")
logger.debug(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: file_path '{pattern}' is absolute and exists. Using it directly.")
return potential_abs_path
elif potential_abs_path.is_absolute():
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: file_path '{pattern}' is absolute but does not exist.")
logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: file_path '{pattern}' is absolute but does not exist.")
# Fall through to try resolving against base_path if it's just a name/relative pattern
# Treat pattern as relative to base_path
@ -565,46 +623,49 @@ class IndividualMapProcessingStage(ProcessingStage):
# First, check if pattern is an exact relative path
exact_match_path = base_path / pattern
if exact_match_path.exists() and exact_match_path.is_file():
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Found exact match for '{pattern}' at '{exact_match_path}'.")
logger.debug(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Found exact match for '{pattern}' at '{exact_match_path}'.")
return exact_match_path
# If not an exact match, try as a glob pattern (recursive)
matched_files_rglob = list(base_path.rglob(pattern))
if matched_files_rglob:
if len(matched_files_rglob) > 1:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Multiple files ({len(matched_files_rglob)}) found for pattern '{pattern}' in '{base_path}' (recursive). Using first: {matched_files_rglob[0]}. Files: {matched_files_rglob}")
logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Multiple files ({len(matched_files_rglob)}) found for pattern '{pattern}' in '{base_path}' (recursive). Using first: {matched_files_rglob[0]}. Files: {matched_files_rglob}")
return matched_files_rglob[0]
# Try non-recursive glob if rglob fails
matched_files_glob = list(base_path.glob(pattern))
if matched_files_glob:
if len(matched_files_glob) > 1:
logger.warning(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Multiple files ({len(matched_files_glob)}) found for pattern '{pattern}' in '{base_path}' (non-recursive). Using first: {matched_files_glob[0]}. Files: {matched_files_glob}")
logger.warning(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Multiple files ({len(matched_files_glob)}) found for pattern '{pattern}' in '{base_path}' (non-recursive). Using first: {matched_files_glob[0]}. Files: {matched_files_glob}")
return matched_files_glob[0]
logger.debug(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: No files found matching pattern '{pattern}' in '{base_path}' (exact, recursive, or non-recursive).")
logger.debug(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: No files found matching pattern '{pattern}' in '{base_path}' (exact, recursive, or non-recursive).")
return None
except Exception as e:
logger.error(f"Asset '{asset_name_for_log}', Map ID {current_map_id_hex}: Error searching for file with pattern '{pattern}' in '{base_path}': {e}")
logger.error(f"Asset '{asset_name_for_log}', Proc. Tag {processing_instance_tag}: Error searching for file with pattern '{pattern}' in '{base_path}': {e}")
return None
def _update_file_rule_status(self, context: AssetProcessingContext, map_id_hex: str, status: str, **kwargs): # file_rule_id_hex -> map_id_hex
"""Helper to update processed_maps_details for a map."""
def _update_file_rule_status(self, context: AssetProcessingContext, map_key_index: int, status: str, **kwargs): # Renamed map_id_hex to map_key_index
"""Helper to update processed_maps_details for a map, keyed by file_rule_idx."""
asset_name_for_log = context.asset_rule.asset_name if context.asset_rule else "Unknown Asset"
if map_id_hex not in context.processed_maps_details:
context.processed_maps_details[map_id_hex] = {}
if map_key_index not in context.processed_maps_details:
context.processed_maps_details[map_key_index] = {}
context.processed_maps_details[map_id_hex]['status'] = status
context.processed_maps_details[map_key_index]['status'] = status
for key, value in kwargs.items():
context.processed_maps_details[map_id_hex][key] = value
# Ensure source_file_rule_id_hex is not added if it was somehow passed (it shouldn't be)
if key == 'source_file_rule_id_hex':
continue
context.processed_maps_details[map_key_index][key] = value
if 'map_type' not in context.processed_maps_details[map_id_hex] and 'map_type' in kwargs:
context.processed_maps_details[map_id_hex]['map_type'] = kwargs['map_type']
if 'map_type' not in context.processed_maps_details[map_key_index] and 'map_type' in kwargs:
context.processed_maps_details[map_key_index]['map_type'] = kwargs['map_type']
# Add formatted resolution names
if 'original_dimensions' in kwargs and isinstance(kwargs['original_dimensions'], tuple) and len(kwargs['original_dimensions']) == 2:
orig_w, orig_h = kwargs['original_dimensions']
context.processed_maps_details[map_id_hex]['original_resolution_name'] = f"{orig_w}x{orig_h}"
context.processed_maps_details[map_key_index]['original_resolution_name'] = f"{orig_w}x{orig_h}"
# Determine the correct dimensions to use for 'processed_resolution_name'
# This name refers to the base POT scaled image dimensions before variant generation.
@ -619,21 +680,21 @@ class IndividualMapProcessingStage(ProcessingStage):
if dims_to_log_as_base_processed:
proc_w, proc_h = dims_to_log_as_base_processed
resolution_name_str = f"{proc_w}x{proc_h}"
context.processed_maps_details[map_id_hex]['base_pot_resolution_name'] = resolution_name_str
context.processed_maps_details[map_key_index]['base_pot_resolution_name'] = resolution_name_str
# Ensure 'processed_resolution_name' is also set for OutputOrganizationStage compatibility
context.processed_maps_details[map_id_hex]['processed_resolution_name'] = resolution_name_str
context.processed_maps_details[map_key_index]['processed_resolution_name'] = resolution_name_str
elif 'processed_dimensions' in kwargs or 'base_pot_dimensions' in kwargs:
details_for_warning = kwargs.get('processed_dimensions', kwargs.get('base_pot_dimensions'))
logger.warning(f"Asset '{asset_name_for_log}', Map ID {map_id_hex}: 'processed_dimensions' or 'base_pot_dimensions' key present but its value is not a valid 2-element tuple: {details_for_warning}")
logger.warning(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: 'processed_dimensions' or 'base_pot_dimensions' key present but its value is not a valid 2-element tuple: {details_for_warning}")
# If temp_processed_file was passed, ensure it's in the details
if 'temp_processed_file' in kwargs:
context.processed_maps_details[map_id_hex]['temp_processed_file'] = kwargs['temp_processed_file']
context.processed_maps_details[map_key_index]['temp_processed_file'] = kwargs['temp_processed_file']
# Log all details being stored for clarity, including the newly added resolution names
log_details = context.processed_maps_details[map_id_hex].copy()
log_details = context.processed_maps_details[map_key_index].copy()
# Avoid logging full image data if it accidentally gets into kwargs
if 'image_data' in log_details: del log_details['image_data']
if 'base_pot_image_data' in log_details: del log_details['base_pot_image_data']
logger.debug(f"Asset '{asset_name_for_log}', Map ID {map_id_hex}: Status updated to '{status}'. Details: {log_details}")
logger.debug(f"Asset '{asset_name_for_log}', Map Key Index {map_key_index}: Status updated to '{status}'. Details: {log_details}")

231
processing/pipeline/stages/map_merging.py
View File

@ -125,72 +125,120 @@ class MapMergingStage(ProcessingStage):
required_input_map_types = set(inputs_map_type_to_channel.values())
for required_map_type in required_input_map_types:
found_processed_map = None
processed_map_key = None
for p_key, p_details in context.processed_maps_details.items():
processed_map_type_in_details = p_details.get('map_type')
# Check for direct match or match with "MAP_" prefix
if (processed_map_type_in_details == required_map_type or \
processed_map_type_in_details == f"MAP_{required_map_type}") and \
p_details.get('status') == 'Processed':
found_processed_map = p_details
processed_map_key = p_key # The UUID hex key from individual processing
found_processed_map_details = None
# The key `p_key_idx` is the file_rule_idx from the IndividualMapProcessingStage
for p_key_idx, p_details in context.processed_maps_details.items(): # p_key_idx is an int
processed_map_identifier = p_details.get('processing_map_type', p_details.get('map_type'))
# Comprehensive list of valid statuses for an input map to be used in merging
valid_input_statuses = ['BasePOTSaved', 'Processed_With_Variants', 'Processed_No_Variants', 'Converted_To_Rough']
is_match = False
if processed_map_identifier == required_map_type:
is_match = True
elif required_map_type.startswith("MAP_") and processed_map_identifier == required_map_type.split("MAP_")[-1]:
is_match = True
elif not required_map_type.startswith("MAP_") and processed_map_identifier == f"MAP_{required_map_type}":
is_match = True
if is_match and p_details.get('status') in valid_input_statuses:
found_processed_map_details = p_details
# The key `p_key_idx` (which is the FileRule index) is implicitly associated with these details.
break
if not found_processed_map:
logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Required input map_type '{required_map_type}' for output '{output_map_type}' not found or not processed in processed_maps_details.")
# Option: Use default value for the entire map if one could be constructed for this map_type
# For now, we fail the merge if a required map is missing.
all_inputs_valid = False
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Required input map_type '{required_map_type}' missing."}
break # Break from finding inputs for this merge rule
if not found_processed_map_details:
can_be_fully_defaulted = True
channels_requiring_this_map = [
ch_key for ch_key, map_type_val in inputs_map_type_to_channel.items()
if map_type_val == required_map_type
]
temp_file_path = Path(found_processed_map['temp_processed_file'])
if not temp_file_path.exists():
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Temp file {temp_file_path} for input map_type '{required_map_type}' does not exist.")
all_inputs_valid = False
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Temp file for input '{required_map_type}' missing."}
break
try:
image_data = ipu.load_image(temp_file_path)
if image_data is None: raise ValueError("Loaded image is None")
except Exception as e:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error loading image {temp_file_path} for input map_type '{required_map_type}': {e}")
all_inputs_valid = False
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Error loading input '{required_map_type}'."}
break
loaded_input_maps[required_map_type] = image_data
input_map_paths[required_map_type] = str(temp_file_path)
current_dims = (image_data.shape[1], image_data.shape[0])
if target_dims is None:
target_dims = current_dims
elif current_dims != target_dims:
logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{required_map_type}' dims {current_dims} differ from target {target_dims}. Resizing.")
try:
image_data = ipu.resize_image(image_data, target_dims[0], target_dims[1])
if image_data is None: raise ValueError("Resize returned None")
loaded_input_maps[required_map_type] = image_data
except Exception as e:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to resize '{required_map_type}': {e}")
if not channels_requiring_this_map:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Internal logic error. Required map_type '{required_map_type}' is not actually used by any output channel. Configuration: {inputs_map_type_to_channel}")
all_inputs_valid = False
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Failed to resize input '{required_map_type}'."}
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Internal error: required map_type '{required_map_type}' not in use."}
break
for channel_char_needing_default in channels_requiring_this_map:
if default_values.get(channel_char_needing_default) is None:
can_be_fully_defaulted = False
break
if can_be_fully_defaulted:
logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Required input map_type '{required_map_type}' for output '{output_map_type}' not found or not in usable state. Will attempt to use default values for its channels: {channels_requiring_this_map}.")
else:
logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Required input map_type '{required_map_type}' for output '{output_map_type}' not found/unusable, AND not all its required channels ({channels_requiring_this_map}) have defaults. Failing merge op.")
all_inputs_valid = False
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Input '{required_map_type}' missing and defaults incomplete."}
break
if found_processed_map_details:
temp_file_path_str = found_processed_map_details.get('temp_processed_file')
if not temp_file_path_str:
# Log with p_key_idx if available, or just the map type if not (though it should be if found_processed_map_details is set)
log_key_info = f"(Associated Key Index: {p_key_idx})" if 'p_key_idx' in locals() and found_processed_map_details else "" # Use locals() to check if p_key_idx is defined in this scope
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: 'temp_processed_file' missing in details for found map_type '{required_map_type}' {log_key_info}.")
all_inputs_valid = False
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Temp file path missing for input '{required_map_type}'."}
break
temp_file_path = Path(temp_file_path_str)
if not temp_file_path.exists():
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Temp file {temp_file_path} for input map_type '{required_map_type}' does not exist.")
all_inputs_valid = False
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Temp file for input '{required_map_type}' missing."}
break
try:
image_data = ipu.load_image(str(temp_file_path))
if image_data is None: raise ValueError("Loaded image is None")
except Exception as e:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error loading image {temp_file_path} for input map_type '{required_map_type}': {e}")
all_inputs_valid = False
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Error loading input '{required_map_type}'."}
break
loaded_input_maps[required_map_type] = image_data
input_map_paths[required_map_type] = str(temp_file_path)
current_dims = (image_data.shape[1], image_data.shape[0])
if target_dims is None:
target_dims = current_dims
elif current_dims != target_dims:
logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{required_map_type}' dims {current_dims} differ from target {target_dims}. Resizing.")
try:
image_data_resized = ipu.resize_image(image_data, target_dims[0], target_dims[1])
if image_data_resized is None: raise ValueError("Resize returned None")
loaded_input_maps[required_map_type] = image_data_resized
except Exception as e:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to resize '{required_map_type}': {e}")
all_inputs_valid = False
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f"Failed to resize input '{required_map_type}'."}
break
if not all_inputs_valid:
logger.warning(f"Asset {asset_name_for_log}: Skipping merge for Op ID {merge_op_id} ('{output_map_type}') due to invalid inputs.")
continue
if not loaded_input_maps or target_dims is None:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: No input maps loaded or target_dims not set for '{output_map_type}'. This shouldn't happen if all_inputs_valid was true.")
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'Internal error: input maps not loaded or target_dims missing.'}
continue
if not loaded_input_maps and not any(default_values.get(ch) is not None for ch in inputs_map_type_to_channel.keys()):
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: No input maps loaded and no defaults available for any channel for '{output_map_type}'. Cannot proceed.")
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'No input maps loaded and no defaults available.'}
continue
# Determine output channels (e.g., 3 for RGB, 1 for Grayscale)
# This depends on the keys in inputs_map_type_to_channel (R,G,B,A)
output_channel_keys = sorted(list(inputs_map_type_to_channel.keys())) # e.g. ['B', 'G', 'R']
if target_dims is None:
default_res_key = context.config_obj.get("default_output_resolution_key_for_merge", "1K")
image_resolutions_cfg = getattr(context.config_obj, "image_resolutions", {})
default_max_dim = image_resolutions_cfg.get(default_res_key)
if default_max_dim:
target_dims = (default_max_dim, default_max_dim)
logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Target dimensions not set by inputs (all defaulted). Using configured default resolution '{default_res_key}': {target_dims}.")
else:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Target dimensions could not be determined for '{output_map_type}' (all inputs defaulted and no default output resolution configured).")
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'Target dimensions undetermined for fully defaulted merge.'}
continue
output_channel_keys = sorted(list(inputs_map_type_to_channel.keys()))
num_output_channels = len(output_channel_keys)
if num_output_channels == 0:
@ -199,79 +247,86 @@ class MapMergingStage(ProcessingStage):
continue
try:
if num_output_channels == 1: # Grayscale output
merged_image = np.zeros((target_dims[1], target_dims[0]), dtype=np.uint8)
else: # Color output
merged_image = np.zeros((target_dims[1], target_dims[0], num_output_channels), dtype=np.uint8)
output_dtype = np.uint8
if num_output_channels == 1:
merged_image = np.zeros((target_dims[1], target_dims[0]), dtype=output_dtype)
else:
merged_image = np.zeros((target_dims[1], target_dims[0], num_output_channels), dtype=output_dtype)
except Exception as e:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error creating empty merged image for '{output_map_type}': {e}")
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': f'Error creating output canvas: {e}'}
continue
merge_op_failed_detail = False
for i, out_channel_char in enumerate(output_channel_keys): # e.g. R, G, B
for i, out_channel_char in enumerate(output_channel_keys):
input_map_type_for_this_channel = inputs_map_type_to_channel[out_channel_char]
source_image = loaded_input_maps.get(input_map_type_for_this_channel)
source_data_this_channel = None
if source_image is not None:
if source_image.ndim == 2: # Grayscale source
if source_image.dtype != np.uint8:
logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{input_map_type_for_this_channel}' has dtype {source_image.dtype}, expected uint8. Attempting conversion.")
source_image = ipu.convert_to_uint8(source_image)
if source_image is None:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to convert input '{input_map_type_for_this_channel}' to uint8.")
merge_op_failed_detail = True; break
if source_image.ndim == 2:
source_data_this_channel = source_image
elif source_image.ndim == 3 or source_image.ndim == 4: # Color source (3-channel BGR or 4-channel BGRA), assumed loaded by ipu.load_image
# Standard BGR(A) channel indexing: B=0, G=1, R=2, A=3 (if present)
# This map helps get NRM's Red data for 'R' output, NRM's Green for 'G' output etc.
# based on the semantic meaning of out_channel_char.
elif source_image.ndim == 3:
semantic_to_bgr_idx = {'R': 2, 'G': 1, 'B': 0, 'A': 3}
if input_map_type_for_this_channel == "NRM":
idx_to_extract = semantic_to_bgr_idx.get(out_channel_char)
idx_to_extract = semantic_to_bgr_idx.get(out_channel_char.upper())
if idx_to_extract is not None and idx_to_extract < source_image.shape[2]:
source_data_this_channel = source_image[:, :, idx_to_extract]
logger.debug(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: For output '{out_channel_char}', using NRM's semantic '{out_channel_char}' channel (BGR(A) index {idx_to_extract}).")
else:
# Fallback if out_channel_char isn't R,G,B,A or NRM doesn't have the channel (e.g. 3-channel NRM and 'A' requested)
logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Could not map output '{out_channel_char}' to a specific BGR(A) channel of NRM (shape {source_image.shape}). Defaulting to NRM's channel 0 (Blue).")
source_data_this_channel = source_image[:, :, 0]
if idx_to_extract is not None and idx_to_extract < source_image.shape[2]:
source_data_this_channel = source_image[:, :, idx_to_extract]
logger.debug(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: For output '{out_channel_char}', using source '{input_map_type_for_this_channel}' semantic '{out_channel_char}' (BGR(A) index {idx_to_extract}).")
else:
# For other multi-channel sources (e.g., ROUGH as RGB, or other color maps not "NRM")
# Default to taking the first channel (Blue in BGR).
# This covers "Roughness map's greyscale data" if ROUGH is RGB (by taking one of its channels as a proxy).
logger.warning(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Could not map output '{out_channel_char}' to a specific BGR(A) channel of '{input_map_type_for_this_channel}' (shape {source_image.shape}). Defaulting to its channel 0 (Blue).")
source_data_this_channel = source_image[:, :, 0]
logger.debug(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: For output '{out_channel_char}', source {input_map_type_for_this_channel} (shape {source_image.shape}) is multi-channel but not NRM. Using its channel 0 (Blue).")
else: # Source map was not found, use default
else:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Source image '{input_map_type_for_this_channel}' has unexpected dimensions: {source_image.ndim} (shape {source_image.shape}).")
merge_op_failed_detail = True; break
else:
default_val_for_channel = default_values.get(out_channel_char)
if default_val_for_channel is not None:
# Convert 0-1 float default to 0-255 uint8
source_data_this_channel = np.full((target_dims[1], target_dims[0]), int(default_val_for_channel * 255), dtype=np.uint8)
logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Using default value {default_val_for_channel} for output channel '{out_channel_char}' as input map '{input_map_type_for_this_channel}' was missing.")
try:
scaled_default_val = int(float(default_val_for_channel) * 255)
source_data_this_channel = np.full((target_dims[1], target_dims[0]), scaled_default_val, dtype=np.uint8)
logger.info(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Using default value {default_val_for_channel} (scaled to {scaled_default_val}) for output channel '{out_channel_char}' as input map '{input_map_type_for_this_channel}' was missing.")
except ValueError:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Default value '{default_val_for_channel}' for channel '{out_channel_char}' is not a valid float. Cannot scale.")
merge_op_failed_detail = True; break
else:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Input map '{input_map_type_for_this_channel}' for output channel '{out_channel_char}' is missing and no default value provided.")
merge_op_failed_detail = True; break
if source_data_this_channel is None: # Should be caught by default value logic or earlier checks
if source_data_this_channel is None:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Failed to get source data for output channel '{out_channel_char}'.")
merge_op_failed_detail = True; break
try:
if merged_image.ndim == 2: # Single channel output
if merged_image.ndim == 2:
merged_image = source_data_this_channel
else: # Multi-channel output
else:
merged_image[:, :, i] = source_data_this_channel
except Exception as e:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error assigning data to output channel '{out_channel_char}' (index {i}): {e}")
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error assigning data to output channel '{out_channel_char}' (index {i}): {e}. Merged shape: {merged_image.shape}, Source data shape: {source_data_this_channel.shape}")
merge_op_failed_detail = True; break
if merge_op_failed_detail:
context.merged_maps_details[merge_op_id] = {'map_type': output_map_type, 'status': 'Failed', 'reason': 'Error during channel assignment.'}
continue
output_format = 'png' # Default, can be configured per rule later
output_format = 'png'
temp_merged_filename = f"merged_{sanitize_filename(output_map_type)}_{merge_op_id}.{output_format}"
temp_merged_path = context.engine_temp_dir / temp_merged_filename
try:
save_success = ipu.save_image(temp_merged_path, merged_image)
save_success = ipu.save_image(str(temp_merged_path), merged_image)
if not save_success: raise ValueError("Save image returned false")
except Exception as e:
logger.error(f"Asset {asset_name_for_log}, Merge Op ID {merge_op_id}: Error saving merged image {temp_merged_path}: {e}")

51
processing/pipeline/stages/metadata_finalization_save.py
View File

@ -66,7 +66,56 @@ class MetadataFinalizationAndSaveStage(ProcessingStage):
context.asset_metadata['status'] = "Processed"
# Add details of processed and merged maps
context.asset_metadata['processed_map_details'] = getattr(context, 'processed_maps_details', {})
# Restructure processed_map_details before assigning
restructured_processed_maps = {}
# getattr(context, 'processed_maps_details', {}) is the source (plural 'maps')
original_processed_maps = getattr(context, 'processed_maps_details', {})
# Define keys to remove at the top level of each map entry
map_keys_to_remove = [
"status", "source_file_path", "temp_processed_file", # Assuming "source_file_path" is the correct key
"original_resolution_name", "base_pot_resolution_name", "processed_resolution_name"
]
# Define keys to remove from each variant
variant_keys_to_remove = ["temp_path", "dimensions"]
for map_key, map_detail_original in original_processed_maps.items():
# Create a new dictionary for the modified map entry
new_map_entry = {}
for key, value in map_detail_original.items():
if key not in map_keys_to_remove:
new_map_entry[key] = value
if "variants" in map_detail_original and isinstance(map_detail_original["variants"], dict):
new_variants_dict = {}
for variant_name, variant_data_original in map_detail_original["variants"].items():
new_variant_entry = {}
for key, value in variant_data_original.items():
if key not in variant_keys_to_remove:
new_variant_entry[key] = value
# Add 'path_to_file'
# This path is expected to be set by OutputOrganizationStage in the context.
# It should be a Path object representing the path relative to the metadata directory,
# or an absolute Path that make_serializable can convert.
# Using 'final_output_path_for_metadata' as the key from context.
if 'final_output_path_for_metadata' in variant_data_original:
new_variant_entry['path_to_file'] = variant_data_original['final_output_path_for_metadata']
else:
# Log a warning if the expected path is not found
logger.warning(
f"Asset '{asset_name_for_log}': 'final_output_path_for_metadata' "
f"missing for variant '{variant_name}' in map '{map_key}'. "
f"Metadata will be incomplete for this variant's path."
)
new_variant_entry['path_to_file'] = "ERROR_PATH_NOT_FOUND" # Placeholder
new_variants_dict[variant_name] = new_variant_entry
new_map_entry["variants"] = new_variants_dict
restructured_processed_maps[map_key] = new_map_entry
# Assign the restructured details. Note: 'processed_map_details' (singular 'map') is the key in asset_metadata.
context.asset_metadata['processed_map_details'] = restructured_processed_maps
context.asset_metadata['merged_map_details'] = getattr(context, 'merged_maps_details', {})
# (Optional) Add a list of all temporary files

10
processing/pipeline/stages/output_organization.py
View File

@ -211,10 +211,12 @@ class OutputOrganizationStage(ProcessingStage):
final_output_files.append(str(final_variant_path))
variant_detail['status'] = 'Organized'
variant_detail['final_output_path'] = str(final_variant_path)
relative_final_variant_path_str = str(Path(relative_dir_path_str_variant) / Path(output_filename_variant))
map_metadata_entry['variant_paths'][variant_resolution_key] = relative_final_variant_path_str
processed_any_variant_successfully = True
variant_detail['final_output_path'] = str(final_variant_path)
# Store the Path object for metadata stage to make it relative later
variant_detail['final_output_path_for_metadata'] = final_variant_path
relative_final_variant_path_str = str(Path(relative_dir_path_str_variant) / Path(output_filename_variant))
map_metadata_entry['variant_paths'][variant_resolution_key] = relative_final_variant_path_str
processed_any_variant_successfully = True
except Exception as e:
logger.error(f"Asset '{asset_name_for_log}': Failed to copy variant {temp_variant_path} for map key '{processed_map_key}' (res: {variant_resolution_key}). Error: {e}", exc_info=True)