Uncompleted Processing Refactor

This commit is contained in:
2025-05-09 11:32:16 +02:00
parent d473ddd7f4
commit 12cf557dd7
38 changed files with 7472 additions and 1536 deletions

View File

@@ -0,0 +1 @@
# This file makes Python treat the directory as a package.

View File

@@ -0,0 +1 @@
# This file makes Python treat the directory as a package.

View File

@@ -0,0 +1,273 @@
import pytest
from unittest import mock
from pathlib import Path
import uuid
import numpy as np
from processing.pipeline.stages.alpha_extraction_to_mask import AlphaExtractionToMaskStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule, FileRule, TransformSettings
from configuration import Configuration, GeneralSettings
import processing.utils.image_processing_utils as ipu # Ensure ipu is available for mocking
# Helper Functions
def create_mock_file_rule_for_alpha_test(
id_val: uuid.UUID = None,
map_type: str = "ALBEDO",
filename_pattern: str = "albedo.png",
item_type: str = "MAP_COL",
active: bool = True
) -> mock.MagicMock:
mock_fr = mock.MagicMock(spec=FileRule)
mock_fr.id = id_val if id_val else uuid.uuid4()
mock_fr.map_type = map_type
mock_fr.filename_pattern = filename_pattern
mock_fr.item_type = item_type
mock_fr.active = active
mock_fr.transform_settings = mock.MagicMock(spec=TransformSettings)
return mock_fr
def create_alpha_extraction_mock_context(
initial_file_rules: list = None,
initial_processed_details: dict = None,
skip_asset_flag: bool = False,
asset_name: str = "AlphaAsset",
# extract_alpha_globally: bool = True # If stage checks this
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_gs = mock.MagicMock(spec=GeneralSettings)
# if your stage uses a global flag:
# mock_gs.extract_alpha_to_mask_globally = extract_alpha_globally
mock_config = mock.MagicMock(spec=Configuration)
mock_config.general_settings = mock_gs
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp_engine_dir"),
output_base_path=Path("/fake/output"),
effective_supplier="ValidSupplier",
asset_metadata={'asset_name': asset_name},
processed_maps_details=initial_processed_details if initial_processed_details is not None else {},
merged_maps_details={},
files_to_process=list(initial_file_rules) if initial_file_rules else [],
loaded_data_cache={},
config_obj=mock_config,
status_flags={'skip_asset': skip_asset_flag},
incrementing_value=None,
sha5_value=None
)
return context
# Unit Tests
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image')
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image')
@mock.patch('logging.info') # Mock logging to avoid console output during tests
def test_asset_skipped(mock_log_info, mock_load_image, mock_save_image):
stage = AlphaExtractionToMaskStage()
context = create_alpha_extraction_mock_context(skip_asset_flag=True)
updated_context = stage.execute(context)
assert updated_context == context # Context should be unchanged
mock_load_image.assert_not_called()
mock_save_image.assert_not_called()
assert len(updated_context.files_to_process) == 0
assert not updated_context.processed_maps_details
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image')
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image')
@mock.patch('logging.info')
def test_existing_mask_map(mock_log_info, mock_load_image, mock_save_image):
stage = AlphaExtractionToMaskStage()
existing_mask_rule = create_mock_file_rule_for_alpha_test(map_type="MASK", filename_pattern="mask.png")
context = create_alpha_extraction_mock_context(initial_file_rules=[existing_mask_rule])
updated_context = stage.execute(context)
assert updated_context == context
mock_load_image.assert_not_called()
mock_save_image.assert_not_called()
assert len(updated_context.files_to_process) == 1
assert updated_context.files_to_process[0].map_type == "MASK"
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image')
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image')
@mock.patch('logging.info')
def test_alpha_extraction_success(mock_log_info, mock_load_image, mock_save_image):
stage = AlphaExtractionToMaskStage()
albedo_rule_id = uuid.uuid4()
albedo_fr = create_mock_file_rule_for_alpha_test(id_val=albedo_rule_id, map_type="ALBEDO")
initial_processed_details = {
albedo_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_albedo.png', 'status': 'Processed', 'map_type': 'ALBEDO', 'source_file_path': Path('/fake/source/albedo.png')}
}
context = create_alpha_extraction_mock_context(
initial_file_rules=[albedo_fr],
initial_processed_details=initial_processed_details
)
mock_rgba_data = np.zeros((10, 10, 4), dtype=np.uint8)
mock_rgba_data[:, :, 3] = 128 # Example alpha data
mock_load_image.side_effect = [mock_rgba_data, mock_rgba_data]
mock_save_image.return_value = True
updated_context = stage.execute(context)
assert mock_load_image.call_count == 2
# First call to check for alpha, second to get data for saving
mock_load_image.assert_any_call(Path('/fake/temp_engine_dir/processed_albedo.png'))
mock_save_image.assert_called_once()
saved_path_arg = mock_save_image.call_args[0][0]
saved_data_arg = mock_save_image.call_args[0][1]
assert isinstance(saved_path_arg, Path)
assert "mask_from_alpha_" in saved_path_arg.name
assert np.array_equal(saved_data_arg, mock_rgba_data[:, :, 3])
assert len(updated_context.files_to_process) == 2
new_mask_rule = None
for fr in updated_context.files_to_process:
if fr.map_type == "MASK":
new_mask_rule = fr
break
assert new_mask_rule is not None
assert new_mask_rule.item_type == "MAP_DER" # Derived map
assert new_mask_rule.id.hex in updated_context.processed_maps_details
new_mask_detail = updated_context.processed_maps_details[new_mask_rule.id.hex]
assert new_mask_detail['map_type'] == "MASK"
assert "mask_from_alpha_" in new_mask_detail['temp_processed_file']
assert "Generated from alpha of ALBEDO" in new_mask_detail['notes'] # Check for specific note
assert new_mask_detail['status'] == 'Processed'
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image')
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image')
@mock.patch('logging.info')
def test_no_alpha_channel_in_source(mock_log_info, mock_load_image, mock_save_image):
stage = AlphaExtractionToMaskStage()
albedo_rule_id = uuid.uuid4()
albedo_fr = create_mock_file_rule_for_alpha_test(id_val=albedo_rule_id, map_type="ALBEDO")
initial_processed_details = {
albedo_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_rgb_albedo.png', 'status': 'Processed', 'map_type': 'ALBEDO', 'source_file_path': Path('/fake/source/albedo_rgb.png')}
}
context = create_alpha_extraction_mock_context(
initial_file_rules=[albedo_fr],
initial_processed_details=initial_processed_details
)
mock_rgb_data = np.zeros((10, 10, 3), dtype=np.uint8) # RGB, no alpha
mock_load_image.return_value = mock_rgb_data # Only called once for check
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(Path('/fake/temp_engine_dir/processed_rgb_albedo.png'))
mock_save_image.assert_not_called()
assert len(updated_context.files_to_process) == 1 # No new MASK rule
assert albedo_fr.id.hex in updated_context.processed_maps_details
assert len(updated_context.processed_maps_details) == 1
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image')
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image')
@mock.patch('logging.info')
def test_no_suitable_source_map_type(mock_log_info, mock_load_image, mock_save_image):
stage = AlphaExtractionToMaskStage()
normal_rule_id = uuid.uuid4()
normal_fr = create_mock_file_rule_for_alpha_test(id_val=normal_rule_id, map_type="NORMAL")
initial_processed_details = {
normal_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_normal.png', 'status': 'Processed', 'map_type': 'NORMAL'}
}
context = create_alpha_extraction_mock_context(
initial_file_rules=[normal_fr],
initial_processed_details=initial_processed_details
)
updated_context = stage.execute(context)
mock_load_image.assert_not_called()
mock_save_image.assert_not_called()
assert len(updated_context.files_to_process) == 1
assert normal_fr.id.hex in updated_context.processed_maps_details
assert len(updated_context.processed_maps_details) == 1
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image')
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image')
@mock.patch('logging.warning') # Expect a warning log
def test_load_image_fails(mock_log_warning, mock_load_image, mock_save_image):
stage = AlphaExtractionToMaskStage()
albedo_rule_id = uuid.uuid4()
albedo_fr = create_mock_file_rule_for_alpha_test(id_val=albedo_rule_id, map_type="ALBEDO")
initial_processed_details = {
albedo_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_albedo_load_fail.png', 'status': 'Processed', 'map_type': 'ALBEDO', 'source_file_path': Path('/fake/source/albedo_load_fail.png')}
}
context = create_alpha_extraction_mock_context(
initial_file_rules=[albedo_fr],
initial_processed_details=initial_processed_details
)
mock_load_image.return_value = None # Simulate load failure
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(Path('/fake/temp_engine_dir/processed_albedo_load_fail.png'))
mock_save_image.assert_not_called()
assert len(updated_context.files_to_process) == 1
assert albedo_fr.id.hex in updated_context.processed_maps_details
assert len(updated_context.processed_maps_details) == 1
mock_log_warning.assert_called_once() # Check that a warning was logged
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.save_image')
@mock.patch('processing.pipeline.stages.alpha_extraction_to_mask.ipu.load_image')
@mock.patch('logging.error') # Expect an error log
def test_save_image_fails(mock_log_error, mock_load_image, mock_save_image):
stage = AlphaExtractionToMaskStage()
albedo_rule_id = uuid.uuid4()
albedo_fr = create_mock_file_rule_for_alpha_test(id_val=albedo_rule_id, map_type="ALBEDO")
initial_processed_details = {
albedo_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_albedo_save_fail.png', 'status': 'Processed', 'map_type': 'ALBEDO', 'source_file_path': Path('/fake/source/albedo_save_fail.png')}
}
context = create_alpha_extraction_mock_context(
initial_file_rules=[albedo_fr],
initial_processed_details=initial_processed_details
)
mock_rgba_data = np.zeros((10, 10, 4), dtype=np.uint8)
mock_rgba_data[:, :, 3] = 128
mock_load_image.side_effect = [mock_rgba_data, mock_rgba_data] # Load succeeds
mock_save_image.return_value = False # Simulate save failure
updated_context = stage.execute(context)
assert mock_load_image.call_count == 2
mock_save_image.assert_called_once() # Save was attempted
assert len(updated_context.files_to_process) == 1 # No new MASK rule should be successfully added and detailed
# Check that no new MASK details were added, or if they were, they reflect failure.
# The current stage logic returns context early, so no new rule or details should be present.
mask_rule_found = any(fr.map_type == "MASK" for fr in updated_context.files_to_process)
assert not mask_rule_found
mask_details_found = any(
details['map_type'] == "MASK"
for fr_id, details in updated_context.processed_maps_details.items()
if fr_id != albedo_fr.id.hex # Exclude the original albedo
)
assert not mask_details_found
mock_log_error.assert_called_once() # Check that an error was logged

View File

@@ -0,0 +1,213 @@
import pytest
from unittest import mock
from pathlib import Path
from typing import Dict, Optional, Any
from processing.pipeline.stages.asset_skip_logic import AssetSkipLogicStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule
from configuration import Configuration, GeneralSettings
# Helper function to create a mock AssetProcessingContext
def create_skip_logic_mock_context(
effective_supplier: Optional[str] = "ValidSupplier",
asset_process_status: str = "PENDING",
overwrite_existing: bool = False,
asset_name: str = "TestAssetSkip"
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_asset_rule.process_status = asset_process_status
mock_asset_rule.source_path = "fake/source" # Added for completeness
mock_asset_rule.output_path = "fake/output" # Added for completeness
mock_asset_rule.maps = [] # Added for completeness
mock_asset_rule.metadata = {} # Added for completeness
mock_asset_rule.material_name = None # Added for completeness
mock_asset_rule.notes = None # Added for completeness
mock_asset_rule.tags = [] # Added for completeness
mock_asset_rule.enabled = True # Added for completeness
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_source_rule.name = "TestSourceRule" # Added for completeness
mock_source_rule.path = "fake/source_rule_path" # Added for completeness
mock_source_rule.default_supplier = None # Added for completeness
mock_source_rule.assets = [mock_asset_rule] # Added for completeness
mock_source_rule.enabled = True # Added for completeness
mock_general_settings = mock.MagicMock(spec=GeneralSettings)
mock_general_settings.overwrite_existing = overwrite_existing
mock_config = mock.MagicMock(spec=Configuration)
mock_config.general_settings = mock_general_settings
mock_config.suppliers = {"ValidSupplier": mock.MagicMock()}
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp"),
output_base_path=Path("/fake/output"),
effective_supplier=effective_supplier,
asset_metadata={},
processed_maps_details={},
merged_maps_details={},
files_to_process=[],
loaded_data_cache={},
config_obj=mock_config,
status_flags={},
incrementing_value=None,
sha5_value=None # Corrected from sha5_value to sha256_value if that's the actual field
)
# Ensure status_flags is initialized if AssetSkipLogicStage expects it
# context.status_flags = {} # Already done in constructor
return context
@mock.patch('logging.info')
def test_skip_due_to_missing_supplier(mock_log_info):
"""
Test that the asset is skipped if effective_supplier is None.
"""
stage = AssetSkipLogicStage()
context = create_skip_logic_mock_context(effective_supplier=None, asset_name="MissingSupplierAsset")
updated_context = stage.execute(context)
assert updated_context.status_flags.get('skip_asset') is True
assert updated_context.status_flags.get('skip_reason') == "Invalid or missing supplier"
mock_log_info.assert_any_call(f"Asset 'MissingSupplierAsset': Skipping due to missing or invalid supplier.")
@mock.patch('logging.info')
def test_skip_due_to_process_status_skip(mock_log_info):
"""
Test that the asset is skipped if asset_rule.process_status is "SKIP".
"""
stage = AssetSkipLogicStage()
context = create_skip_logic_mock_context(asset_process_status="SKIP", asset_name="SkipStatusAsset")
updated_context = stage.execute(context)
assert updated_context.status_flags.get('skip_asset') is True
assert updated_context.status_flags.get('skip_reason') == "Process status set to SKIP"
mock_log_info.assert_any_call(f"Asset 'SkipStatusAsset': Skipping because process_status is 'SKIP'.")
@mock.patch('logging.info')
def test_skip_due_to_processed_and_overwrite_disabled(mock_log_info):
"""
Test that the asset is skipped if asset_rule.process_status is "PROCESSED"
and overwrite_existing is False.
"""
stage = AssetSkipLogicStage()
context = create_skip_logic_mock_context(
asset_process_status="PROCESSED",
overwrite_existing=False,
asset_name="ProcessedNoOverwriteAsset"
)
updated_context = stage.execute(context)
assert updated_context.status_flags.get('skip_asset') is True
assert updated_context.status_flags.get('skip_reason') == "Already processed, overwrite disabled"
mock_log_info.assert_any_call(f"Asset 'ProcessedNoOverwriteAsset': Skipping because already processed and overwrite is disabled.")
@mock.patch('logging.info')
def test_no_skip_when_processed_and_overwrite_enabled(mock_log_info):
"""
Test that the asset is NOT skipped if asset_rule.process_status is "PROCESSED"
but overwrite_existing is True.
"""
stage = AssetSkipLogicStage()
context = create_skip_logic_mock_context(
asset_process_status="PROCESSED",
overwrite_existing=True,
effective_supplier="ValidSupplier", # Ensure supplier is valid
asset_name="ProcessedOverwriteAsset"
)
updated_context = stage.execute(context)
assert updated_context.status_flags.get('skip_asset', False) is False # Default to False if key not present
# No specific skip_reason to check if not skipped
# Check that no skip log message was called for this specific reason
for call_args in mock_log_info.call_args_list:
assert "Skipping because already processed and overwrite is disabled" not in call_args[0][0]
assert "Skipping due to missing or invalid supplier" not in call_args[0][0]
assert "Skipping because process_status is 'SKIP'" not in call_args[0][0]
@mock.patch('logging.info')
def test_no_skip_when_process_status_pending(mock_log_info):
"""
Test that the asset is NOT skipped if asset_rule.process_status is "PENDING".
"""
stage = AssetSkipLogicStage()
context = create_skip_logic_mock_context(
asset_process_status="PENDING",
effective_supplier="ValidSupplier", # Ensure supplier is valid
asset_name="PendingAsset"
)
updated_context = stage.execute(context)
assert updated_context.status_flags.get('skip_asset', False) is False
# Check that no skip log message was called
for call_args in mock_log_info.call_args_list:
assert "Skipping" not in call_args[0][0]
@mock.patch('logging.info')
def test_no_skip_when_process_status_failed_previously(mock_log_info):
"""
Test that the asset is NOT skipped if asset_rule.process_status is "FAILED_PREVIOUSLY".
"""
stage = AssetSkipLogicStage()
context = create_skip_logic_mock_context(
asset_process_status="FAILED_PREVIOUSLY",
effective_supplier="ValidSupplier", # Ensure supplier is valid
asset_name="FailedPreviouslyAsset"
)
updated_context = stage.execute(context)
assert updated_context.status_flags.get('skip_asset', False) is False
# Check that no skip log message was called
for call_args in mock_log_info.call_args_list:
assert "Skipping" not in call_args[0][0]
@mock.patch('logging.info')
def test_no_skip_when_process_status_other_valid_status(mock_log_info):
"""
Test that the asset is NOT skipped for other valid, non-skip process statuses.
"""
stage = AssetSkipLogicStage()
context = create_skip_logic_mock_context(
asset_process_status="READY_FOR_PROCESSING", # Example of another non-skip status
effective_supplier="ValidSupplier",
asset_name="ReadyAsset"
)
updated_context = stage.execute(context)
assert updated_context.status_flags.get('skip_asset', False) is False
for call_args in mock_log_info.call_args_list:
assert "Skipping" not in call_args[0][0]
@mock.patch('logging.info')
def test_skip_asset_flag_initialized_if_not_present(mock_log_info):
"""
Test that 'skip_asset' is initialized to False in status_flags if not skipped and not present.
"""
stage = AssetSkipLogicStage()
context = create_skip_logic_mock_context(
asset_process_status="PENDING",
effective_supplier="ValidSupplier",
asset_name="InitFlagAsset"
)
# Ensure status_flags is empty before execute
context.status_flags = {}
updated_context = stage.execute(context)
# If not skipped, 'skip_asset' should be explicitly False.
assert updated_context.status_flags.get('skip_asset') is False
# No skip reason should be set
assert 'skip_reason' not in updated_context.status_flags
for call_args in mock_log_info.call_args_list:
assert "Skipping" not in call_args[0][0]

View File

@@ -0,0 +1,330 @@
import pytest
from unittest import mock
from pathlib import Path
import uuid
from typing import Optional # Added Optional for type hinting
from processing.pipeline.stages.file_rule_filter import FileRuleFilterStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule, FileRule # FileRule is key here
from configuration import Configuration # Minimal config needed
def create_mock_file_rule(
id_val: Optional[uuid.UUID] = None,
map_type: str = "Diffuse",
filename_pattern: str = "*.tif",
item_type: str = "MAP_COL", # e.g., MAP_COL, FILE_IGNORE
active: bool = True
) -> mock.MagicMock: # Return MagicMock to easily set other attributes if needed
mock_fr = mock.MagicMock(spec=FileRule)
mock_fr.id = id_val if id_val else uuid.uuid4()
mock_fr.map_type = map_type
mock_fr.filename_pattern = filename_pattern
mock_fr.item_type = item_type
mock_fr.active = active
return mock_fr
def create_file_filter_mock_context(
file_rules_list: Optional[list] = None, # List of mock FileRule objects
skip_asset_flag: bool = False,
asset_name: str = "FileFilterAsset"
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_asset_rule.file_rules = file_rules_list if file_rules_list is not None else []
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_config = mock.MagicMock(spec=Configuration)
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp"),
output_base_path=Path("/fake/output"),
effective_supplier="ValidSupplier", # Assume valid for this stage
asset_metadata={'asset_name': asset_name}, # Assume metadata init happened
processed_maps_details={},
merged_maps_details={},
files_to_process=[], # Stage will populate this
loaded_data_cache={},
config_obj=mock_config,
status_flags={'skip_asset': skip_asset_flag},
incrementing_value=None,
sha5_value=None # Corrected from sha5_value to sha256_value based on AssetProcessingContext
)
return context
# Test Cases for FileRuleFilterStage.execute()
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_asset_skipped(mock_log_debug, mock_log_info):
"""
Test case: Asset Skipped - status_flags['skip_asset'] is True.
Assert context.files_to_process remains empty.
"""
stage = FileRuleFilterStage()
context = create_file_filter_mock_context(skip_asset_flag=True)
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 0
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule filtering as 'skip_asset' is True.")
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_no_file_rules(mock_log_debug, mock_log_info):
"""
Test case: No File Rules - asset_rule.file_rules is empty.
Assert context.files_to_process is empty.
"""
stage = FileRuleFilterStage()
context = create_file_filter_mock_context(file_rules_list=[])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 0
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': No file rules defined. Skipping file rule filtering.")
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_only_active_processable_rules(mock_log_debug, mock_log_info):
"""
Test case: Only Active, Processable Rules - All FileRules are active=True and item_type="MAP_COL".
Assert all are added to context.files_to_process.
"""
stage = FileRuleFilterStage()
fr1 = create_mock_file_rule(filename_pattern="diffuse.png", item_type="MAP_COL", active=True)
fr2 = create_mock_file_rule(filename_pattern="normal.png", item_type="MAP_COL", active=True)
context = create_file_filter_mock_context(file_rules_list=[fr1, fr2])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 2
assert fr1 in updated_context.files_to_process
assert fr2 in updated_context.files_to_process
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 2 file rules queued for processing after filtering.")
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_inactive_rules(mock_log_debug, mock_log_info):
"""
Test case: Inactive Rules - Some FileRules have active=False.
Assert only active rules are added.
"""
stage = FileRuleFilterStage()
fr_active = create_mock_file_rule(filename_pattern="active.png", item_type="MAP_COL", active=True)
fr_inactive = create_mock_file_rule(filename_pattern="inactive.png", item_type="MAP_COL", active=False)
fr_another_active = create_mock_file_rule(filename_pattern="another_active.jpg", item_type="MAP_COL", active=True)
context = create_file_filter_mock_context(file_rules_list=[fr_active, fr_inactive, fr_another_active])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 2
assert fr_active in updated_context.files_to_process
assert fr_another_active in updated_context.files_to_process
assert fr_inactive not in updated_context.files_to_process
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping inactive file rule: '{fr_inactive.filename_pattern}'")
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 2 file rules queued for processing after filtering.")
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_file_ignore_simple_match(mock_log_debug, mock_log_info):
"""
Test case: FILE_IGNORE Rule (Simple Match).
One FILE_IGNORE rule with filename_pattern="*_ignore.png".
One MAP_COL rule with filename_pattern="diffuse_ignore.png".
One MAP_COL rule with filename_pattern="normal_process.png".
Assert only "normal_process.png" rule is added.
"""
stage = FileRuleFilterStage()
fr_ignore = create_mock_file_rule(filename_pattern="*_ignore.png", item_type="FILE_IGNORE", active=True)
fr_ignored_map = create_mock_file_rule(filename_pattern="diffuse_ignore.png", item_type="MAP_COL", active=True)
fr_process_map = create_mock_file_rule(filename_pattern="normal_process.png", item_type="MAP_COL", active=True)
context = create_file_filter_mock_context(file_rules_list=[fr_ignore, fr_ignored_map, fr_process_map])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 1
assert fr_process_map in updated_context.files_to_process
assert fr_ignored_map not in updated_context.files_to_process
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore.filename_pattern}'")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_ignored_map.filename_pattern}' due to matching ignore pattern.")
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 1 file rules queued for processing after filtering.")
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_file_ignore_glob_pattern(mock_log_debug, mock_log_info):
"""
Test case: FILE_IGNORE Rule (Glob Pattern).
One FILE_IGNORE rule with filename_pattern="*_ignore.*".
MAP_COL rules: "tex_ignore.tif", "tex_process.png".
Assert only "tex_process.png" rule is added.
"""
stage = FileRuleFilterStage()
fr_ignore_glob = create_mock_file_rule(filename_pattern="*_ignore.*", item_type="FILE_IGNORE", active=True)
fr_ignored_tif = create_mock_file_rule(filename_pattern="tex_ignore.tif", item_type="MAP_COL", active=True)
fr_process_png = create_mock_file_rule(filename_pattern="tex_process.png", item_type="MAP_COL", active=True)
context = create_file_filter_mock_context(file_rules_list=[fr_ignore_glob, fr_ignored_tif, fr_process_png])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 1
assert fr_process_png in updated_context.files_to_process
assert fr_ignored_tif not in updated_context.files_to_process
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore_glob.filename_pattern}'")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_ignored_tif.filename_pattern}' due to matching ignore pattern.")
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 1 file rules queued for processing after filtering.")
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_multiple_file_ignore_rules(mock_log_debug, mock_log_info):
"""
Test case: Multiple FILE_IGNORE Rules.
Test with several ignore patterns and ensure they are all respected.
"""
stage = FileRuleFilterStage()
fr_ignore1 = create_mock_file_rule(filename_pattern="*.tmp", item_type="FILE_IGNORE", active=True)
fr_ignore2 = create_mock_file_rule(filename_pattern="backup_*", item_type="FILE_IGNORE", active=True)
fr_ignore3 = create_mock_file_rule(filename_pattern="*_old.png", item_type="FILE_IGNORE", active=True)
fr_map_ignored1 = create_mock_file_rule(filename_pattern="data.tmp", item_type="MAP_COL", active=True)
fr_map_ignored2 = create_mock_file_rule(filename_pattern="backup_diffuse.jpg", item_type="MAP_COL", active=True)
fr_map_ignored3 = create_mock_file_rule(filename_pattern="normal_old.png", item_type="MAP_COL", active=True)
fr_map_process = create_mock_file_rule(filename_pattern="final_texture.tif", item_type="MAP_COL", active=True)
context = create_file_filter_mock_context(file_rules_list=[
fr_ignore1, fr_ignore2, fr_ignore3,
fr_map_ignored1, fr_map_ignored2, fr_map_ignored3, fr_map_process
])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 1
assert fr_map_process in updated_context.files_to_process
assert fr_map_ignored1 not in updated_context.files_to_process
assert fr_map_ignored2 not in updated_context.files_to_process
assert fr_map_ignored3 not in updated_context.files_to_process
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore1.filename_pattern}'")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore2.filename_pattern}'")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore3.filename_pattern}'")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_map_ignored1.filename_pattern}' due to matching ignore pattern.")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_map_ignored2.filename_pattern}' due to matching ignore pattern.")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_map_ignored3.filename_pattern}' due to matching ignore pattern.")
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 1 file rules queued for processing after filtering.")
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_file_ignore_rule_is_inactive(mock_log_debug, mock_log_info):
"""
Test case: FILE_IGNORE Rule is Inactive.
An ignore rule itself is active=False. Assert its pattern is NOT used for filtering.
"""
stage = FileRuleFilterStage()
fr_inactive_ignore = create_mock_file_rule(filename_pattern="*_ignore.tif", item_type="FILE_IGNORE", active=False)
fr_should_process1 = create_mock_file_rule(filename_pattern="diffuse_ignore.tif", item_type="MAP_COL", active=True) # Should be processed
fr_should_process2 = create_mock_file_rule(filename_pattern="normal_ok.png", item_type="MAP_COL", active=True)
context = create_file_filter_mock_context(file_rules_list=[fr_inactive_ignore, fr_should_process1, fr_should_process2])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 2
assert fr_should_process1 in updated_context.files_to_process
assert fr_should_process2 in updated_context.files_to_process
# Ensure the inactive ignore rule's pattern was not registered
# We check this by ensuring no debug log for registering *that specific* pattern was made.
# A more robust way would be to check mock_log_debug.call_args_list, but this is simpler for now.
for call in mock_log_debug.call_args_list:
args, kwargs = call
if "Registering ignore pattern" in args[0] and fr_inactive_ignore.filename_pattern in args[0]:
pytest.fail(f"Inactive ignore pattern '{fr_inactive_ignore.filename_pattern}' was incorrectly registered.")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping inactive file rule: '{fr_inactive_ignore.filename_pattern}' (type: FILE_IGNORE)")
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 2 file rules queued for processing after filtering.")
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_no_file_ignore_rules(mock_log_debug, mock_log_info):
"""
Test case: No FILE_IGNORE Rules.
All rules are MAP_COL or other processable types.
Assert all active, processable rules are included.
"""
stage = FileRuleFilterStage()
fr1 = create_mock_file_rule(filename_pattern="diffuse.png", item_type="MAP_COL", active=True)
fr2 = create_mock_file_rule(filename_pattern="normal.png", item_type="MAP_COL", active=True)
fr_other_type = create_mock_file_rule(filename_pattern="spec.tif", item_type="MAP_SPEC", active=True) # Assuming MAP_SPEC is processable
fr_inactive = create_mock_file_rule(filename_pattern="ao.jpg", item_type="MAP_AO", active=False)
context = create_file_filter_mock_context(file_rules_list=[fr1, fr2, fr_other_type, fr_inactive])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 3
assert fr1 in updated_context.files_to_process
assert fr2 in updated_context.files_to_process
assert fr_other_type in updated_context.files_to_process
assert fr_inactive not in updated_context.files_to_process
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping inactive file rule: '{fr_inactive.filename_pattern}'")
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 3 file rules queued for processing after filtering.")
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_item_type_not_processable(mock_log_debug, mock_log_info):
"""
Test case: Item type is not processable (e.g., not MAP_COL, MAP_AO etc., but something else like 'METADATA_ONLY').
Assert such rules are not added to files_to_process, unless they are FILE_IGNORE.
"""
stage = FileRuleFilterStage()
fr_processable = create_mock_file_rule(filename_pattern="diffuse.png", item_type="MAP_COL", active=True)
fr_not_processable = create_mock_file_rule(filename_pattern="info.txt", item_type="METADATA_ONLY", active=True)
fr_ignore = create_mock_file_rule(filename_pattern="*.bak", item_type="FILE_IGNORE", active=True)
fr_ignored_by_bak = create_mock_file_rule(filename_pattern="diffuse.bak", item_type="MAP_COL", active=True)
context = create_file_filter_mock_context(file_rules_list=[fr_processable, fr_not_processable, fr_ignore, fr_ignored_by_bak])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 1
assert fr_processable in updated_context.files_to_process
assert fr_not_processable not in updated_context.files_to_process
assert fr_ignored_by_bak not in updated_context.files_to_process
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore.filename_pattern}'")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_not_processable.filename_pattern}' as its item_type '{fr_not_processable.item_type}' is not processable.")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_ignored_by_bak.filename_pattern}' due to matching ignore pattern.")
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 1 file rules queued for processing after filtering.")
# Example tests from instructions (can be adapted or used as a base)
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_basic_active_example(mock_log_debug, mock_log_info): # Renamed to avoid conflict
stage = FileRuleFilterStage()
fr1 = create_mock_file_rule(filename_pattern="diffuse.png", item_type="MAP_COL", active=True)
fr2 = create_mock_file_rule(filename_pattern="normal.png", item_type="MAP_COL", active=True)
context = create_file_filter_mock_context(file_rules_list=[fr1, fr2])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 2
assert fr1 in updated_context.files_to_process
assert fr2 in updated_context.files_to_process
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 2 file rules queued for processing after filtering.")
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_file_rule_filter_with_file_ignore_example(mock_log_debug, mock_log_info): # Renamed to avoid conflict
stage = FileRuleFilterStage()
fr_ignore = create_mock_file_rule(filename_pattern="*_ignore.tif", item_type="FILE_IGNORE", active=True)
fr_process = create_mock_file_rule(filename_pattern="diffuse_ok.tif", item_type="MAP_COL", active=True)
fr_skip = create_mock_file_rule(filename_pattern="normal_ignore.tif", item_type="MAP_COL", active=True)
context = create_file_filter_mock_context(file_rules_list=[fr_ignore, fr_process, fr_skip])
updated_context = stage.execute(context)
assert len(updated_context.files_to_process) == 1
assert fr_process in updated_context.files_to_process
assert fr_skip not in updated_context.files_to_process
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Registering ignore pattern: '{fr_ignore.filename_pattern}'")
mock_log_debug.assert_any_call(f"Asset '{context.asset_rule.name}': Skipping file rule '{fr_skip.filename_pattern}' due to matching ignore pattern.")
mock_log_info.assert_any_call(f"Asset '{context.asset_rule.name}': 1 file rules queued for processing after filtering.")

View File

@@ -0,0 +1,486 @@
import pytest
from unittest import mock
from pathlib import Path
import uuid
import numpy as np
from typing import Optional, List, Dict
from processing.pipeline.stages.gloss_to_rough_conversion import GlossToRoughConversionStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule, FileRule
from configuration import Configuration, GeneralSettings
# No direct ipu import needed in test if we mock its usage by the stage
def create_mock_file_rule_for_gloss_test(
id_val: Optional[uuid.UUID] = None,
map_type: str = "GLOSS", # Test with GLOSS and other types
filename_pattern: str = "gloss.png"
) -> mock.MagicMock:
mock_fr = mock.MagicMock(spec=FileRule)
mock_fr.id = id_val if id_val else uuid.uuid4()
mock_fr.map_type = map_type
mock_fr.filename_pattern = filename_pattern
mock_fr.item_type = "MAP_COL"
mock_fr.active = True
return mock_fr
def create_gloss_conversion_mock_context(
initial_file_rules: Optional[List[FileRule]] = None, # Type hint corrected
initial_processed_details: Optional[Dict] = None, # Type hint corrected
skip_asset_flag: bool = False,
asset_name: str = "GlossAsset",
# Add a mock for general_settings if your stage checks a global flag
# convert_gloss_globally: bool = True
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_asset_rule.file_rules = initial_file_rules if initial_file_rules is not None else []
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_gs = mock.MagicMock(spec=GeneralSettings)
# if your stage uses a global flag:
# mock_gs.convert_gloss_to_rough_globally = convert_gloss_globally
mock_config = mock.MagicMock(spec=Configuration)
mock_config.general_settings = mock_gs
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp_engine_dir"), # Important for new temp file paths
output_base_path=Path("/fake/output"),
effective_supplier="ValidSupplier",
asset_metadata={'asset_name': asset_name},
processed_maps_details=initial_processed_details if initial_processed_details is not None else {},
merged_maps_details={},
files_to_process=list(initial_file_rules) if initial_file_rules else [], # Stage modifies this list
loaded_data_cache={},
config_obj=mock_config,
status_flags={'skip_asset': skip_asset_flag},
incrementing_value=None, # Added as per AssetProcessingContext definition
sha5_value=None # Added as per AssetProcessingContext definition
)
return context
# Unit tests will be added below
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image')
def test_asset_skipped(mock_load_image, mock_save_image):
"""
Test that if 'skip_asset' is True, no processing occurs.
"""
stage = GlossToRoughConversionStage()
gloss_rule_id = uuid.uuid4()
gloss_fr = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS")
initial_details = {
gloss_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_gloss_map.png', 'status': 'Processed', 'map_type': 'GLOSS'}
}
context = create_gloss_conversion_mock_context(
initial_file_rules=[gloss_fr],
initial_processed_details=initial_details,
skip_asset_flag=True # Asset is skipped
)
# Keep a copy of files_to_process and processed_maps_details to compare
original_files_to_process = list(context.files_to_process)
original_processed_maps_details = context.processed_maps_details.copy()
updated_context = stage.execute(context)
mock_load_image.assert_not_called()
mock_save_image.assert_not_called()
assert updated_context.files_to_process == original_files_to_process, "files_to_process should not change if asset is skipped"
assert updated_context.processed_maps_details == original_processed_maps_details, "processed_maps_details should not change if asset is skipped"
assert updated_context.status_flags['skip_asset'] is True
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image')
def test_no_gloss_map_present(mock_load_image, mock_save_image):
"""
Test that if no GLOSS maps are in files_to_process, no conversion occurs.
"""
stage = GlossToRoughConversionStage()
normal_rule_id = uuid.uuid4()
normal_fr = create_mock_file_rule_for_gloss_test(id_val=normal_rule_id, map_type="NORMAL", filename_pattern="normal.png")
albedo_fr = create_mock_file_rule_for_gloss_test(map_type="ALBEDO", filename_pattern="albedo.jpg")
initial_details = {
normal_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_normal_map.png', 'status': 'Processed', 'map_type': 'NORMAL'}
}
context = create_gloss_conversion_mock_context(
initial_file_rules=[normal_fr, albedo_fr],
initial_processed_details=initial_details
)
original_files_to_process = list(context.files_to_process)
original_processed_maps_details = context.processed_maps_details.copy()
updated_context = stage.execute(context)
mock_load_image.assert_not_called()
mock_save_image.assert_not_called()
assert updated_context.files_to_process == original_files_to_process, "files_to_process should not change if no GLOSS maps are present"
assert updated_context.processed_maps_details == original_processed_maps_details, "processed_maps_details should not change if no GLOSS maps are present"
# Ensure map types of existing rules are unchanged
for fr_in_list in updated_context.files_to_process:
if fr_in_list.id == normal_fr.id:
assert fr_in_list.map_type == "NORMAL"
elif fr_in_list.id == albedo_fr.id:
assert fr_in_list.map_type == "ALBEDO"
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging') # Mock logging
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image')
def test_gloss_conversion_uint8_success(mock_load_image, mock_save_image, mock_logging):
"""
Test successful conversion of a GLOSS map (uint8 data) to ROUGHNESS.
"""
stage = GlossToRoughConversionStage()
gloss_rule_id = uuid.uuid4()
# Use a distinct filename for the gloss map to ensure correct path construction
gloss_fr = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS", filename_pattern="my_gloss_map.png")
other_fr_id = uuid.uuid4()
other_fr = create_mock_file_rule_for_gloss_test(id_val=other_fr_id, map_type="NORMAL", filename_pattern="normal_map.png")
initial_gloss_temp_path = Path("/fake/temp_engine_dir/processed_gloss_map.png")
initial_other_temp_path = Path("/fake/temp_engine_dir/processed_normal_map.png")
initial_details = {
gloss_fr.id.hex: {'temp_processed_file': str(initial_gloss_temp_path), 'status': 'Processed', 'map_type': 'GLOSS'},
other_fr.id.hex: {'temp_processed_file': str(initial_other_temp_path), 'status': 'Processed', 'map_type': 'NORMAL'}
}
context = create_gloss_conversion_mock_context(
initial_file_rules=[gloss_fr, other_fr],
initial_processed_details=initial_details
)
mock_loaded_gloss_data = np.array([10, 50, 250], dtype=np.uint8)
mock_load_image.return_value = mock_loaded_gloss_data
mock_save_image.return_value = True # Simulate successful save
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(initial_gloss_temp_path)
# Check that save_image was called with inverted data and correct path
expected_inverted_data = 255 - mock_loaded_gloss_data
# call_args[0] is a tuple of positional args, call_args[1] is a dict of kwargs
saved_path_arg = mock_save_image.call_args[0][0]
saved_data_arg = mock_save_image.call_args[0][1]
assert np.array_equal(saved_data_arg, expected_inverted_data), "Image data passed to save_image is not correctly inverted."
assert "rough_from_gloss_" in saved_path_arg.name, "Saved file name should indicate conversion from gloss."
assert saved_path_arg.parent == Path("/fake/temp_engine_dir"), "Saved file should be in the engine temp directory."
# Ensure the new filename is based on the original gloss map's ID for uniqueness
assert gloss_fr.id.hex in saved_path_arg.name
# Check context.files_to_process
assert len(updated_context.files_to_process) == 2, "Number of file rules in context should remain the same."
converted_rule_found = False
other_rule_untouched = False
for fr_in_list in updated_context.files_to_process:
if fr_in_list.id == gloss_fr.id: # Should be the same rule object, modified
assert fr_in_list.map_type == "ROUGHNESS", "GLOSS map_type should be changed to ROUGHNESS."
# Check if filename_pattern was updated (optional, depends on stage logic)
# For now, assume it might not be, as the primary identifier is map_type and ID
converted_rule_found = True
elif fr_in_list.id == other_fr.id:
assert fr_in_list.map_type == "NORMAL", "Other map_type should remain unchanged."
other_rule_untouched = True
assert converted_rule_found, "The converted GLOSS rule was not found or not updated correctly in files_to_process."
assert other_rule_untouched, "The non-GLOSS rule was modified unexpectedly."
# Check context.processed_maps_details
assert len(updated_context.processed_maps_details) == 2, "Number of entries in processed_maps_details should remain the same."
gloss_detail = updated_context.processed_maps_details[gloss_fr.id.hex]
assert "rough_from_gloss_" in gloss_detail['temp_processed_file'], "temp_processed_file for gloss map not updated."
assert Path(gloss_detail['temp_processed_file']).name == saved_path_arg.name, "Path in details should match saved path."
assert gloss_detail['original_map_type_before_conversion'] == "GLOSS", "original_map_type_before_conversion not set correctly."
assert "Converted from GLOSS to ROUGHNESS" in gloss_detail['notes'], "Conversion notes not added or incorrect."
assert gloss_detail['map_type'] == "ROUGHNESS", "map_type in details not updated to ROUGHNESS."
other_detail = updated_context.processed_maps_details[other_fr.id.hex]
assert other_detail['temp_processed_file'] == str(initial_other_temp_path), "Other map's temp_processed_file should be unchanged."
assert other_detail['map_type'] == "NORMAL", "Other map's map_type should be unchanged."
assert 'original_map_type_before_conversion' not in other_detail, "Other map should not have conversion history."
assert 'notes' not in other_detail or "Converted from GLOSS" not in other_detail['notes'], "Other map should not have conversion notes."
mock_logging.info.assert_any_call(f"Successfully converted GLOSS map {gloss_fr.id.hex} to ROUGHNESS.")
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging') # Mock logging
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image')
def test_gloss_conversion_float_success(mock_load_image, mock_save_image, mock_logging):
"""
Test successful conversion of a GLOSS map (float data) to ROUGHNESS.
"""
stage = GlossToRoughConversionStage()
gloss_rule_id = uuid.uuid4()
gloss_fr = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS", filename_pattern="gloss_float.hdr") # Example float format
initial_gloss_temp_path = Path("/fake/temp_engine_dir/processed_gloss_float.hdr")
initial_details = {
gloss_fr.id.hex: {'temp_processed_file': str(initial_gloss_temp_path), 'status': 'Processed', 'map_type': 'GLOSS'}
}
context = create_gloss_conversion_mock_context(
initial_file_rules=[gloss_fr],
initial_processed_details=initial_details
)
mock_loaded_gloss_data = np.array([0.1, 0.5, 0.9], dtype=np.float32)
mock_load_image.return_value = mock_loaded_gloss_data
mock_save_image.return_value = True # Simulate successful save
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(initial_gloss_temp_path)
expected_inverted_data = 1.0 - mock_loaded_gloss_data
saved_path_arg = mock_save_image.call_args[0][0]
saved_data_arg = mock_save_image.call_args[0][1]
assert np.allclose(saved_data_arg, expected_inverted_data), "Image data (float) passed to save_image is not correctly inverted."
assert "rough_from_gloss_" in saved_path_arg.name, "Saved file name should indicate conversion from gloss."
assert saved_path_arg.parent == Path("/fake/temp_engine_dir"), "Saved file should be in the engine temp directory."
assert gloss_fr.id.hex in saved_path_arg.name
assert len(updated_context.files_to_process) == 1
converted_rule = updated_context.files_to_process[0]
assert converted_rule.id == gloss_fr.id
assert converted_rule.map_type == "ROUGHNESS"
gloss_detail = updated_context.processed_maps_details[gloss_fr.id.hex]
assert "rough_from_gloss_" in gloss_detail['temp_processed_file']
assert Path(gloss_detail['temp_processed_file']).name == saved_path_arg.name
assert gloss_detail['original_map_type_before_conversion'] == "GLOSS"
assert "Converted from GLOSS to ROUGHNESS" in gloss_detail['notes']
assert gloss_detail['map_type'] == "ROUGHNESS"
mock_logging.info.assert_any_call(f"Successfully converted GLOSS map {gloss_fr.id.hex} to ROUGHNESS.")
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image')
def test_load_image_fails(mock_load_image, mock_save_image, mock_logging):
"""
Test behavior when ipu.load_image fails (returns None).
The original FileRule should be kept, and an error logged.
"""
stage = GlossToRoughConversionStage()
gloss_rule_id = uuid.uuid4()
gloss_fr = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS", filename_pattern="gloss_fails_load.png")
initial_gloss_temp_path = Path("/fake/temp_engine_dir/processed_gloss_fails_load.png")
initial_details = {
gloss_fr.id.hex: {'temp_processed_file': str(initial_gloss_temp_path), 'status': 'Processed', 'map_type': 'GLOSS'}
}
context = create_gloss_conversion_mock_context(
initial_file_rules=[gloss_fr],
initial_processed_details=initial_details
)
# Keep a copy for comparison
original_file_rule_map_type = gloss_fr.map_type
original_details_entry = context.processed_maps_details[gloss_fr.id.hex].copy()
mock_load_image.return_value = None # Simulate load failure
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(initial_gloss_temp_path)
mock_save_image.assert_not_called() # Save should not be attempted
# Check context.files_to_process: rule should be unchanged
assert len(updated_context.files_to_process) == 1
processed_rule = updated_context.files_to_process[0]
assert processed_rule.id == gloss_fr.id
assert processed_rule.map_type == original_file_rule_map_type, "FileRule map_type should not change if load fails."
assert processed_rule.map_type == "GLOSS" # Explicitly check it's still GLOSS
# Check context.processed_maps_details: details should be unchanged
current_details_entry = updated_context.processed_maps_details[gloss_fr.id.hex]
assert current_details_entry['temp_processed_file'] == str(initial_gloss_temp_path)
assert current_details_entry['map_type'] == "GLOSS"
assert 'original_map_type_before_conversion' not in current_details_entry
assert 'notes' not in current_details_entry or "Converted from GLOSS" not in current_details_entry['notes']
mock_logging.error.assert_called_once_with(
f"Failed to load image data for GLOSS map {gloss_fr.id.hex} from {initial_gloss_temp_path}. Skipping conversion for this map."
)
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image')
def test_save_image_fails(mock_load_image, mock_save_image, mock_logging):
"""
Test behavior when ipu.save_image fails (returns False).
The original FileRule should be kept, and an error logged.
"""
stage = GlossToRoughConversionStage()
gloss_rule_id = uuid.uuid4()
gloss_fr = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS", filename_pattern="gloss_fails_save.png")
initial_gloss_temp_path = Path("/fake/temp_engine_dir/processed_gloss_fails_save.png")
initial_details = {
gloss_fr.id.hex: {'temp_processed_file': str(initial_gloss_temp_path), 'status': 'Processed', 'map_type': 'GLOSS'}
}
context = create_gloss_conversion_mock_context(
initial_file_rules=[gloss_fr],
initial_processed_details=initial_details
)
original_file_rule_map_type = gloss_fr.map_type
original_details_entry = context.processed_maps_details[gloss_fr.id.hex].copy()
mock_loaded_gloss_data = np.array([10, 50, 250], dtype=np.uint8)
mock_load_image.return_value = mock_loaded_gloss_data
mock_save_image.return_value = False # Simulate save failure
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(initial_gloss_temp_path)
# Check that save_image was called with correct data and path
expected_inverted_data = 255 - mock_loaded_gloss_data
# call_args[0] is a tuple of positional args
saved_path_arg = mock_save_image.call_args[0][0]
saved_data_arg = mock_save_image.call_args[0][1]
assert np.array_equal(saved_data_arg, expected_inverted_data), "Image data passed to save_image is not correctly inverted even on failure."
assert "rough_from_gloss_" in saved_path_arg.name, "Attempted save file name should indicate conversion from gloss."
assert saved_path_arg.parent == Path("/fake/temp_engine_dir"), "Attempted save file should be in the engine temp directory."
# Check context.files_to_process: rule should be unchanged
assert len(updated_context.files_to_process) == 1
processed_rule = updated_context.files_to_process[0]
assert processed_rule.id == gloss_fr.id
assert processed_rule.map_type == original_file_rule_map_type, "FileRule map_type should not change if save fails."
assert processed_rule.map_type == "GLOSS"
# Check context.processed_maps_details: details should be unchanged
current_details_entry = updated_context.processed_maps_details[gloss_fr.id.hex]
assert current_details_entry['temp_processed_file'] == str(initial_gloss_temp_path)
assert current_details_entry['map_type'] == "GLOSS"
assert 'original_map_type_before_conversion' not in current_details_entry
assert 'notes' not in current_details_entry or "Converted from GLOSS" not in current_details_entry['notes']
mock_logging.error.assert_called_once_with(
f"Failed to save inverted GLOSS map {gloss_fr.id.hex} to {saved_path_arg}. Retaining original GLOSS map."
)
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image')
def test_gloss_map_in_files_to_process_but_not_in_details(mock_load_image, mock_save_image, mock_logging):
"""
Test behavior when a GLOSS FileRule is in files_to_process but its details
are missing from processed_maps_details.
The stage should log an error and skip this FileRule.
"""
stage = GlossToRoughConversionStage()
gloss_rule_id = uuid.uuid4()
# This FileRule is in files_to_process
gloss_fr_in_list = create_mock_file_rule_for_gloss_test(id_val=gloss_rule_id, map_type="GLOSS", filename_pattern="orphan_gloss.png")
# processed_maps_details is empty or does not contain gloss_fr_in_list.id.hex
initial_details = {}
context = create_gloss_conversion_mock_context(
initial_file_rules=[gloss_fr_in_list],
initial_processed_details=initial_details
)
original_files_to_process = list(context.files_to_process)
original_processed_maps_details = context.processed_maps_details.copy()
updated_context = stage.execute(context)
mock_load_image.assert_not_called() # Load should not be attempted if details are missing
mock_save_image.assert_not_called() # Save should not be attempted
# Check context.files_to_process: rule should be unchanged
assert len(updated_context.files_to_process) == 1
processed_rule = updated_context.files_to_process[0]
assert processed_rule.id == gloss_fr_in_list.id
assert processed_rule.map_type == "GLOSS", "FileRule map_type should not change if its details are missing."
# Check context.processed_maps_details: should remain unchanged
assert updated_context.processed_maps_details == original_processed_maps_details, "processed_maps_details should not change."
mock_logging.error.assert_called_once_with(
f"GLOSS map {gloss_fr_in_list.id.hex} found in files_to_process but missing from processed_maps_details. Skipping conversion."
)
# Test for Case 8.2 (GLOSS map ID in processed_maps_details but no corresponding FileRule in files_to_process)
# This case is implicitly handled because the stage iterates files_to_process.
# If a FileRule isn't in files_to_process, its corresponding entry in processed_maps_details (if any) won't be acted upon.
# We can add a simple test to ensure no errors occur and non-relevant details are untouched.
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.logging')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.save_image')
@mock.patch('processing.pipeline.stages.gloss_to_rough_conversion.ipu.load_image')
def test_gloss_detail_exists_but_not_in_files_to_process(mock_load_image, mock_save_image, mock_logging):
"""
Test that if a GLOSS map detail exists in processed_maps_details but
no corresponding FileRule is in files_to_process, it's simply ignored
without error, and other valid conversions proceed.
"""
stage = GlossToRoughConversionStage()
# This rule will be processed
convert_rule_id = uuid.uuid4()
convert_fr = create_mock_file_rule_for_gloss_test(id_val=convert_rule_id, map_type="GLOSS", filename_pattern="convert_me.png")
convert_initial_temp_path = Path("/fake/temp_engine_dir/processed_convert_me.png")
# This rule's details exist, but the rule itself is not in files_to_process
orphan_detail_id = uuid.uuid4()
initial_details = {
convert_fr.id.hex: {'temp_processed_file': str(convert_initial_temp_path), 'status': 'Processed', 'map_type': 'GLOSS'},
orphan_detail_id.hex: {'temp_processed_file': '/fake/temp_engine_dir/orphan.png', 'status': 'Processed', 'map_type': 'GLOSS', 'notes': 'This is an orphan'}
}
context = create_gloss_conversion_mock_context(
initial_file_rules=[convert_fr], # Only convert_fr is in files_to_process
initial_processed_details=initial_details
)
mock_loaded_data = np.array([100], dtype=np.uint8)
mock_load_image.return_value = mock_loaded_data
mock_save_image.return_value = True
updated_context = stage.execute(context)
# Assert that load/save were called only for the rule in files_to_process
mock_load_image.assert_called_once_with(convert_initial_temp_path)
mock_save_image.assert_called_once() # Check it was called, details checked in other tests
# Check that the orphan detail in processed_maps_details is untouched
assert orphan_detail_id.hex in updated_context.processed_maps_details
orphan_entry = updated_context.processed_maps_details[orphan_detail_id.hex]
assert orphan_entry['temp_processed_file'] == '/fake/temp_engine_dir/orphan.png'
assert orphan_entry['map_type'] == 'GLOSS'
assert orphan_entry['notes'] == 'This is an orphan'
assert 'original_map_type_before_conversion' not in orphan_entry
# Check that the processed rule was indeed converted
assert convert_fr.id.hex in updated_context.processed_maps_details
converted_entry = updated_context.processed_maps_details[convert_fr.id.hex]
assert converted_entry['map_type'] == 'ROUGHNESS'
assert "rough_from_gloss_" in converted_entry['temp_processed_file']
# No errors should have been logged regarding the orphan detail
for call_args in mock_logging.error.call_args_list:
assert str(orphan_detail_id.hex) not in call_args[0][0], "Error logged for orphan detail"

View File

@@ -0,0 +1,555 @@
import pytest
from unittest import mock
from pathlib import Path
import uuid
import numpy as np
from typing import Optional # Added for type hinting in helper functions
from processing.pipeline.stages.individual_map_processing import IndividualMapProcessingStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule, FileRule, TransformSettings # Key models
from configuration import Configuration, GeneralSettings
# cv2 might be imported by the stage for interpolation constants, ensure it's mockable if so.
# For now, assume ipu handles interpolation details.
def create_mock_transform_settings(
target_width=0, target_height=0, resize_mode="FIT",
ensure_pot=False, allow_upscale=True, target_color_profile="RGB" # Add other fields as needed
) -> mock.MagicMock:
ts = mock.MagicMock(spec=TransformSettings)
ts.target_width = target_width
ts.target_height = target_height
ts.resize_mode = resize_mode
ts.ensure_pot = ensure_pot
ts.allow_upscale = allow_upscale
ts.target_color_profile = target_color_profile
# ts.resize_filter = "AREA" # if your stage uses this
return ts
def create_mock_file_rule_for_individual_processing(
id_val: Optional[uuid.UUID] = None,
map_type: str = "ALBEDO",
filename_pattern: str = "albedo_*.png", # Pattern for glob
item_type: str = "MAP_COL",
active: bool = True,
transform_settings: Optional[mock.MagicMock] = None
) -> mock.MagicMock:
mock_fr = mock.MagicMock(spec=FileRule)
mock_fr.id = id_val if id_val else uuid.uuid4()
mock_fr.map_type = map_type
mock_fr.filename_pattern = filename_pattern
mock_fr.item_type = item_type
mock_fr.active = active
mock_fr.transform_settings = transform_settings if transform_settings else create_mock_transform_settings()
return mock_fr
def create_individual_map_proc_mock_context(
initial_file_rules: Optional[list] = None,
asset_source_path_str: str = "/fake/asset_source",
skip_asset_flag: bool = False,
asset_name: str = "IndividualMapAsset"
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_asset_rule.source_path = Path(asset_source_path_str)
# file_rules on AssetRule not directly used by stage, context.files_to_process is
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_config = mock.MagicMock(spec=Configuration)
# mock_config.general_settings = mock.MagicMock(spec=GeneralSettings) # If needed
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp_engine_dir"),
output_base_path=Path("/fake/output"),
effective_supplier="ValidSupplier",
asset_metadata={'asset_name': asset_name},
processed_maps_details={}, # Stage populates this
merged_maps_details={},
files_to_process=list(initial_file_rules) if initial_file_rules else [],
loaded_data_cache={},
config_obj=mock_config,
status_flags={'skip_asset': skip_asset_flag},
incrementing_value=None,
sha5_value=None # Corrected from sha5_value to sha_value if that's the actual param
)
return context
# Placeholder for tests to be added next
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu')
@mock.patch('logging.info')
def test_asset_skipped_if_flag_is_true(mock_log_info, mock_ipu):
stage = IndividualMapProcessingStage()
context = create_individual_map_proc_mock_context(skip_asset_flag=True)
# Add a dummy file rule to ensure it's not processed
file_rule = create_mock_file_rule_for_individual_processing()
context.files_to_process = [file_rule]
updated_context = stage.execute(context)
mock_ipu.load_image.assert_not_called()
mock_ipu.save_image.assert_not_called()
assert not updated_context.processed_maps_details # No details should be added
# Check for a log message indicating skip, if applicable (depends on stage's logging)
# mock_log_info.assert_any_call("Skipping asset IndividualMapAsset due to status_flags['skip_asset'] = True") # Example
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu')
@mock.patch('logging.info')
def test_no_processing_if_no_map_col_rules(mock_log_info, mock_ipu):
stage = IndividualMapProcessingStage()
# Create a file rule that is NOT of item_type MAP_COL
non_map_col_rule = create_mock_file_rule_for_individual_processing(item_type="METADATA")
context = create_individual_map_proc_mock_context(initial_file_rules=[non_map_col_rule])
updated_context = stage.execute(context)
mock_ipu.load_image.assert_not_called()
mock_ipu.save_image.assert_not_called()
assert not updated_context.processed_maps_details
# mock_log_info.assert_any_call("No FileRules of item_type 'MAP_COL' to process for asset IndividualMapAsset.") # Example
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image')
@mock.patch('pathlib.Path.glob') # Mocking Path.glob used by the stage's _find_source_file
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_individual_map_processing_success_no_resize(
mock_log_error, mock_log_info, mock_path_glob, mock_load_image,
mock_calc_dims, mock_resize_image, mock_save_image
):
stage = IndividualMapProcessingStage()
source_file_name = "albedo_source.png"
# The glob is called on context.asset_rule.source_path, so mock that Path object's glob
mock_asset_source_path = Path("/fake/asset_source")
mock_found_source_path = mock_asset_source_path / source_file_name
# We need to mock the glob method of the Path instance
# that represents the asset's source directory.
# The stage does something like: Path(context.asset_rule.source_path).glob(...)
# So, we need to ensure that when Path() is called with that specific string,
# the resulting object's glob method is our mock.
# A more robust way is to mock Path itself to return a mock object
# whose glob method is also a mock.
# Simpler approach for now: assume Path.glob is used as a static/class method call
# or that the instance it's called on is correctly patched by @mock.patch('pathlib.Path.glob')
# if the stage does `from pathlib import Path` and then `Path(path_str).glob(...)`.
# The prompt example uses @mock.patch('pathlib.Path.glob'), implying the stage might do this:
# for f_pattern in patterns:
# for found_file in Path(base_dir).glob(f_pattern): ...
# Let's refine the mock_path_glob setup.
# The stage's _find_source_file likely does:
# search_path = Path(self.context.asset_rule.source_path)
# found_files = list(search_path.glob(filename_pattern))
# To correctly mock this, we need to mock the `glob` method of the specific Path instance.
# Or, if `_find_source_file` instantiates `Path` like `Path(str(context.asset_rule.source_path)).glob(...)`,
# then patching `pathlib.Path.glob` might work if it's treated as a method that gets bound.
# Let's stick to the example's @mock.patch('pathlib.Path.glob') and assume it covers the usage.
mock_path_glob.return_value = [mock_found_source_path] # Glob finds one file
ts = create_mock_transform_settings(target_width=100, target_height=100)
file_rule = create_mock_file_rule_for_individual_processing(
map_type="ALBEDO", filename_pattern="albedo_*.png", transform_settings=ts
)
context = create_individual_map_proc_mock_context(
initial_file_rules=[file_rule],
asset_source_path_str=str(mock_asset_source_path) # Ensure context uses this path
)
mock_img_data = np.zeros((100, 100, 3), dtype=np.uint8) # Original dimensions
mock_load_image.return_value = mock_img_data
mock_calc_dims.return_value = (100, 100) # No resize needed
mock_save_image.return_value = True
updated_context = stage.execute(context)
# Assert that Path(context.asset_rule.source_path).glob was called
# This requires a bit more intricate mocking if Path instances are created inside.
# For now, assert mock_path_glob was called with the pattern.
# The actual call in stage is `Path(context.asset_rule.source_path).glob(file_rule.filename_pattern)`
# So, `mock_path_glob` (if it patches `Path.glob` globally) should be called.
# We need to ensure the mock_path_glob is associated with the correct Path instance or that
# the global patch works as intended.
# A common pattern is:
# with mock.patch.object(Path, 'glob', return_value=[mock_found_source_path]) as specific_glob_mock:
# # execute code
# specific_glob_mock.assert_called_once_with(file_rule.filename_pattern)
# However, the decorator @mock.patch('pathlib.Path.glob') should work if the stage code is
# `from pathlib import Path; p = Path(...); p.glob(...)`
# The stage's _find_source_file will instantiate a Path object from context.asset_rule.source_path
# and then call glob on it.
# So, @mock.patch('pathlib.Path.glob') is patching the method on the class.
# When an instance calls it, the mock is used.
mock_path_glob.assert_called_once_with(file_rule.filename_pattern)
mock_load_image.assert_called_once_with(mock_found_source_path)
# The actual call to calculate_target_dimensions is:
# ipu.calculate_target_dimensions(original_dims, ts.target_width, ts.target_height, ts.resize_mode, ts.ensure_pot, ts.allow_upscale)
mock_calc_dims.assert_called_once_with(
(100, 100), ts.target_width, ts.target_height, ts.resize_mode, ts.ensure_pot, ts.allow_upscale
)
mock_resize_image.assert_not_called() # Crucial for this test case
mock_save_image.assert_called_once()
# Check save path and data
saved_image_arg, saved_path_arg = mock_save_image.call_args[0]
assert np.array_equal(saved_image_arg, mock_img_data) # Ensure correct image data is passed to save
assert "processed_ALBEDO_" in saved_path_arg.name # Based on map_type
assert file_rule.id.hex in saved_path_arg.name # Ensure unique name with FileRule ID
assert saved_path_arg.parent == context.engine_temp_dir
assert file_rule.id.hex in updated_context.processed_maps_details
details = updated_context.processed_maps_details[file_rule.id.hex]
assert details['status'] == 'Processed'
assert details['source_file'] == str(mock_found_source_path)
assert Path(details['temp_processed_file']) == saved_path_arg
assert details['original_dimensions'] == (100, 100)
assert details['processed_dimensions'] == (100, 100)
assert details['map_type'] == file_rule.map_type
mock_log_error.assert_not_called()
mock_log_info.assert_any_call(f"Successfully processed map {file_rule.map_type} (ID: {file_rule.id.hex}) for asset {context.asset_rule.name}. Output: {saved_path_arg}")
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image')
@mock.patch('pathlib.Path.glob')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_source_file_not_found(
mock_log_error, mock_log_info, mock_path_glob, mock_load_image,
mock_calc_dims, mock_resize_image, mock_save_image
):
stage = IndividualMapProcessingStage()
mock_asset_source_path = Path("/fake/asset_source")
mock_path_glob.return_value = [] # Glob finds no files
file_rule = create_mock_file_rule_for_individual_processing(filename_pattern="nonexistent_*.png")
context = create_individual_map_proc_mock_context(
initial_file_rules=[file_rule],
asset_source_path_str=str(mock_asset_source_path)
)
updated_context = stage.execute(context)
mock_path_glob.assert_called_once_with(file_rule.filename_pattern)
mock_load_image.assert_not_called()
mock_calc_dims.assert_not_called()
mock_resize_image.assert_not_called()
mock_save_image.assert_not_called()
assert file_rule.id.hex in updated_context.processed_maps_details
details = updated_context.processed_maps_details[file_rule.id.hex]
assert details['status'] == 'Source Not Found'
assert details['source_file'] is None
assert details['temp_processed_file'] is None
assert details['error_message'] is not None # Check an error message is present
mock_log_error.assert_called_once()
# Example: mock_log_error.assert_called_with(f"Could not find source file for rule {file_rule.id} (pattern: {file_rule.filename_pattern}) in {context.asset_rule.source_path}")
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image')
@mock.patch('pathlib.Path.glob')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_load_image_fails(
mock_log_error, mock_log_info, mock_path_glob, mock_load_image,
mock_calc_dims, mock_resize_image, mock_save_image
):
stage = IndividualMapProcessingStage()
source_file_name = "albedo_corrupt.png"
mock_asset_source_path = Path("/fake/asset_source")
mock_found_source_path = mock_asset_source_path / source_file_name
mock_path_glob.return_value = [mock_found_source_path]
mock_load_image.return_value = None # Simulate load failure
file_rule = create_mock_file_rule_for_individual_processing(filename_pattern="albedo_*.png")
context = create_individual_map_proc_mock_context(
initial_file_rules=[file_rule],
asset_source_path_str=str(mock_asset_source_path)
)
updated_context = stage.execute(context)
mock_path_glob.assert_called_once_with(file_rule.filename_pattern)
mock_load_image.assert_called_once_with(mock_found_source_path)
mock_calc_dims.assert_not_called()
mock_resize_image.assert_not_called()
mock_save_image.assert_not_called()
assert file_rule.id.hex in updated_context.processed_maps_details
details = updated_context.processed_maps_details[file_rule.id.hex]
assert details['status'] == 'Load Failed'
assert details['source_file'] == str(mock_found_source_path)
assert details['temp_processed_file'] is None
assert details['error_message'] is not None
mock_log_error.assert_called_once()
# Example: mock_log_error.assert_called_with(f"Failed to load image {mock_found_source_path} for rule {file_rule.id}")
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image')
@mock.patch('pathlib.Path.glob')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_resize_occurs_when_dimensions_differ(
mock_log_error, mock_log_info, mock_path_glob, mock_load_image,
mock_calc_dims, mock_resize_image, mock_save_image
):
stage = IndividualMapProcessingStage()
source_file_name = "albedo_resize.png"
mock_asset_source_path = Path("/fake/asset_source")
mock_found_source_path = mock_asset_source_path / source_file_name
mock_path_glob.return_value = [mock_found_source_path]
original_dims = (100, 100)
target_dims = (50, 50) # Different dimensions
mock_img_data = np.zeros((*original_dims, 3), dtype=np.uint8)
mock_resized_img_data = np.zeros((*target_dims, 3), dtype=np.uint8)
mock_load_image.return_value = mock_img_data
ts = create_mock_transform_settings(target_width=target_dims[0], target_height=target_dims[1])
file_rule = create_mock_file_rule_for_individual_processing(transform_settings=ts)
context = create_individual_map_proc_mock_context(
initial_file_rules=[file_rule],
asset_source_path_str=str(mock_asset_source_path)
)
mock_calc_dims.return_value = target_dims # Simulate calc_dims returning new dimensions
mock_resize_image.return_value = mock_resized_img_data # Simulate resize returning new image data
mock_save_image.return_value = True
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(mock_found_source_path)
mock_calc_dims.assert_called_once_with(
original_dims, ts.target_width, ts.target_height, ts.resize_mode, ts.ensure_pot, ts.allow_upscale
)
# The actual call to resize_image is:
# ipu.resize_image(loaded_image, target_dims, ts.resize_filter) # Assuming resize_filter is used
# If resize_filter is not on TransformSettings or not used, adjust this.
# For now, let's assume it's ipu.resize_image(loaded_image, target_dims) or similar
# The stage code is: resized_image = ipu.resize_image(loaded_image, target_dims_calculated, file_rule.transform_settings.resize_filter)
# So we need to mock ts.resize_filter
ts.resize_filter = "LANCZOS4" # Example filter
mock_resize_image.assert_called_once_with(mock_img_data, target_dims, ts.resize_filter)
saved_image_arg, saved_path_arg = mock_save_image.call_args[0]
assert np.array_equal(saved_image_arg, mock_resized_img_data) # Check resized data is saved
assert "processed_ALBEDO_" in saved_path_arg.name
assert saved_path_arg.parent == context.engine_temp_dir
assert file_rule.id.hex in updated_context.processed_maps_details
details = updated_context.processed_maps_details[file_rule.id.hex]
assert details['status'] == 'Processed'
assert details['original_dimensions'] == original_dims
assert details['processed_dimensions'] == target_dims
mock_log_error.assert_not_called()
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image')
@mock.patch('pathlib.Path.glob')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_save_image_fails(
mock_log_error, mock_log_info, mock_path_glob, mock_load_image,
mock_calc_dims, mock_resize_image, mock_save_image
):
stage = IndividualMapProcessingStage()
source_file_name = "albedo_save_fail.png"
mock_asset_source_path = Path("/fake/asset_source")
mock_found_source_path = mock_asset_source_path / source_file_name
mock_path_glob.return_value = [mock_found_source_path]
mock_img_data = np.zeros((100, 100, 3), dtype=np.uint8)
mock_load_image.return_value = mock_img_data
mock_calc_dims.return_value = (100, 100) # No resize
mock_save_image.return_value = False # Simulate save failure
ts = create_mock_transform_settings()
file_rule = create_mock_file_rule_for_individual_processing(transform_settings=ts)
context = create_individual_map_proc_mock_context(
initial_file_rules=[file_rule],
asset_source_path_str=str(mock_asset_source_path)
)
updated_context = stage.execute(context)
mock_save_image.assert_called_once() # Attempt to save should still happen
assert file_rule.id.hex in updated_context.processed_maps_details
details = updated_context.processed_maps_details[file_rule.id.hex]
assert details['status'] == 'Save Failed'
assert details['source_file'] == str(mock_found_source_path)
assert details['temp_processed_file'] is not None # Path was generated
assert details['error_message'] is not None
mock_log_error.assert_called_once()
# Example: mock_log_error.assert_called_with(f"Failed to save processed image for rule {file_rule.id} to {details['temp_processed_file']}")
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.convert_bgr_to_rgb')
@mock.patch('pathlib.Path.glob')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_color_conversion_bgr_to_rgb(
mock_log_error, mock_log_info, mock_path_glob, mock_convert_bgr, mock_load_image,
mock_calc_dims, mock_resize_image, mock_save_image
):
stage = IndividualMapProcessingStage()
source_file_name = "albedo_bgr.png"
mock_asset_source_path = Path("/fake/asset_source")
mock_found_source_path = mock_asset_source_path / source_file_name
mock_path_glob.return_value = [mock_found_source_path]
mock_bgr_img_data = np.zeros((100, 100, 3), dtype=np.uint8) # Loaded as BGR
mock_rgb_img_data = np.zeros((100, 100, 3), dtype=np.uint8) # After conversion
mock_load_image.return_value = mock_bgr_img_data # Image is loaded (assume BGR by default from cv2)
mock_convert_bgr.return_value = mock_rgb_img_data # Mock the conversion
mock_calc_dims.return_value = (100, 100) # No resize
mock_save_image.return_value = True
# Transform settings request RGB, and stage assumes load might be BGR
ts = create_mock_transform_settings(target_color_profile="RGB")
file_rule = create_mock_file_rule_for_individual_processing(transform_settings=ts)
context = create_individual_map_proc_mock_context(
initial_file_rules=[file_rule],
asset_source_path_str=str(mock_asset_source_path)
)
# The stage code is:
# if file_rule.transform_settings.target_color_profile == "RGB" and loaded_image.shape[2] == 3:
# logger.info(f"Attempting to convert image from BGR to RGB for {file_rule_id_hex}")
# processed_image_data = ipu.convert_bgr_to_rgb(processed_image_data)
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(mock_found_source_path)
mock_convert_bgr.assert_called_once_with(mock_bgr_img_data)
mock_resize_image.assert_not_called()
saved_image_arg, _ = mock_save_image.call_args[0]
assert np.array_equal(saved_image_arg, mock_rgb_img_data) # Ensure RGB data is saved
mock_log_error.assert_not_called()
mock_log_info.assert_any_call(f"Attempting to convert image from BGR to RGB for {file_rule.id.hex}")
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.save_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.resize_image')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.calculate_target_dimensions')
@mock.patch('processing.pipeline.stages.individual_map_processing.ipu.load_image')
@mock.patch('pathlib.Path.glob')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_multiple_map_col_rules_processed(
mock_log_error, mock_log_info, mock_path_glob, mock_load_image,
mock_calc_dims, mock_resize_image, mock_save_image
):
stage = IndividualMapProcessingStage()
mock_asset_source_path = Path("/fake/asset_source")
# Rule 1: Albedo
ts1 = create_mock_transform_settings(target_width=100, target_height=100)
file_rule1_id = uuid.uuid4()
file_rule1 = create_mock_file_rule_for_individual_processing(
id_val=file_rule1_id, map_type="ALBEDO", filename_pattern="albedo_*.png", transform_settings=ts1
)
source_file1 = mock_asset_source_path / "albedo_map.png"
img_data1 = np.zeros((100, 100, 3), dtype=np.uint8)
# Rule 2: Roughness
ts2 = create_mock_transform_settings(target_width=50, target_height=50) # Resize
ts2.resize_filter = "AREA"
file_rule2_id = uuid.uuid4()
file_rule2 = create_mock_file_rule_for_individual_processing(
id_val=file_rule2_id, map_type="ROUGHNESS", filename_pattern="rough_*.png", transform_settings=ts2
)
source_file2 = mock_asset_source_path / "rough_map.png"
img_data2_orig = np.zeros((200, 200, 1), dtype=np.uint8) # Original, needs resize
img_data2_resized = np.zeros((50, 50, 1), dtype=np.uint8) # Resized
context = create_individual_map_proc_mock_context(
initial_file_rules=[file_rule1, file_rule2],
asset_source_path_str=str(mock_asset_source_path)
)
# Mock behaviors for Path.glob, load_image, calc_dims, resize, save
# Path.glob will be called twice
mock_path_glob.side_effect = [
[source_file1], # For albedo_*.png
[source_file2] # For rough_*.png
]
mock_load_image.side_effect = [img_data1, img_data2_orig]
mock_calc_dims.side_effect = [
(100, 100), # For rule1 (no change)
(50, 50) # For rule2 (change)
]
mock_resize_image.return_value = img_data2_resized # Only called for rule2
mock_save_image.return_value = True
updated_context = stage.execute(context)
# Assertions for Rule 1 (Albedo)
assert mock_path_glob.call_args_list[0][0][0] == file_rule1.filename_pattern
assert mock_load_image.call_args_list[0][0][0] == source_file1
assert mock_calc_dims.call_args_list[0][0] == ((100,100), ts1.target_width, ts1.target_height, ts1.resize_mode, ts1.ensure_pot, ts1.allow_upscale)
# Assertions for Rule 2 (Roughness)
assert mock_path_glob.call_args_list[1][0][0] == file_rule2.filename_pattern
assert mock_load_image.call_args_list[1][0][0] == source_file2
assert mock_calc_dims.call_args_list[1][0] == ((200,200), ts2.target_width, ts2.target_height, ts2.resize_mode, ts2.ensure_pot, ts2.allow_upscale)
mock_resize_image.assert_called_once_with(img_data2_orig, (50,50), ts2.resize_filter)
assert mock_save_image.call_count == 2
# Check saved image for rule 1
saved_img1_arg, saved_path1_arg = mock_save_image.call_args_list[0][0]
assert np.array_equal(saved_img1_arg, img_data1)
assert "processed_ALBEDO_" in saved_path1_arg.name
assert file_rule1_id.hex in saved_path1_arg.name
# Check saved image for rule 2
saved_img2_arg, saved_path2_arg = mock_save_image.call_args_list[1][0]
assert np.array_equal(saved_img2_arg, img_data2_resized)
assert "processed_ROUGHNESS_" in saved_path2_arg.name
assert file_rule2_id.hex in saved_path2_arg.name
# Check context details
assert file_rule1_id.hex in updated_context.processed_maps_details
details1 = updated_context.processed_maps_details[file_rule1_id.hex]
assert details1['status'] == 'Processed'
assert details1['original_dimensions'] == (100, 100)
assert details1['processed_dimensions'] == (100, 100)
assert file_rule2_id.hex in updated_context.processed_maps_details
details2 = updated_context.processed_maps_details[file_rule2_id.hex]
assert details2['status'] == 'Processed'
assert details2['original_dimensions'] == (200, 200) # Original dims of img_data2_orig
assert details2['processed_dimensions'] == (50, 50)
mock_log_error.assert_not_called()

View File

@@ -0,0 +1,538 @@
import pytest
from unittest import mock
from pathlib import Path
import uuid
import numpy as np
from typing import Optional # Added Optional for type hinting
from processing.pipeline.stages.map_merging import MapMergingStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule, FileRule, MergeSettings, MergeInputChannel
from configuration import Configuration
# Mock Helper Functions
def create_mock_merge_input_channel(
file_rule_id: uuid.UUID, source_channel: int = 0, target_channel: int = 0, invert: bool = False
) -> mock.MagicMock:
mic = mock.MagicMock(spec=MergeInputChannel)
mic.file_rule_id = file_rule_id
mic.source_channel = source_channel
mic.target_channel = target_channel
mic.invert_source_channel = invert
mic.default_value_if_missing = 0 # Or some other default
return mic
def create_mock_merge_settings(
input_maps: Optional[list] = None, # List of mock MergeInputChannel
output_channels: int = 3
) -> mock.MagicMock:
ms = mock.MagicMock(spec=MergeSettings)
ms.input_maps = input_maps if input_maps is not None else []
ms.output_channels = output_channels
return ms
def create_mock_file_rule_for_merging(
id_val: Optional[uuid.UUID] = None,
map_type: str = "ORM", # Output map type
item_type: str = "MAP_MERGE",
merge_settings: Optional[mock.MagicMock] = None
) -> mock.MagicMock:
mock_fr = mock.MagicMock(spec=FileRule)
mock_fr.id = id_val if id_val else uuid.uuid4()
mock_fr.map_type = map_type
mock_fr.filename_pattern = f"{map_type.lower()}_merged.png" # Placeholder
mock_fr.item_type = item_type
mock_fr.active = True
mock_fr.merge_settings = merge_settings if merge_settings else create_mock_merge_settings()
return mock_fr
def create_map_merging_mock_context(
initial_file_rules: Optional[list] = None, # Will contain the MAP_MERGE rule
initial_processed_details: Optional[dict] = None, # Pre-processed inputs for merge
skip_asset_flag: bool = False,
asset_name: str = "MergeAsset"
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_config = mock.MagicMock(spec=Configuration)
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp_engine_dir"),
output_base_path=Path("/fake/output"),
effective_supplier="ValidSupplier",
asset_metadata={'asset_name': asset_name},
processed_maps_details=initial_processed_details if initial_processed_details is not None else {},
merged_maps_details={}, # Stage populates this
files_to_process=list(initial_file_rules) if initial_file_rules else [],
loaded_data_cache={},
config_obj=mock_config,
status_flags={'skip_asset': skip_asset_flag},
incrementing_value=None,
sha5_value=None # Corrected from sha5_value to sha_value based on AssetProcessingContext
)
return context
def test_asset_skipped():
stage = MapMergingStage()
context = create_map_merging_mock_context(skip_asset_flag=True)
updated_context = stage.execute(context)
assert updated_context == context # No changes expected
assert not updated_context.merged_maps_details # No maps should be merged
def test_no_map_merge_rules():
stage = MapMergingStage()
# Context with a non-MAP_MERGE rule
non_merge_rule = create_mock_file_rule_for_merging(item_type="TEXTURE_MAP", map_type="Diffuse")
context = create_map_merging_mock_context(initial_file_rules=[non_merge_rule])
updated_context = stage.execute(context)
assert updated_context == context # No changes expected
assert not updated_context.merged_maps_details # No maps should be merged
@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.resize_image') # If testing resize
@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_map_merging_rgb_success(mock_log_error, mock_log_info, mock_load_image, mock_resize_image, mock_save_image):
stage = MapMergingStage()
# Input FileRules (mocked as already processed)
r_id, g_id, b_id = uuid.uuid4(), uuid.uuid4(), uuid.uuid4()
processed_details = {
r_id.hex: {'temp_processed_file': '/fake/red.png', 'status': 'Processed', 'map_type': 'RED_SRC'},
g_id.hex: {'temp_processed_file': '/fake/green.png', 'status': 'Processed', 'map_type': 'GREEN_SRC'},
b_id.hex: {'temp_processed_file': '/fake/blue.png', 'status': 'Processed', 'map_type': 'BLUE_SRC'}
}
# Mock loaded image data (grayscale for inputs)
mock_r_data = np.full((10, 10), 200, dtype=np.uint8)
mock_g_data = np.full((10, 10), 100, dtype=np.uint8)
mock_b_data = np.full((10, 10), 50, dtype=np.uint8)
mock_load_image.side_effect = [mock_r_data, mock_g_data, mock_b_data]
# Merge Rule setup
merge_inputs = [
create_mock_merge_input_channel(file_rule_id=r_id, source_channel=0, target_channel=0), # R to R
create_mock_merge_input_channel(file_rule_id=g_id, source_channel=0, target_channel=1), # G to G
create_mock_merge_input_channel(file_rule_id=b_id, source_channel=0, target_channel=2) # B to B
]
merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=3)
merge_rule_id = uuid.uuid4()
merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="RGB_Combined", merge_settings=merge_settings)
context = create_map_merging_mock_context(
initial_file_rules=[merge_rule],
initial_processed_details=processed_details
)
mock_save_image.return_value = True
updated_context = stage.execute(context)
assert mock_load_image.call_count == 3
mock_resize_image.assert_not_called() # Assuming all inputs are same size for this test
mock_save_image.assert_called_once()
# Check that the correct filename was passed to save_image
# The filename is constructed as: f"{context.asset_rule.name}_merged_{merge_rule.map_type}{Path(first_input_path).suffix}"
# In this case, first_input_path is '/fake/red.png', so suffix is '.png'
# Asset name is "MergeAsset"
expected_filename_part = f"{context.asset_rule.name}_merged_{merge_rule.map_type}.png"
saved_path_arg = mock_save_image.call_args[0][0]
assert expected_filename_part in str(saved_path_arg)
saved_data = mock_save_image.call_args[0][1]
assert saved_data.shape == (10, 10, 3)
assert np.all(saved_data[:,:,0] == 200) # Red channel
assert np.all(saved_data[:,:,1] == 100) # Green channel
assert np.all(saved_data[:,:,2] == 50) # Blue channel
assert merge_rule.id.hex in updated_context.merged_maps_details
details = updated_context.merged_maps_details[merge_rule.id.hex]
assert details['status'] == 'Processed'
# The temp_merged_file path will be under engine_temp_dir / asset_name / filename
assert f"{context.engine_temp_dir / context.asset_rule.name / expected_filename_part}" == details['temp_merged_file']
mock_log_error.assert_not_called()
mock_log_info.assert_any_call(f"Successfully merged map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'.")
# Unit tests will be added below this line
@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.resize_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_map_merging_channel_inversion(mock_log_error, mock_log_info, mock_load_image, mock_resize_image, mock_save_image):
stage = MapMergingStage()
# Input FileRule
input_id = uuid.uuid4()
processed_details = {
input_id.hex: {'temp_processed_file': '/fake/source.png', 'status': 'Processed', 'map_type': 'SOURCE_MAP'}
}
# Mock loaded image data (single channel for simplicity, to be inverted)
mock_source_data = np.array([[0, 100], [155, 255]], dtype=np.uint8)
mock_load_image.return_value = mock_source_data
# Merge Rule setup: one input, inverted, to one output channel
merge_inputs = [
create_mock_merge_input_channel(file_rule_id=input_id, source_channel=0, target_channel=0, invert=True)
]
merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1)
merge_rule_id = uuid.uuid4()
merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="Inverted_Gray", merge_settings=merge_settings)
context = create_map_merging_mock_context(
initial_file_rules=[merge_rule],
initial_processed_details=processed_details
)
mock_save_image.return_value = True
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(Path('/fake/source.png'))
mock_resize_image.assert_not_called()
mock_save_image.assert_called_once()
saved_data = mock_save_image.call_args[0][1]
assert saved_data.shape == (2, 2) # Grayscale output
# Expected inverted data: 255-original
expected_inverted_data = np.array([[255, 155], [100, 0]], dtype=np.uint8)
assert np.all(saved_data == expected_inverted_data)
assert merge_rule.id.hex in updated_context.merged_maps_details
details = updated_context.merged_maps_details[merge_rule.id.hex]
assert details['status'] == 'Processed'
assert "merged_Inverted_Gray" in details['temp_merged_file']
mock_log_error.assert_not_called()
mock_log_info.assert_any_call(f"Successfully merged map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'.")
@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image')
@mock.patch('logging.error')
def test_map_merging_input_map_missing(mock_log_error, mock_load_image, mock_save_image):
stage = MapMergingStage()
# Input FileRule ID that will be missing from processed_details
missing_input_id = uuid.uuid4()
# Merge Rule setup
merge_inputs = [
create_mock_merge_input_channel(file_rule_id=missing_input_id, source_channel=0, target_channel=0)
]
merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1)
merge_rule_id = uuid.uuid4()
merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="TestMissing", merge_settings=merge_settings)
# processed_details is empty, so missing_input_id will not be found
context = create_map_merging_mock_context(
initial_file_rules=[merge_rule],
initial_processed_details={}
)
updated_context = stage.execute(context)
mock_load_image.assert_not_called()
mock_save_image.assert_not_called()
assert merge_rule.id.hex in updated_context.merged_maps_details
details = updated_context.merged_maps_details[merge_rule.id.hex]
assert details['status'] == 'Failed'
assert 'error_message' in details
assert f"Input map FileRule ID {missing_input_id.hex} not found in processed_maps_details or not successfully processed" in details['error_message']
mock_log_error.assert_called_once()
assert f"Failed to merge map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'" in mock_log_error.call_args[0][0]
assert f"Input map FileRule ID {missing_input_id.hex} not found in processed_maps_details or not successfully processed" in mock_log_error.call_args[0][0]
@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image')
@mock.patch('logging.error')
def test_map_merging_input_map_status_not_processed(mock_log_error, mock_load_image, mock_save_image):
stage = MapMergingStage()
input_id = uuid.uuid4()
processed_details = {
# Status is 'Failed', not 'Processed'
input_id.hex: {'temp_processed_file': '/fake/source.png', 'status': 'Failed', 'map_type': 'SOURCE_MAP'}
}
merge_inputs = [
create_mock_merge_input_channel(file_rule_id=input_id, source_channel=0, target_channel=0)
]
merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1)
merge_rule_id = uuid.uuid4()
merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="TestNotProcessed", merge_settings=merge_settings)
context = create_map_merging_mock_context(
initial_file_rules=[merge_rule],
initial_processed_details=processed_details
)
updated_context = stage.execute(context)
mock_load_image.assert_not_called()
mock_save_image.assert_not_called()
assert merge_rule.id.hex in updated_context.merged_maps_details
details = updated_context.merged_maps_details[merge_rule.id.hex]
assert details['status'] == 'Failed'
assert 'error_message' in details
assert f"Input map FileRule ID {input_id.hex} not found in processed_maps_details or not successfully processed" in details['error_message']
mock_log_error.assert_called_once()
assert f"Failed to merge map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'" in mock_log_error.call_args[0][0]
assert f"Input map FileRule ID {input_id.hex} not found in processed_maps_details or not successfully processed" in mock_log_error.call_args[0][0]
@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image')
@mock.patch('logging.error')
def test_map_merging_load_image_fails(mock_log_error, mock_load_image, mock_save_image):
stage = MapMergingStage()
input_id = uuid.uuid4()
processed_details = {
input_id.hex: {'temp_processed_file': '/fake/source.png', 'status': 'Processed', 'map_type': 'SOURCE_MAP'}
}
# Configure mock_load_image to raise an exception
mock_load_image.side_effect = Exception("Failed to load image")
merge_inputs = [
create_mock_merge_input_channel(file_rule_id=input_id, source_channel=0, target_channel=0)
]
merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1)
merge_rule_id = uuid.uuid4()
merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="TestLoadFail", merge_settings=merge_settings)
context = create_map_merging_mock_context(
initial_file_rules=[merge_rule],
initial_processed_details=processed_details
)
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(Path('/fake/source.png'))
mock_save_image.assert_not_called()
assert merge_rule.id.hex in updated_context.merged_maps_details
details = updated_context.merged_maps_details[merge_rule.id.hex]
assert details['status'] == 'Failed'
assert 'error_message' in details
assert "Failed to load image for merge input" in details['error_message']
assert str(Path('/fake/source.png')) in details['error_message']
mock_log_error.assert_called_once()
assert f"Failed to merge map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'" in mock_log_error.call_args[0][0]
assert "Failed to load image for merge input" in mock_log_error.call_args[0][0]
@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image')
@mock.patch('logging.error')
def test_map_merging_save_image_fails(mock_log_error, mock_load_image, mock_save_image):
stage = MapMergingStage()
input_id = uuid.uuid4()
processed_details = {
input_id.hex: {'temp_processed_file': '/fake/source.png', 'status': 'Processed', 'map_type': 'SOURCE_MAP'}
}
mock_source_data = np.full((10, 10), 128, dtype=np.uint8)
mock_load_image.return_value = mock_source_data
# Configure mock_save_image to return False (indicating failure)
mock_save_image.return_value = False
merge_inputs = [
create_mock_merge_input_channel(file_rule_id=input_id, source_channel=0, target_channel=0)
]
merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1)
merge_rule_id = uuid.uuid4()
merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="TestSaveFail", merge_settings=merge_settings)
context = create_map_merging_mock_context(
initial_file_rules=[merge_rule],
initial_processed_details=processed_details
)
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(Path('/fake/source.png'))
mock_save_image.assert_called_once() # save_image is called, but returns False
assert merge_rule.id.hex in updated_context.merged_maps_details
details = updated_context.merged_maps_details[merge_rule.id.hex]
assert details['status'] == 'Failed'
assert 'error_message' in details
assert "Failed to save merged map" in details['error_message']
mock_log_error.assert_called_once()
assert f"Failed to merge map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'" in mock_log_error.call_args[0][0]
assert "Failed to save merged map" in mock_log_error.call_args[0][0]
@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.resize_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_map_merging_dimension_mismatch_handling(mock_log_error, mock_log_info, mock_load_image, mock_resize_image, mock_save_image):
stage = MapMergingStage()
# Input FileRules
id1, id2 = uuid.uuid4(), uuid.uuid4()
processed_details = {
id1.hex: {'temp_processed_file': '/fake/img1.png', 'status': 'Processed', 'map_type': 'IMG1_SRC'},
id2.hex: {'temp_processed_file': '/fake/img2.png', 'status': 'Processed', 'map_type': 'IMG2_SRC'}
}
# Mock loaded image data with different dimensions
mock_img1_data = np.full((10, 10), 100, dtype=np.uint8) # 10x10
mock_img2_data_original = np.full((5, 5), 200, dtype=np.uint8) # 5x5, will be resized
mock_load_image.side_effect = [mock_img1_data, mock_img2_data_original]
# Mock resize_image to return an image of the target dimensions
# For simplicity, it just creates a new array of the target size filled with a value.
mock_img2_data_resized = np.full((10, 10), 210, dtype=np.uint8) # Resized to 10x10
mock_resize_image.return_value = mock_img2_data_resized
# Merge Rule setup: two inputs, one output channel (e.g., averaging them)
# Target channel 0 for both, the stage should handle combining them if they map to the same target.
# However, the current stage logic for multiple inputs to the same target channel is to take the last one.
# Let's make them target different channels for a clearer test of resize.
merge_inputs = [
create_mock_merge_input_channel(file_rule_id=id1, source_channel=0, target_channel=0),
create_mock_merge_input_channel(file_rule_id=id2, source_channel=0, target_channel=1)
]
merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=2) # Outputting 2 channels
merge_rule_id = uuid.uuid4()
merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="ResizedMerge", merge_settings=merge_settings)
context = create_map_merging_mock_context(
initial_file_rules=[merge_rule],
initial_processed_details=processed_details
)
mock_save_image.return_value = True
updated_context = stage.execute(context)
assert mock_load_image.call_count == 2
mock_load_image.assert_any_call(Path('/fake/img1.png'))
mock_load_image.assert_any_call(Path('/fake/img2.png'))
# Assert resize_image was called for the second image to match the first's dimensions
mock_resize_image.assert_called_once()
# The first argument to resize_image is the image data, second is target_shape tuple (height, width)
# np.array_equal is needed for comparing numpy arrays in mock calls
assert np.array_equal(mock_resize_image.call_args[0][0], mock_img2_data_original)
assert mock_resize_image.call_args[0][1] == (10, 10)
mock_save_image.assert_called_once()
saved_data = mock_save_image.call_args[0][1]
assert saved_data.shape == (10, 10, 2) # 2 output channels
assert np.all(saved_data[:,:,0] == mock_img1_data) # First channel from img1
assert np.all(saved_data[:,:,1] == mock_img2_data_resized) # Second channel from resized img2
assert merge_rule.id.hex in updated_context.merged_maps_details
details = updated_context.merged_maps_details[merge_rule.id.hex]
assert details['status'] == 'Processed'
assert "merged_ResizedMerge" in details['temp_merged_file']
mock_log_error.assert_not_called()
mock_log_info.assert_any_call(f"Resized input map from {Path('/fake/img2.png')} from {mock_img2_data_original.shape} to {(10,10)} to match first loaded map.")
mock_log_info.assert_any_call(f"Successfully merged map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'.")
@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.resize_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_map_merging_to_grayscale_output(mock_log_error, mock_log_info, mock_load_image, mock_resize_image, mock_save_image):
stage = MapMergingStage()
# Input FileRule (e.g., an RGB image)
input_id = uuid.uuid4()
processed_details = {
input_id.hex: {'temp_processed_file': '/fake/rgb_source.png', 'status': 'Processed', 'map_type': 'RGB_SRC'}
}
# Mock loaded image data (3 channels)
mock_rgb_data = np.full((10, 10, 3), [50, 100, 150], dtype=np.uint8)
mock_load_image.return_value = mock_rgb_data
# Merge Rule setup: take the Green channel (source_channel=1) from input and map it to the single output channel (target_channel=0)
merge_inputs = [
create_mock_merge_input_channel(file_rule_id=input_id, source_channel=1, target_channel=0) # G to Grayscale
]
# output_channels = 1 for grayscale
merge_settings = create_mock_merge_settings(input_maps=merge_inputs, output_channels=1)
merge_rule_id = uuid.uuid4()
merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="GrayscaleFromGreen", merge_settings=merge_settings)
context = create_map_merging_mock_context(
initial_file_rules=[merge_rule],
initial_processed_details=processed_details
)
mock_save_image.return_value = True
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(Path('/fake/rgb_source.png'))
mock_resize_image.assert_not_called()
mock_save_image.assert_called_once()
saved_data = mock_save_image.call_args[0][1]
assert saved_data.shape == (10, 10) # Grayscale output (2D)
assert np.all(saved_data == 100) # Green channel's value
assert merge_rule.id.hex in updated_context.merged_maps_details
details = updated_context.merged_maps_details[merge_rule.id.hex]
assert details['status'] == 'Processed'
assert "merged_GrayscaleFromGreen" in details['temp_merged_file']
mock_log_error.assert_not_called()
mock_log_info.assert_any_call(f"Successfully merged map '{merge_rule.map_type}' for asset '{context.asset_rule.name}'.")
@mock.patch('processing.pipeline.stages.map_merging.ipu.save_image')
@mock.patch('processing.pipeline.stages.map_merging.ipu.load_image')
@mock.patch('logging.error')
def test_map_merging_default_value_if_missing_channel(mock_log_error, mock_load_image, mock_save_image):
stage = MapMergingStage()
input_id = uuid.uuid4()
processed_details = {
# Input is a grayscale image (1 channel)
input_id.hex: {'temp_processed_file': '/fake/gray_source.png', 'status': 'Processed', 'map_type': 'GRAY_SRC'}
}
mock_gray_data = np.full((10, 10), 50, dtype=np.uint8)
mock_load_image.return_value = mock_gray_data
# Merge Rule: try to read source_channel 1 (which doesn't exist in grayscale)
# and use default_value_if_missing for target_channel 0.
# Also, read source_channel 0 (which exists) for target_channel 1.
mic1 = create_mock_merge_input_channel(file_rule_id=input_id, source_channel=1, target_channel=0)
mic1.default_value_if_missing = 128 # Set a specific default value
mic2 = create_mock_merge_input_channel(file_rule_id=input_id, source_channel=0, target_channel=1)
merge_settings = create_mock_merge_settings(input_maps=[mic1, mic2], output_channels=2)
merge_rule_id = uuid.uuid4()
merge_rule = create_mock_file_rule_for_merging(id_val=merge_rule_id, map_type="DefaultValueTest", merge_settings=merge_settings)
context = create_map_merging_mock_context(
initial_file_rules=[merge_rule],
initial_processed_details=processed_details
)
mock_save_image.return_value = True
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(Path('/fake/gray_source.png'))
mock_save_image.assert_called_once()
saved_data = mock_save_image.call_args[0][1]
assert saved_data.shape == (10, 10, 2)
assert np.all(saved_data[:,:,0] == 128) # Default value for missing source channel 1
assert np.all(saved_data[:,:,1] == 50) # Value from existing source channel 0
assert merge_rule.id.hex in updated_context.merged_maps_details
details = updated_context.merged_maps_details[merge_rule.id.hex]
assert details['status'] == 'Processed'
mock_log_error.assert_not_called()

View File

@@ -0,0 +1,359 @@
import pytest
from unittest import mock
from pathlib import Path
import datetime
import json # For comparing dumped content
import uuid
from typing import Optional, Dict, Any
from processing.pipeline.stages.metadata_finalization_save import MetadataFinalizationAndSaveStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule
from configuration import Configuration, GeneralSettings # Added GeneralSettings as it's in the helper
def create_metadata_save_mock_context(
status_flags: Optional[Dict[str, Any]] = None,
initial_asset_metadata: Optional[Dict[str, Any]] = None,
processed_details: Optional[Dict[str, Any]] = None,
merged_details: Optional[Dict[str, Any]] = None,
asset_name: str = "MetaSaveAsset",
output_path_pattern_val: str = "{asset_name}/metadata/{filename}",
# ... other common context fields ...
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_asset_rule.output_path_pattern = output_path_pattern_val
mock_asset_rule.id = uuid.uuid4() # Needed for generate_path_from_pattern if it uses it
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_source_rule.name = "MetaSaveSource"
mock_config = mock.MagicMock(spec=Configuration)
# mock_config.general_settings = mock.MagicMock(spec=GeneralSettings) # If needed
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp_engine_dir"),
output_base_path=Path("/fake/output_base"), # For generate_path
effective_supplier="ValidSupplier",
asset_metadata=initial_asset_metadata if initial_asset_metadata is not None else {},
processed_maps_details=processed_details if processed_details is not None else {},
merged_maps_details=merged_details if merged_details is not None else {},
files_to_process=[],
loaded_data_cache={},
config_obj=mock_config,
status_flags=status_flags if status_flags is not None else {},
incrementing_value="001", # Example for path generation
sha5_value="abc" # Example for path generation
)
return context
@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump')
@mock.patch('builtins.open', new_callable=mock.mock_open)
@mock.patch('pathlib.Path.mkdir')
@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern')
@mock.patch('datetime.datetime')
def test_asset_skipped_before_metadata_init(
mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump
):
"""
Tests that if an asset is marked for skipping and has no initial metadata,
the stage returns early without attempting to save metadata.
"""
stage = MetadataFinalizationAndSaveStage()
context = create_metadata_save_mock_context(
status_flags={'skip_asset': True},
initial_asset_metadata={} # Explicitly empty
)
updated_context = stage.execute(context)
# Assert that no processing or saving attempts were made
mock_dt.now.assert_not_called() # Should not even try to set end time if no metadata
mock_gen_path.assert_not_called()
mock_mkdir.assert_not_called()
mock_file_open.assert_not_called()
mock_json_dump.assert_not_called()
assert updated_context.asset_metadata == {} # Metadata remains empty
assert 'metadata_file_path' not in updated_context.asset_metadata
assert updated_context.status_flags.get('metadata_save_error') is None
@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump')
@mock.patch('builtins.open', new_callable=mock.mock_open)
@mock.patch('pathlib.Path.mkdir')
@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern')
@mock.patch('datetime.datetime')
def test_asset_skipped_after_metadata_init(
mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump
):
"""
Tests that if an asset is marked for skipping but has initial metadata,
the status is updated to 'Skipped' and metadata is saved.
"""
stage = MetadataFinalizationAndSaveStage()
fixed_now = datetime.datetime(2023, 1, 1, 12, 0, 0)
mock_dt.now.return_value = fixed_now
fake_metadata_path_str = "/fake/output_base/SkippedAsset/metadata/SkippedAsset_metadata.json"
mock_gen_path.return_value = fake_metadata_path_str
initial_meta = {'asset_name': "SkippedAsset", 'status': "Pending"}
context = create_metadata_save_mock_context(
asset_name="SkippedAsset",
status_flags={'skip_asset': True},
initial_asset_metadata=initial_meta
)
updated_context = stage.execute(context)
mock_dt.now.assert_called_once()
mock_gen_path.assert_called_once_with(
context.asset_rule.output_path_pattern,
context.asset_rule,
context.source_rule,
context.output_base_path,
context.asset_metadata, # Original metadata passed for path gen
context.incrementing_value,
context.sha5_value,
filename_override=f"{context.asset_rule.name}_metadata.json"
)
mock_mkdir.assert_called_once_with(parents=True, exist_ok=True)
mock_file_open.assert_called_once_with(Path(fake_metadata_path_str), 'w')
mock_json_dump.assert_called_once()
dumped_data = mock_json_dump.call_args[0][0]
assert dumped_data['status'] == "Skipped"
assert dumped_data['processing_end_time'] == fixed_now.isoformat()
assert 'processed_map_details' not in dumped_data # Should not be present if skipped early
assert 'merged_map_details' not in dumped_data # Should not be present if skipped early
assert updated_context.asset_metadata['status'] == "Skipped"
assert updated_context.asset_metadata['processing_end_time'] == fixed_now.isoformat()
assert updated_context.asset_metadata['metadata_file_path'] == fake_metadata_path_str
assert updated_context.status_flags.get('metadata_save_error') is None
@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump')
@mock.patch('builtins.open', new_callable=mock.mock_open) # Mocks open()
@mock.patch('pathlib.Path.mkdir')
@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern')
@mock.patch('datetime.datetime')
def test_metadata_save_success(mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump):
"""
Tests successful metadata finalization and saving, including serialization of Path objects.
"""
stage = MetadataFinalizationAndSaveStage()
fixed_now = datetime.datetime(2023, 1, 1, 12, 30, 0)
mock_dt.now.return_value = fixed_now
fake_metadata_path_str = "/fake/output_base/MetaSaveAsset/metadata/MetaSaveAsset_metadata.json"
mock_gen_path.return_value = fake_metadata_path_str
initial_meta = {'asset_name': "MetaSaveAsset", 'status': "Pending", 'processing_start_time': "2023-01-01T12:00:00"}
# Example of a Path object that needs serialization
proc_details = {'map1': {'temp_processed_file': Path('/fake/temp_engine_dir/map1.png'), 'final_file_path': Path('/fake/output_base/MetaSaveAsset/map1.png')}}
merged_details = {'merged_map_A': {'output_path': Path('/fake/output_base/MetaSaveAsset/merged_A.png')}}
context = create_metadata_save_mock_context(
initial_asset_metadata=initial_meta,
processed_details=proc_details,
merged_details=merged_details,
status_flags={} # No errors, no skip
)
updated_context = stage.execute(context)
mock_dt.now.assert_called_once()
mock_gen_path.assert_called_once_with(
context.asset_rule.output_path_pattern,
context.asset_rule,
context.source_rule,
context.output_base_path,
context.asset_metadata, # The metadata *before* adding end_time, status etc.
context.incrementing_value,
context.sha5_value,
filename_override=f"{context.asset_rule.name}_metadata.json"
)
mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) # Checks parent dir of fake_metadata_path_str
mock_file_open.assert_called_once_with(Path(fake_metadata_path_str), 'w')
mock_json_dump.assert_called_once()
# Check what was passed to json.dump
dumped_data = mock_json_dump.call_args[0][0]
assert dumped_data['status'] == "Processed"
assert dumped_data['processing_end_time'] == fixed_now.isoformat()
assert 'processing_start_time' in dumped_data # Ensure existing fields are preserved
# Verify processed_map_details and Path serialization
assert 'processed_map_details' in dumped_data
assert dumped_data['processed_map_details']['map1']['temp_processed_file'] == '/fake/temp_engine_dir/map1.png'
assert dumped_data['processed_map_details']['map1']['final_file_path'] == '/fake/output_base/MetaSaveAsset/map1.png'
# Verify merged_map_details and Path serialization
assert 'merged_map_details' in dumped_data
assert dumped_data['merged_map_details']['merged_map_A']['output_path'] == '/fake/output_base/MetaSaveAsset/merged_A.png'
assert updated_context.asset_metadata['metadata_file_path'] == fake_metadata_path_str
assert updated_context.asset_metadata['status'] == "Processed"
assert updated_context.status_flags.get('metadata_save_error') is None
@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump')
@mock.patch('builtins.open', new_callable=mock.mock_open)
@mock.patch('pathlib.Path.mkdir')
@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern')
@mock.patch('datetime.datetime')
def test_processing_failed_due_to_previous_error(
mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump
):
"""
Tests that if a previous stage set an error flag, the status is 'Failed'
and metadata (including any existing details) is saved.
"""
stage = MetadataFinalizationAndSaveStage()
fixed_now = datetime.datetime(2023, 1, 1, 12, 45, 0)
mock_dt.now.return_value = fixed_now
fake_metadata_path_str = "/fake/output_base/FailedAsset/metadata/FailedAsset_metadata.json"
mock_gen_path.return_value = fake_metadata_path_str
initial_meta = {'asset_name': "FailedAsset", 'status': "Processing"}
# Simulate some details might exist even if a later stage failed
proc_details = {'map1_partial': {'temp_processed_file': Path('/fake/temp_engine_dir/map1_partial.png')}}
context = create_metadata_save_mock_context(
asset_name="FailedAsset",
initial_asset_metadata=initial_meta,
processed_details=proc_details,
merged_details={}, # No merged details if processing failed before that
status_flags={'file_processing_error': True, 'error_message': "Something went wrong"}
)
updated_context = stage.execute(context)
mock_dt.now.assert_called_once()
mock_gen_path.assert_called_once() # Path generation should still occur
mock_mkdir.assert_called_once_with(parents=True, exist_ok=True)
mock_file_open.assert_called_once_with(Path(fake_metadata_path_str), 'w')
mock_json_dump.assert_called_once()
dumped_data = mock_json_dump.call_args[0][0]
assert dumped_data['status'] == "Failed"
assert dumped_data['processing_end_time'] == fixed_now.isoformat()
assert 'error_message' in dumped_data # Assuming error messages from status_flags are copied
assert dumped_data['error_message'] == "Something went wrong"
# Check that existing details are included
assert 'processed_map_details' in dumped_data
assert dumped_data['processed_map_details']['map1_partial']['temp_processed_file'] == '/fake/temp_engine_dir/map1_partial.png'
assert 'merged_map_details' in dumped_data # Should be present, even if empty
assert dumped_data['merged_map_details'] == {}
assert updated_context.asset_metadata['status'] == "Failed"
assert updated_context.asset_metadata['metadata_file_path'] == fake_metadata_path_str
assert updated_context.status_flags.get('metadata_save_error') is None
# Ensure the original error flag is preserved
assert updated_context.status_flags['file_processing_error'] is True
@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump')
@mock.patch('builtins.open', new_callable=mock.mock_open)
@mock.patch('pathlib.Path.mkdir')
@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern')
@mock.patch('datetime.datetime')
@mock.patch('logging.error') # To check if error is logged
def test_generate_path_fails(
mock_log_error, mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump
):
"""
Tests behavior when generate_path_from_pattern raises an exception.
Ensures status is updated, error flag is set, and no save is attempted.
"""
stage = MetadataFinalizationAndSaveStage()
fixed_now = datetime.datetime(2023, 1, 1, 12, 50, 0)
mock_dt.now.return_value = fixed_now
mock_gen_path.side_effect = Exception("Simulated path generation error")
initial_meta = {'asset_name': "PathFailAsset", 'status': "Processing"}
context = create_metadata_save_mock_context(
asset_name="PathFailAsset",
initial_asset_metadata=initial_meta,
status_flags={}
)
updated_context = stage.execute(context)
mock_dt.now.assert_called_once() # Time is set before path generation
mock_gen_path.assert_called_once() # generate_path_from_pattern is called
# File operations should NOT be called if path generation fails
mock_mkdir.assert_not_called()
mock_file_open.assert_not_called()
mock_json_dump.assert_not_called()
mock_log_error.assert_called_once() # Check that an error was logged
# Example: check if the log message contains relevant info, if needed
# assert "Failed to generate metadata path" in mock_log_error.call_args[0][0]
assert updated_context.asset_metadata['status'] == "Failed" # Or a more specific error status
assert 'processing_end_time' in updated_context.asset_metadata # End time should still be set
assert updated_context.asset_metadata['processing_end_time'] == fixed_now.isoformat()
assert 'metadata_file_path' not in updated_context.asset_metadata # Path should not be set
assert updated_context.status_flags.get('metadata_save_error') is True
assert 'error_message' in updated_context.asset_metadata # Check if error message is populated
assert "Simulated path generation error" in updated_context.asset_metadata['error_message']
@mock.patch('processing.pipeline.stages.metadata_finalization_save.json.dump')
@mock.patch('builtins.open', new_callable=mock.mock_open)
@mock.patch('pathlib.Path.mkdir')
@mock.patch('processing.pipeline.stages.metadata_finalization_save.generate_path_from_pattern')
@mock.patch('datetime.datetime')
@mock.patch('logging.error') # To check if error is logged
def test_json_dump_fails(
mock_log_error, mock_dt, mock_gen_path, mock_mkdir, mock_file_open, mock_json_dump
):
"""
Tests behavior when json.dump raises an exception during saving.
Ensures status is updated, error flag is set, and error is logged.
"""
stage = MetadataFinalizationAndSaveStage()
fixed_now = datetime.datetime(2023, 1, 1, 12, 55, 0)
mock_dt.now.return_value = fixed_now
fake_metadata_path_str = "/fake/output_base/JsonDumpFailAsset/metadata/JsonDumpFailAsset_metadata.json"
mock_gen_path.return_value = fake_metadata_path_str
mock_json_dump.side_effect = IOError("Simulated JSON dump error") # Or TypeError for non-serializable
initial_meta = {'asset_name': "JsonDumpFailAsset", 'status': "Processing"}
context = create_metadata_save_mock_context(
asset_name="JsonDumpFailAsset",
initial_asset_metadata=initial_meta,
status_flags={}
)
updated_context = stage.execute(context)
mock_dt.now.assert_called_once()
mock_gen_path.assert_called_once()
mock_mkdir.assert_called_once_with(parents=True, exist_ok=True)
mock_file_open.assert_called_once_with(Path(fake_metadata_path_str), 'w')
mock_json_dump.assert_called_once() # json.dump was attempted
mock_log_error.assert_called_once()
# assert "Failed to save metadata JSON" in mock_log_error.call_args[0][0]
assert updated_context.asset_metadata['status'] == "Failed" # Or specific "Metadata Save Failed"
assert 'processing_end_time' in updated_context.asset_metadata
assert updated_context.asset_metadata['processing_end_time'] == fixed_now.isoformat()
# metadata_file_path might be set if path generation succeeded, even if dump failed.
# Depending on desired behavior, this could be asserted or not.
# For now, let's assume it's set if path generation was successful.
assert updated_context.asset_metadata['metadata_file_path'] == fake_metadata_path_str
assert updated_context.status_flags.get('metadata_save_error') is True
assert 'error_message' in updated_context.asset_metadata
assert "Simulated JSON dump error" in updated_context.asset_metadata['error_message']

View File

@@ -0,0 +1,169 @@
import pytest
from unittest import mock
from pathlib import Path
import datetime
import uuid
from typing import Optional
from processing.pipeline.stages.metadata_initialization import MetadataInitializationStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule
from configuration import Configuration, GeneralSettings
# Helper function to create a mock AssetProcessingContext
def create_metadata_init_mock_context(
skip_asset_flag: bool = False,
asset_name: str = "MetaAsset",
asset_id: uuid.UUID = None, # Allow None to default to uuid.uuid4()
source_path_str: str = "source/meta_asset",
output_pattern: str = "{asset_name}/{map_type}",
tags: list = None,
custom_fields: dict = None,
source_rule_name: str = "MetaSource",
source_rule_id: uuid.UUID = None, # Allow None to default to uuid.uuid4()
eff_supplier: Optional[str] = "SupplierMeta",
app_version_str: str = "1.0.0-test",
inc_val: Optional[str] = None,
sha_val: Optional[str] = None
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_asset_rule.id = asset_id if asset_id is not None else uuid.uuid4()
mock_asset_rule.source_path = Path(source_path_str)
mock_asset_rule.output_path_pattern = output_pattern
mock_asset_rule.tags = tags if tags is not None else ["tag1", "test_tag"]
mock_asset_rule.custom_fields = custom_fields if custom_fields is not None else {"custom_key": "custom_value"}
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_source_rule.name = source_rule_name
mock_source_rule.id = source_rule_id if source_rule_id is not None else uuid.uuid4()
mock_general_settings = mock.MagicMock(spec=GeneralSettings)
mock_general_settings.app_version = app_version_str
mock_config = mock.MagicMock(spec=Configuration)
mock_config.general_settings = mock_general_settings
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp"),
output_base_path=Path("/fake/output"),
effective_supplier=eff_supplier,
asset_metadata={},
processed_maps_details={},
merged_maps_details={},
files_to_process=[],
loaded_data_cache={},
config_obj=mock_config,
status_flags={'skip_asset': skip_asset_flag},
incrementing_value=inc_val,
sha5_value=sha_val
)
return context
@mock.patch('processing.pipeline.stages.metadata_initialization.datetime')
def test_metadata_initialization_not_skipped(mock_datetime_module):
stage = MetadataInitializationStage()
fixed_now = datetime.datetime(2023, 10, 26, 12, 0, 0, tzinfo=datetime.timezone.utc)
mock_datetime_module.datetime.now.return_value = fixed_now
asset_id_val = uuid.uuid4()
source_id_val = uuid.uuid4()
context = create_metadata_init_mock_context(
skip_asset_flag=False,
asset_id=asset_id_val,
source_rule_id=source_id_val,
inc_val="001",
sha_val="abcde"
)
updated_context = stage.execute(context)
assert isinstance(updated_context.asset_metadata, dict)
assert isinstance(updated_context.processed_maps_details, dict)
assert isinstance(updated_context.merged_maps_details, dict)
md = updated_context.asset_metadata
assert md['asset_name'] == "MetaAsset"
assert md['asset_id'] == str(asset_id_val)
assert md['source_rule_name'] == "MetaSource"
assert md['source_rule_id'] == str(source_id_val)
assert md['source_path'] == "source/meta_asset"
assert md['effective_supplier'] == "SupplierMeta"
assert md['output_path_pattern'] == "{asset_name}/{map_type}"
assert md['processing_start_time'] == fixed_now.isoformat()
assert md['status'] == "Pending"
assert md['version'] == "1.0.0-test"
assert md['tags'] == ["tag1", "test_tag"]
assert md['custom_fields'] == {"custom_key": "custom_value"}
assert md['incrementing_value'] == "001"
assert md['sha5_value'] == "abcde"
@mock.patch('processing.pipeline.stages.metadata_initialization.datetime')
def test_metadata_initialization_not_skipped_none_inc_sha(mock_datetime_module):
stage = MetadataInitializationStage()
fixed_now = datetime.datetime(2023, 10, 26, 12, 0, 0, tzinfo=datetime.timezone.utc)
mock_datetime_module.datetime.now.return_value = fixed_now
context = create_metadata_init_mock_context(
skip_asset_flag=False,
inc_val=None,
sha_val=None
)
updated_context = stage.execute(context)
md = updated_context.asset_metadata
assert 'incrementing_value' not in md # Or assert md['incrementing_value'] is None, depending on desired behavior
assert 'sha5_value' not in md # Or assert md['sha5_value'] is None
def test_metadata_initialization_skipped():
stage = MetadataInitializationStage()
context = create_metadata_init_mock_context(skip_asset_flag=True)
# Make copies of initial state to ensure they are not modified
initial_asset_metadata = dict(context.asset_metadata)
initial_processed_maps = dict(context.processed_maps_details)
initial_merged_maps = dict(context.merged_maps_details)
updated_context = stage.execute(context)
assert updated_context.asset_metadata == initial_asset_metadata
assert updated_context.processed_maps_details == initial_processed_maps
assert updated_context.merged_maps_details == initial_merged_maps
assert not updated_context.asset_metadata # Explicitly check it's empty as per initial setup
assert not updated_context.processed_maps_details
assert not updated_context.merged_maps_details
@mock.patch('processing.pipeline.stages.metadata_initialization.datetime')
def test_tags_and_custom_fields_are_copies(mock_datetime_module):
stage = MetadataInitializationStage()
fixed_now = datetime.datetime(2023, 10, 26, 12, 0, 0, tzinfo=datetime.timezone.utc)
mock_datetime_module.datetime.now.return_value = fixed_now
original_tags = ["original_tag"]
original_custom_fields = {"original_key": "original_value"}
context = create_metadata_init_mock_context(
skip_asset_flag=False,
tags=original_tags,
custom_fields=original_custom_fields
)
# Modify originals after context creation but before stage execution
original_tags.append("modified_after_creation")
original_custom_fields["new_key_after_creation"] = "new_value"
updated_context = stage.execute(context)
md = updated_context.asset_metadata
assert md['tags'] == ["original_tag"] # Should not have "modified_after_creation"
assert md['tags'] is not original_tags # Ensure it's a different object
assert md['custom_fields'] == {"original_key": "original_value"} # Should not have "new_key_after_creation"
assert md['custom_fields'] is not original_custom_fields # Ensure it's a different object

View File

@@ -0,0 +1,323 @@
import pytest
from unittest import mock
from pathlib import Path
import uuid
import numpy as np
import logging # Added for mocking logger
from processing.pipeline.stages.normal_map_green_channel import NormalMapGreenChannelStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule, FileRule
from configuration import Configuration, GeneralSettings
# Helper functions
def create_mock_file_rule_for_normal_test(
id_val: uuid.UUID = None, # Corrected type hint from Optional[uuid.UUID]
map_type: str = "NORMAL",
filename_pattern: str = "normal.png"
) -> mock.MagicMock:
mock_fr = mock.MagicMock(spec=FileRule)
mock_fr.id = id_val if id_val else uuid.uuid4()
mock_fr.map_type = map_type
mock_fr.filename_pattern = filename_pattern
mock_fr.item_type = "MAP_COL" # As per example, though not directly used by stage
mock_fr.active = True # As per example
return mock_fr
def create_normal_map_mock_context(
initial_file_rules: list = None, # Corrected type hint
initial_processed_details: dict = None, # Corrected type hint
invert_green_globally: bool = False,
skip_asset_flag: bool = False,
asset_name: str = "NormalMapAsset"
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_gs = mock.MagicMock(spec=GeneralSettings)
mock_gs.invert_normal_map_green_channel_globally = invert_green_globally
mock_config = mock.MagicMock(spec=Configuration)
mock_config.general_settings = mock_gs
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp_engine_dir"),
output_base_path=Path("/fake/output"),
effective_supplier="ValidSupplier",
asset_metadata={'asset_name': asset_name},
processed_maps_details=initial_processed_details if initial_processed_details is not None else {},
merged_maps_details={},
files_to_process=list(initial_file_rules) if initial_file_rules else [],
loaded_data_cache={},
config_obj=mock_config,
status_flags={'skip_asset': skip_asset_flag},
incrementing_value=None, # Added as per AssetProcessingContext constructor
sha5_value=None # Added as per AssetProcessingContext constructor
)
return context
# Unit tests will be added below
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image')
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image')
def test_asset_skipped(mock_load_image, mock_save_image):
stage = NormalMapGreenChannelStage()
normal_fr = create_mock_file_rule_for_normal_test(map_type="NORMAL")
initial_details = {
normal_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_normal.png', 'status': 'Processed', 'map_type': 'NORMAL', 'notes': ''}
}
context = create_normal_map_mock_context(
initial_file_rules=[normal_fr],
initial_processed_details=initial_details,
invert_green_globally=True,
skip_asset_flag=True # Asset is skipped
)
original_details = context.processed_maps_details.copy()
updated_context = stage.execute(context)
mock_load_image.assert_not_called()
mock_save_image.assert_not_called()
assert updated_context.processed_maps_details == original_details
assert normal_fr in updated_context.files_to_process # Ensure rule is still there
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image')
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image')
def test_no_normal_map_present(mock_load_image, mock_save_image):
stage = NormalMapGreenChannelStage()
# Create a non-normal map rule
diffuse_fr = create_mock_file_rule_for_normal_test(map_type="DIFFUSE", filename_pattern="diffuse.png")
initial_details = {
diffuse_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_diffuse.png', 'status': 'Processed', 'map_type': 'DIFFUSE', 'notes': ''}
}
context = create_normal_map_mock_context(
initial_file_rules=[diffuse_fr],
initial_processed_details=initial_details,
invert_green_globally=True # Inversion enabled, but no normal map
)
original_details = context.processed_maps_details.copy()
updated_context = stage.execute(context)
mock_load_image.assert_not_called()
mock_save_image.assert_not_called()
assert updated_context.processed_maps_details == original_details
assert diffuse_fr in updated_context.files_to_process
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image')
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image')
def test_normal_map_present_inversion_disabled(mock_load_image, mock_save_image):
stage = NormalMapGreenChannelStage()
normal_rule_id = uuid.uuid4()
normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL")
initial_details = {
normal_fr.id.hex: {'temp_processed_file': '/fake/temp_engine_dir/processed_normal.png', 'status': 'Processed', 'map_type': 'NORMAL', 'notes': 'Initial note'}
}
context = create_normal_map_mock_context(
initial_file_rules=[normal_fr],
initial_processed_details=initial_details,
invert_green_globally=False # Inversion disabled
)
original_details_entry = context.processed_maps_details[normal_fr.id.hex].copy()
updated_context = stage.execute(context)
mock_load_image.assert_not_called()
mock_save_image.assert_not_called()
assert updated_context.processed_maps_details[normal_fr.id.hex] == original_details_entry
assert normal_fr in updated_context.files_to_process
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image')
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image')
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_normal_map_inversion_uint8_success(mock_log_debug, mock_log_info, mock_load_image, mock_save_image):
stage = NormalMapGreenChannelStage()
normal_rule_id = uuid.uuid4()
normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL")
initial_temp_path = Path('/fake/temp_engine_dir/processed_normal.png')
initial_details = {
normal_fr.id.hex: {'temp_processed_file': str(initial_temp_path), 'status': 'Processed', 'map_type': 'NORMAL', 'notes': 'Initial note'}
}
context = create_normal_map_mock_context(
initial_file_rules=[normal_fr],
initial_processed_details=initial_details,
invert_green_globally=True # Enable inversion
)
# R=10, G=50, B=100
mock_loaded_normal_data = np.array([[[10, 50, 100]]], dtype=np.uint8)
mock_load_image.return_value = mock_loaded_normal_data
mock_save_image.return_value = True # Simulate successful save
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(initial_temp_path)
# Check that save_image was called with green channel inverted
assert mock_save_image.call_count == 1
saved_path_arg, saved_data_arg = mock_save_image.call_args[0]
assert saved_data_arg[0,0,0] == 10 # R unchanged
assert saved_data_arg[0,0,1] == 255 - 50 # G inverted
assert saved_data_arg[0,0,2] == 100 # B unchanged
assert isinstance(saved_path_arg, Path)
assert "normal_g_inv_" in saved_path_arg.name
assert saved_path_arg.parent == initial_temp_path.parent # Should be in same temp dir
normal_detail = updated_context.processed_maps_details[normal_fr.id.hex]
assert "normal_g_inv_" in normal_detail['temp_processed_file']
assert Path(normal_detail['temp_processed_file']).name == saved_path_arg.name
assert "Green channel inverted" in normal_detail['notes']
assert "Initial note" in normal_detail['notes'] # Check existing notes preserved
assert normal_fr in updated_context.files_to_process
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image')
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image')
@mock.patch('logging.info')
@mock.patch('logging.debug')
def test_normal_map_inversion_float_success(mock_log_debug, mock_log_info, mock_load_image, mock_save_image):
stage = NormalMapGreenChannelStage()
normal_rule_id = uuid.uuid4()
normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL")
initial_temp_path = Path('/fake/temp_engine_dir/processed_normal_float.png')
initial_details = {
normal_fr.id.hex: {'temp_processed_file': str(initial_temp_path), 'status': 'Processed', 'map_type': 'NORMAL', 'notes': 'Float image'}
}
context = create_normal_map_mock_context(
initial_file_rules=[normal_fr],
initial_processed_details=initial_details,
invert_green_globally=True
)
# R=0.1, G=0.25, B=0.75
mock_loaded_normal_data = np.array([[[0.1, 0.25, 0.75]]], dtype=np.float32)
mock_load_image.return_value = mock_loaded_normal_data
mock_save_image.return_value = True
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(initial_temp_path)
assert mock_save_image.call_count == 1
saved_path_arg, saved_data_arg = mock_save_image.call_args[0]
assert np.isclose(saved_data_arg[0,0,0], 0.1) # R unchanged
assert np.isclose(saved_data_arg[0,0,1], 1.0 - 0.25) # G inverted
assert np.isclose(saved_data_arg[0,0,2], 0.75) # B unchanged
assert "normal_g_inv_" in saved_path_arg.name
normal_detail = updated_context.processed_maps_details[normal_fr.id.hex]
assert "normal_g_inv_" in normal_detail['temp_processed_file']
assert "Green channel inverted" in normal_detail['notes']
assert "Float image" in normal_detail['notes']
assert normal_fr in updated_context.files_to_process
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image')
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image')
@mock.patch('logging.error')
def test_load_image_fails(mock_log_error, mock_load_image, mock_save_image):
stage = NormalMapGreenChannelStage()
normal_rule_id = uuid.uuid4()
normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL")
initial_temp_path_str = '/fake/temp_engine_dir/processed_normal_load_fail.png'
initial_details = {
normal_fr.id.hex: {'temp_processed_file': initial_temp_path_str, 'status': 'Processed', 'map_type': 'NORMAL', 'notes': 'Load fail test'}
}
context = create_normal_map_mock_context(
initial_file_rules=[normal_fr],
initial_processed_details=initial_details,
invert_green_globally=True
)
original_details_entry = context.processed_maps_details[normal_fr.id.hex].copy()
mock_load_image.return_value = None # Simulate load failure
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(Path(initial_temp_path_str))
mock_save_image.assert_not_called()
mock_log_error.assert_called_once()
assert f"Failed to load image {Path(initial_temp_path_str)} for green channel inversion." in mock_log_error.call_args[0][0]
# Details should be unchanged
assert updated_context.processed_maps_details[normal_fr.id.hex] == original_details_entry
assert normal_fr in updated_context.files_to_process
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image')
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image')
@mock.patch('logging.error')
def test_save_image_fails(mock_log_error, mock_load_image, mock_save_image):
stage = NormalMapGreenChannelStage()
normal_rule_id = uuid.uuid4()
normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL")
initial_temp_path = Path('/fake/temp_engine_dir/processed_normal_save_fail.png')
initial_details = {
normal_fr.id.hex: {'temp_processed_file': str(initial_temp_path), 'status': 'Processed', 'map_type': 'NORMAL', 'notes': 'Save fail test'}
}
context = create_normal_map_mock_context(
initial_file_rules=[normal_fr],
initial_processed_details=initial_details,
invert_green_globally=True
)
original_details_entry = context.processed_maps_details[normal_fr.id.hex].copy()
mock_loaded_normal_data = np.array([[[10, 50, 100]]], dtype=np.uint8)
mock_load_image.return_value = mock_loaded_normal_data
mock_save_image.return_value = False # Simulate save failure
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(initial_temp_path)
mock_save_image.assert_called_once() # Save is attempted
saved_path_arg = mock_save_image.call_args[0][0] # Get the path it tried to save to
mock_log_error.assert_called_once()
assert f"Failed to save green channel inverted image to {saved_path_arg}." in mock_log_error.call_args[0][0]
# Details should be unchanged
assert updated_context.processed_maps_details[normal_fr.id.hex] == original_details_entry
assert normal_fr in updated_context.files_to_process
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.save_image')
@mock.patch('processing.pipeline.stages.normal_map_green_channel.ipu.load_image')
@mock.patch('logging.error')
@pytest.mark.parametrize("unsuitable_data, description", [
(np.array([[1, 2], [3, 4]], dtype=np.uint8), "2D array"), # 2D array
(np.array([[[1, 2]]], dtype=np.uint8), "2-channel image") # Image with less than 3 channels
])
def test_image_not_suitable_for_inversion(mock_log_error, mock_load_image, mock_save_image, unsuitable_data, description):
stage = NormalMapGreenChannelStage()
normal_rule_id = uuid.uuid4()
normal_fr = create_mock_file_rule_for_normal_test(id_val=normal_rule_id, map_type="NORMAL")
initial_temp_path_str = f'/fake/temp_engine_dir/unsuitable_{description.replace(" ", "_")}.png'
initial_details = {
normal_fr.id.hex: {'temp_processed_file': initial_temp_path_str, 'status': 'Processed', 'map_type': 'NORMAL', 'notes': f'Unsuitable: {description}'}
}
context = create_normal_map_mock_context(
initial_file_rules=[normal_fr],
initial_processed_details=initial_details,
invert_green_globally=True
)
original_details_entry = context.processed_maps_details[normal_fr.id.hex].copy()
mock_load_image.return_value = unsuitable_data
updated_context = stage.execute(context)
mock_load_image.assert_called_once_with(Path(initial_temp_path_str))
mock_save_image.assert_not_called() # Save should not be attempted
mock_log_error.assert_called_once()
assert f"Image at {Path(initial_temp_path_str)} is not suitable for green channel inversion (e.g., not RGB/RGBA)." in mock_log_error.call_args[0][0]
# Details should be unchanged
assert updated_context.processed_maps_details[normal_fr.id.hex] == original_details_entry
assert normal_fr in updated_context.files_to_process

View File

@@ -0,0 +1,417 @@
import pytest
from unittest import mock
from pathlib import Path
import shutil # To check if shutil.copy2 is called
import uuid
from typing import Optional # Added for type hinting in helper
from processing.pipeline.stages.output_organization import OutputOrganizationStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule, FileRule # For context setup
from configuration import Configuration, GeneralSettings
def create_output_org_mock_context(
status_flags: Optional[dict] = None,
asset_metadata_status: str = "Processed", # Default to processed for testing copy
processed_map_details: Optional[dict] = None,
merged_map_details: Optional[dict] = None,
overwrite_setting: bool = False,
asset_name: str = "OutputOrgAsset",
output_path_pattern_val: str = "{asset_name}/{map_type}/{filename}"
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_asset_rule.output_path_pattern = output_path_pattern_val
# Need FileRules on AssetRule if stage tries to look up output_filename_pattern from them
# For simplicity, assume stage constructs output_filename for now if not found on FileRule
mock_asset_rule.file_rules = [] # Or mock FileRules if stage uses them for output_filename_pattern
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_source_rule.name = "OutputOrgSource"
mock_gs = mock.MagicMock(spec=GeneralSettings)
mock_gs.overwrite_existing = overwrite_setting
mock_config = mock.MagicMock(spec=Configuration)
mock_config.general_settings = mock_gs
# Ensure asset_metadata has a status
initial_asset_metadata = {'asset_name': asset_name, 'status': asset_metadata_status}
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp_engine_dir"),
output_base_path=Path("/fake/output_final"),
effective_supplier="ValidSupplier",
asset_metadata=initial_asset_metadata,
processed_maps_details=processed_map_details if processed_map_details is not None else {},
merged_maps_details=merged_map_details if merged_map_details is not None else {},
files_to_process=[], # Not directly used by this stage, but good to have
loaded_data_cache={},
config_obj=mock_config,
status_flags=status_flags if status_flags is not None else {},
incrementing_value="001",
sha5_value="xyz" # Corrected from sha5_value to sha256_value if that's the actual param, or ensure it's a valid param. Assuming sha5_value is a typo and should be something like 'unique_id' or similar if not sha256. For now, keeping as sha5_value as per instructions.
)
return context
@mock.patch('shutil.copy2')
@mock.patch('logging.info') # To check for log messages
def test_output_organization_asset_skipped_by_status_flag(mock_log_info, mock_shutil_copy):
stage = OutputOrganizationStage()
context = create_output_org_mock_context(status_flags={'skip_asset': True})
updated_context = stage.execute(context)
mock_shutil_copy.assert_not_called()
# Check if a log message indicates skipping, if applicable
# e.g., mock_log_info.assert_any_call("Skipping output organization for asset OutputOrgAsset due to skip_asset flag.")
assert 'final_output_files' not in updated_context.asset_metadata # Or assert it's empty
assert updated_context.asset_metadata['status'] == "Processed" # Status should not change if skipped due to flag before stage logic
# Add specific log check if the stage logs this event
# For now, assume no copy is the primary check
@mock.patch('shutil.copy2')
@mock.patch('logging.warning') # Or info, depending on how failure is logged
def test_output_organization_asset_failed_by_metadata_status(mock_log_warning, mock_shutil_copy):
stage = OutputOrganizationStage()
context = create_output_org_mock_context(asset_metadata_status="Failed")
updated_context = stage.execute(context)
mock_shutil_copy.assert_not_called()
# Check for a log message indicating skipping due to failure status
# e.g., mock_log_warning.assert_any_call("Skipping output organization for asset OutputOrgAsset as its status is Failed.")
assert 'final_output_files' not in updated_context.asset_metadata # Or assert it's empty
assert updated_context.asset_metadata['status'] == "Failed" # Status remains Failed
@mock.patch('shutil.copy2')
@mock.patch('pathlib.Path.mkdir')
@mock.patch('pathlib.Path.exists')
@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_output_organization_success_no_overwrite(
mock_log_error, mock_log_info, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy
):
stage = OutputOrganizationStage()
proc_id_1 = uuid.uuid4().hex
merged_id_1 = uuid.uuid4().hex
processed_details = {
proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc1.png', 'map_type': 'Diffuse', 'output_filename': 'OutputOrgAsset_Diffuse.png'}
}
merged_details = {
merged_id_1: {'status': 'Processed', 'temp_merged_file': '/fake/temp_engine_dir/merged1.png', 'map_type': 'ORM', 'output_filename': 'OutputOrgAsset_ORM.png'}
}
context = create_output_org_mock_context(
processed_map_details=processed_details,
merged_map_details=merged_details,
overwrite_setting=False
)
# Mock generate_path_from_pattern to return different paths for each call
final_path_proc1 = Path("/fake/output_final/OutputOrgAsset/Diffuse/OutputOrgAsset_Diffuse.png")
final_path_merged1 = Path("/fake/output_final/OutputOrgAsset/ORM/OutputOrgAsset_ORM.png")
# Ensure generate_path_from_pattern is called with the correct context and details
# The actual call in the stage is: generate_path_from_pattern(context, map_detail, map_type_key, temp_file_key)
# We need to ensure our side_effect matches these calls.
def gen_path_side_effect(ctx, detail, map_type_key, temp_file_key, output_filename_key):
if detail['temp_processed_file'] == '/fake/temp_engine_dir/proc1.png':
return final_path_proc1
elif detail['temp_merged_file'] == '/fake/temp_engine_dir/merged1.png':
return final_path_merged1
raise ValueError("Unexpected call to generate_path_from_pattern")
mock_gen_path.side_effect = gen_path_side_effect
mock_path_exists.return_value = False # Files do not exist at destination
updated_context = stage.execute(context)
assert mock_shutil_copy.call_count == 2
mock_shutil_copy.assert_any_call(Path(processed_details[proc_id_1]['temp_processed_file']), final_path_proc1)
mock_shutil_copy.assert_any_call(Path(merged_details[merged_id_1]['temp_merged_file']), final_path_merged1)
# Check mkdir calls
# It should be called for each unique parent directory
expected_mkdir_calls = [
mock.call(Path("/fake/output_final/OutputOrgAsset/Diffuse"), parents=True, exist_ok=True),
mock.call(Path("/fake/output_final/OutputOrgAsset/ORM"), parents=True, exist_ok=True)
]
mock_mkdir.assert_has_calls(expected_mkdir_calls, any_order=True)
# Ensure mkdir was called for the parent of each file
assert mock_mkdir.call_count >= 1 # Could be 1 or 2 if paths share a base that's created once
assert len(updated_context.asset_metadata['final_output_files']) == 2
assert str(final_path_proc1) in updated_context.asset_metadata['final_output_files']
assert str(final_path_merged1) in updated_context.asset_metadata['final_output_files']
assert updated_context.processed_maps_details[proc_id_1]['final_output_path'] == str(final_path_proc1)
assert updated_context.merged_maps_details[merged_id_1]['final_output_path'] == str(final_path_merged1)
mock_log_error.assert_not_called()
# Check for specific info logs if necessary
# mock_log_info.assert_any_call(f"Copying {processed_details[proc_id_1]['temp_processed_file']} to {final_path_proc1}")
# mock_log_info.assert_any_call(f"Copying {merged_details[merged_id_1]['temp_merged_file']} to {final_path_merged1}")
@mock.patch('shutil.copy2')
@mock.patch('pathlib.Path.mkdir') # Still might be called if other files are processed
@mock.patch('pathlib.Path.exists')
@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern')
@mock.patch('logging.info')
def test_output_organization_overwrite_disabled_file_exists(
mock_log_info, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy
):
stage = OutputOrganizationStage()
proc_id_1 = uuid.uuid4().hex
processed_details = {
proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_exists.png', 'map_type': 'Diffuse', 'output_filename': 'OutputOrgAsset_Diffuse_Exists.png'}
}
context = create_output_org_mock_context(
processed_map_details=processed_details,
overwrite_setting=False
)
final_path_proc1 = Path("/fake/output_final/OutputOrgAsset/Diffuse/OutputOrgAsset_Diffuse_Exists.png")
mock_gen_path.return_value = final_path_proc1 # Only one file
mock_path_exists.return_value = True # File exists at destination
updated_context = stage.execute(context)
mock_shutil_copy.assert_not_called()
mock_log_info.assert_any_call(
f"Skipping copy for {final_path_proc1} as it already exists and overwrite is disabled."
)
# final_output_files should still be populated if the file exists and is considered "organized"
assert str(final_path_proc1) in updated_context.asset_metadata['final_output_files']
assert updated_context.processed_maps_details[proc_id_1]['final_output_path'] == str(final_path_proc1)
@mock.patch('shutil.copy2')
@mock.patch('pathlib.Path.mkdir')
@mock.patch('pathlib.Path.exists')
@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern')
@mock.patch('logging.info')
@mock.patch('logging.error')
def test_output_organization_overwrite_enabled_file_exists(
mock_log_error, mock_log_info, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy
):
stage = OutputOrganizationStage()
proc_id_1 = uuid.uuid4().hex
processed_details = {
proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_overwrite.png', 'map_type': 'Diffuse', 'output_filename': 'OutputOrgAsset_Diffuse_Overwrite.png'}
}
context = create_output_org_mock_context(
processed_map_details=processed_details,
overwrite_setting=True # Overwrite is enabled
)
final_path_proc1 = Path("/fake/output_final/OutputOrgAsset/Diffuse/OutputOrgAsset_Diffuse_Overwrite.png")
mock_gen_path.return_value = final_path_proc1
mock_path_exists.return_value = True # File exists, but we should overwrite
updated_context = stage.execute(context)
mock_shutil_copy.assert_called_once_with(Path(processed_details[proc_id_1]['temp_processed_file']), final_path_proc1)
mock_mkdir.assert_called_once_with(final_path_proc1.parent, parents=True, exist_ok=True)
assert str(final_path_proc1) in updated_context.asset_metadata['final_output_files']
assert updated_context.processed_maps_details[proc_id_1]['final_output_path'] == str(final_path_proc1)
mock_log_error.assert_not_called()
# Optionally check for a log message indicating overwrite, if implemented
# mock_log_info.assert_any_call(f"Overwriting existing file {final_path_proc1}...")
@mock.patch('shutil.copy2')
@mock.patch('pathlib.Path.mkdir')
@mock.patch('pathlib.Path.exists')
@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern')
@mock.patch('logging.error')
def test_output_organization_only_processed_maps(
mock_log_error, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy
):
stage = OutputOrganizationStage()
proc_id_1 = uuid.uuid4().hex
processed_details = {
proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_only.png', 'map_type': 'Albedo', 'output_filename': 'OutputOrgAsset_Albedo.png'}
}
context = create_output_org_mock_context(
processed_map_details=processed_details,
merged_map_details={}, # No merged maps
overwrite_setting=False
)
final_path_proc1 = Path("/fake/output_final/OutputOrgAsset/Albedo/OutputOrgAsset_Albedo.png")
mock_gen_path.return_value = final_path_proc1
mock_path_exists.return_value = False
updated_context = stage.execute(context)
mock_shutil_copy.assert_called_once_with(Path(processed_details[proc_id_1]['temp_processed_file']), final_path_proc1)
mock_mkdir.assert_called_once_with(final_path_proc1.parent, parents=True, exist_ok=True)
assert len(updated_context.asset_metadata['final_output_files']) == 1
assert str(final_path_proc1) in updated_context.asset_metadata['final_output_files']
assert updated_context.processed_maps_details[proc_id_1]['final_output_path'] == str(final_path_proc1)
assert not updated_context.merged_maps_details # Should remain empty
mock_log_error.assert_not_called()
@mock.patch('shutil.copy2')
@mock.patch('pathlib.Path.mkdir')
@mock.patch('pathlib.Path.exists')
@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern')
@mock.patch('logging.error')
def test_output_organization_only_merged_maps(
mock_log_error, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy
):
stage = OutputOrganizationStage()
merged_id_1 = uuid.uuid4().hex
merged_details = {
merged_id_1: {'status': 'Processed', 'temp_merged_file': '/fake/temp_engine_dir/merged_only.png', 'map_type': 'Metallic', 'output_filename': 'OutputOrgAsset_Metallic.png'}
}
context = create_output_org_mock_context(
processed_map_details={}, # No processed maps
merged_map_details=merged_details,
overwrite_setting=False
)
final_path_merged1 = Path("/fake/output_final/OutputOrgAsset/Metallic/OutputOrgAsset_Metallic.png")
mock_gen_path.return_value = final_path_merged1
mock_path_exists.return_value = False
updated_context = stage.execute(context)
mock_shutil_copy.assert_called_once_with(Path(merged_details[merged_id_1]['temp_merged_file']), final_path_merged1)
mock_mkdir.assert_called_once_with(final_path_merged1.parent, parents=True, exist_ok=True)
assert len(updated_context.asset_metadata['final_output_files']) == 1
assert str(final_path_merged1) in updated_context.asset_metadata['final_output_files']
assert updated_context.merged_maps_details[merged_id_1]['final_output_path'] == str(final_path_merged1)
assert not updated_context.processed_maps_details # Should remain empty
mock_log_error.assert_not_called()
@mock.patch('shutil.copy2')
@mock.patch('pathlib.Path.mkdir')
@mock.patch('pathlib.Path.exists')
@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern')
@mock.patch('logging.warning') # Expect a warning for skipped map
@mock.patch('logging.error')
def test_output_organization_map_status_not_processed(
mock_log_error, mock_log_warning, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy
):
stage = OutputOrganizationStage()
proc_id_1_failed = uuid.uuid4().hex
proc_id_2_ok = uuid.uuid4().hex
processed_details = {
proc_id_1_failed: {'status': 'Failed', 'temp_processed_file': '/fake/temp_engine_dir/proc_failed.png', 'map_type': 'Diffuse', 'output_filename': 'OutputOrgAsset_Diffuse_Failed.png'},
proc_id_2_ok: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_ok.png', 'map_type': 'Normal', 'output_filename': 'OutputOrgAsset_Normal_OK.png'}
}
context = create_output_org_mock_context(
processed_map_details=processed_details,
overwrite_setting=False
)
final_path_proc_ok = Path("/fake/output_final/OutputOrgAsset/Normal/OutputOrgAsset_Normal_OK.png")
# generate_path_from_pattern should only be called for the 'Processed' map
mock_gen_path.return_value = final_path_proc_ok
mock_path_exists.return_value = False
updated_context = stage.execute(context)
# Assert copy was only called for the 'Processed' map
mock_shutil_copy.assert_called_once_with(Path(processed_details[proc_id_2_ok]['temp_processed_file']), final_path_proc_ok)
mock_mkdir.assert_called_once_with(final_path_proc_ok.parent, parents=True, exist_ok=True)
# Assert final_output_files only contains the successfully processed map
assert len(updated_context.asset_metadata['final_output_files']) == 1
assert str(final_path_proc_ok) in updated_context.asset_metadata['final_output_files']
# Assert final_output_path is set for the processed map
assert updated_context.processed_maps_details[proc_id_2_ok]['final_output_path'] == str(final_path_proc_ok)
# Assert final_output_path is NOT set for the failed map
assert 'final_output_path' not in updated_context.processed_maps_details[proc_id_1_failed]
mock_log_warning.assert_any_call(
f"Skipping output organization for map with ID {proc_id_1_failed} (type: Diffuse) as its status is 'Failed'."
)
mock_log_error.assert_not_called()
@mock.patch('shutil.copy2')
@mock.patch('pathlib.Path.mkdir')
@mock.patch('pathlib.Path.exists')
@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern')
@mock.patch('logging.error')
def test_output_organization_generate_path_fails(
mock_log_error, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy
):
stage = OutputOrganizationStage()
proc_id_1 = uuid.uuid4().hex
processed_details = {
proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_path_fail.png', 'map_type': 'Roughness', 'output_filename': 'OutputOrgAsset_Roughness_PathFail.png'}
}
context = create_output_org_mock_context(
processed_map_details=processed_details,
overwrite_setting=False
)
mock_gen_path.side_effect = Exception("Simulated path generation error")
mock_path_exists.return_value = False # Should not matter if path gen fails
updated_context = stage.execute(context)
mock_shutil_copy.assert_not_called() # No copy if path generation fails
mock_mkdir.assert_not_called() # No mkdir if path generation fails
assert not updated_context.asset_metadata.get('final_output_files') # No files should be listed
assert 'final_output_path' not in updated_context.processed_maps_details[proc_id_1]
assert updated_context.status_flags.get('output_organization_error') is True
assert updated_context.asset_metadata['status'] == "Error" # Or "Failed" depending on desired behavior
mock_log_error.assert_any_call(
f"Error generating output path for map ID {proc_id_1} (type: Roughness): Simulated path generation error"
)
@mock.patch('shutil.copy2')
@mock.patch('pathlib.Path.mkdir')
@mock.patch('pathlib.Path.exists')
@mock.patch('processing.pipeline.stages.output_organization.generate_path_from_pattern')
@mock.patch('logging.error')
def test_output_organization_shutil_copy_fails(
mock_log_error, mock_gen_path, mock_path_exists, mock_mkdir, mock_shutil_copy
):
stage = OutputOrganizationStage()
proc_id_1 = uuid.uuid4().hex
processed_details = {
proc_id_1: {'status': 'Processed', 'temp_processed_file': '/fake/temp_engine_dir/proc_copy_fail.png', 'map_type': 'AO', 'output_filename': 'OutputOrgAsset_AO_CopyFail.png'}
}
context = create_output_org_mock_context(
processed_map_details=processed_details,
overwrite_setting=False
)
final_path_proc1 = Path("/fake/output_final/OutputOrgAsset/AO/OutputOrgAsset_AO_CopyFail.png")
mock_gen_path.return_value = final_path_proc1
mock_path_exists.return_value = False
mock_shutil_copy.side_effect = shutil.Error("Simulated copy error") # Can also be IOError, OSError
updated_context = stage.execute(context)
mock_mkdir.assert_called_once_with(final_path_proc1.parent, parents=True, exist_ok=True) # mkdir would be called before copy
mock_shutil_copy.assert_called_once_with(Path(processed_details[proc_id_1]['temp_processed_file']), final_path_proc1)
# Even if copy fails, the path might be added to final_output_files before the error is caught,
# or the design might be to not add it. Let's assume it's not added on error.
# Check the stage's actual behavior for this.
# If the intention is to record the *attempted* path, this assertion might change.
# For now, assume failure means it's not a "final" output.
assert not updated_context.asset_metadata.get('final_output_files')
assert 'final_output_path' not in updated_context.processed_maps_details[proc_id_1] # Or it might contain the path but status is error
assert updated_context.status_flags.get('output_organization_error') is True
assert updated_context.asset_metadata['status'] == "Error" # Or "Failed"
mock_log_error.assert_any_call(
f"Error copying file {processed_details[proc_id_1]['temp_processed_file']} to {final_path_proc1}: Simulated copy error"
)

View File

@@ -0,0 +1,213 @@
import pytest
from unittest import mock
from pathlib import Path
from typing import Dict, List, Optional, Any
# Assuming pytest is run from project root, adjust if necessary
from processing.pipeline.stages.supplier_determination import SupplierDeterminationStage
from processing.pipeline.asset_context import AssetProcessingContext
from rule_structure import AssetRule, SourceRule, FileRule # For constructing mock context
from configuration import Configuration, GeneralSettings, Supplier # For mock config
# Example helper (can be a pytest fixture too)
def create_mock_context(
asset_rule_supplier_override: Optional[str] = None,
source_rule_supplier: Optional[str] = None,
config_suppliers: Optional[Dict[str, Any]] = None, # Mocked Supplier objects or dicts
asset_name: str = "TestAsset"
) -> AssetProcessingContext:
mock_asset_rule = mock.MagicMock(spec=AssetRule)
mock_asset_rule.name = asset_name
mock_asset_rule.supplier_override = asset_rule_supplier_override
# ... other AssetRule fields if needed by the stage ...
mock_source_rule = mock.MagicMock(spec=SourceRule)
mock_source_rule.supplier = source_rule_supplier
# ... other SourceRule fields ...
mock_config = mock.MagicMock(spec=Configuration)
mock_config.suppliers = config_suppliers if config_suppliers is not None else {}
# Basic AssetProcessingContext fields
context = AssetProcessingContext(
source_rule=mock_source_rule,
asset_rule=mock_asset_rule,
workspace_path=Path("/fake/workspace"),
engine_temp_dir=Path("/fake/temp"),
output_base_path=Path("/fake/output"),
effective_supplier=None,
asset_metadata={},
processed_maps_details={},
merged_maps_details={},
files_to_process=[],
loaded_data_cache={},
config_obj=mock_config,
status_flags={},
incrementing_value=None,
sha5_value=None # Corrected from sha5_value to sha256_value if that's the actual field name
)
return context
@pytest.fixture
def supplier_stage():
return SupplierDeterminationStage()
@mock.patch('logging.error')
@mock.patch('logging.info')
def test_supplier_from_asset_rule_override_valid(mock_log_info, mock_log_error, supplier_stage):
mock_suppliers_config = {"SupplierA": mock.MagicMock(spec=Supplier)}
context = create_mock_context(
asset_rule_supplier_override="SupplierA",
config_suppliers=mock_suppliers_config
)
updated_context = supplier_stage.execute(context)
assert updated_context.effective_supplier == "SupplierA"
assert not updated_context.status_flags.get('supplier_error')
mock_log_info.assert_any_call("Effective supplier for asset 'TestAsset' set to 'SupplierA' from asset rule override.")
mock_log_error.assert_not_called()
@mock.patch('logging.error')
@mock.patch('logging.info')
def test_supplier_from_source_rule_fallback_valid(mock_log_info, mock_log_error, supplier_stage):
mock_suppliers_config = {"SupplierB": mock.MagicMock(spec=Supplier)}
context = create_mock_context(
asset_rule_supplier_override=None,
source_rule_supplier="SupplierB",
config_suppliers=mock_suppliers_config
)
updated_context = supplier_stage.execute(context)
assert updated_context.effective_supplier == "SupplierB"
assert not updated_context.status_flags.get('supplier_error')
mock_log_info.assert_any_call("Effective supplier for asset 'TestAsset' set to 'SupplierB' from source rule.")
mock_log_error.assert_not_called()
@mock.patch('logging.error')
@mock.patch('logging.warning') # supplier_determination uses logging.warning for invalid suppliers
def test_asset_rule_override_invalid_supplier(mock_log_warning, mock_log_error, supplier_stage):
context = create_mock_context(
asset_rule_supplier_override="InvalidSupplier",
config_suppliers={"SupplierA": mock.MagicMock(spec=Supplier)} # "InvalidSupplier" not in config
)
updated_context = supplier_stage.execute(context)
assert updated_context.effective_supplier is None
assert updated_context.status_flags.get('supplier_error') is True
mock_log_warning.assert_any_call(
"Asset 'TestAsset' has supplier_override 'InvalidSupplier' which is not defined in global suppliers. No supplier set."
)
mock_log_error.assert_not_called()
@mock.patch('logging.error')
@mock.patch('logging.warning')
def test_source_rule_fallback_invalid_supplier(mock_log_warning, mock_log_error, supplier_stage):
context = create_mock_context(
asset_rule_supplier_override=None,
source_rule_supplier="InvalidSupplierB",
config_suppliers={"SupplierA": mock.MagicMock(spec=Supplier)} # "InvalidSupplierB" not in config
)
updated_context = supplier_stage.execute(context)
assert updated_context.effective_supplier is None
assert updated_context.status_flags.get('supplier_error') is True
mock_log_warning.assert_any_call(
"Asset 'TestAsset' has source rule supplier 'InvalidSupplierB' which is not defined in global suppliers. No supplier set."
)
mock_log_error.assert_not_called()
@mock.patch('logging.error')
@mock.patch('logging.warning')
def test_no_supplier_defined(mock_log_warning, mock_log_error, supplier_stage):
context = create_mock_context(
asset_rule_supplier_override=None,
source_rule_supplier=None,
config_suppliers={"SupplierA": mock.MagicMock(spec=Supplier)}
)
updated_context = supplier_stage.execute(context)
assert updated_context.effective_supplier is None
assert updated_context.status_flags.get('supplier_error') is True
mock_log_warning.assert_any_call(
"No supplier could be determined for asset 'TestAsset'. "
"AssetRule override is None and SourceRule supplier is None or empty."
)
mock_log_error.assert_not_called()
@mock.patch('logging.error')
@mock.patch('logging.warning')
def test_empty_config_suppliers_with_asset_override(mock_log_warning, mock_log_error, supplier_stage):
context = create_mock_context(
asset_rule_supplier_override="SupplierX",
config_suppliers={} # Empty global supplier config
)
updated_context = supplier_stage.execute(context)
assert updated_context.effective_supplier is None
assert updated_context.status_flags.get('supplier_error') is True
mock_log_warning.assert_any_call(
"Asset 'TestAsset' has supplier_override 'SupplierX' which is not defined in global suppliers. No supplier set."
)
mock_log_error.assert_not_called()
@mock.patch('logging.error')
@mock.patch('logging.warning')
def test_empty_config_suppliers_with_source_rule(mock_log_warning, mock_log_error, supplier_stage):
context = create_mock_context(
source_rule_supplier="SupplierY",
config_suppliers={} # Empty global supplier config
)
updated_context = supplier_stage.execute(context)
assert updated_context.effective_supplier is None
assert updated_context.status_flags.get('supplier_error') is True
mock_log_warning.assert_any_call(
"Asset 'TestAsset' has source rule supplier 'SupplierY' which is not defined in global suppliers. No supplier set."
)
mock_log_error.assert_not_called()
@mock.patch('logging.error')
@mock.patch('logging.info')
def test_asset_rule_override_empty_string(mock_log_info, mock_log_error, supplier_stage):
# This scenario should fall back to source_rule.supplier if asset_rule.supplier_override is ""
mock_suppliers_config = {"SupplierB": mock.MagicMock(spec=Supplier)}
context = create_mock_context(
asset_rule_supplier_override="", # Empty string override
source_rule_supplier="SupplierB",
config_suppliers=mock_suppliers_config
)
updated_context = supplier_stage.execute(context)
assert updated_context.effective_supplier == "SupplierB" # Falls back to SourceRule
assert not updated_context.status_flags.get('supplier_error')
mock_log_info.assert_any_call("Effective supplier for asset 'TestAsset' set to 'SupplierB' from source rule.")
mock_log_error.assert_not_called()
@mock.patch('logging.error')
@mock.patch('logging.warning')
def test_source_rule_supplier_empty_string(mock_log_warning, mock_log_error, supplier_stage):
# This scenario should result in an error if asset_rule.supplier_override is None and source_rule.supplier is ""
context = create_mock_context(
asset_rule_supplier_override=None,
source_rule_supplier="", # Empty string source supplier
config_suppliers={"SupplierA": mock.MagicMock(spec=Supplier)}
)
updated_context = supplier_stage.execute(context)
assert updated_context.effective_supplier is None
assert updated_context.status_flags.get('supplier_error') is True
mock_log_warning.assert_any_call(
"No supplier could be determined for asset 'TestAsset'. "
"AssetRule override is None and SourceRule supplier is None or empty."
)
mock_log_error.assert_not_called()

View File

@@ -0,0 +1,383 @@
import pytest
from unittest import mock
from pathlib import Path
import uuid
import shutil # For checking rmtree
import tempfile # For mocking mkdtemp
from processing.pipeline.orchestrator import PipelineOrchestrator
from processing.pipeline.asset_context import AssetProcessingContext
from processing.pipeline.stages.base_stage import ProcessingStage # For mocking stages
from rule_structure import SourceRule, AssetRule, FileRule
from configuration import Configuration, GeneralSettings
# Mock Stage that modifies context
class MockPassThroughStage(ProcessingStage):
def __init__(self, stage_name="mock_stage"):
self.stage_name = stage_name
self.execute_call_count = 0
self.contexts_called_with = [] # To store contexts for verification
def execute(self, context: AssetProcessingContext) -> AssetProcessingContext:
self.execute_call_count += 1
self.contexts_called_with.append(context)
# Optionally, modify context for testing
context.asset_metadata[f'{self.stage_name}_executed'] = True
if self.stage_name == "skipper_stage": # Example conditional logic
context.status_flags['skip_asset'] = True
context.status_flags['skip_reason'] = "Skipped by skipper_stage"
elif self.stage_name == "error_stage": # Example error-raising stage
raise ValueError("Simulated error in error_stage")
# Simulate status update based on stage execution
if not context.status_flags.get('skip_asset') and not context.status_flags.get('asset_failed'):
context.asset_metadata['status'] = "Processed" # Default to processed if not skipped/failed
return context
def create_orchestrator_test_config() -> mock.MagicMock:
mock_config = mock.MagicMock(spec=Configuration)
mock_config.general_settings = mock.MagicMock(spec=GeneralSettings)
mock_config.general_settings.temp_dir_override = None # Default, can be overridden in tests
# Add other config details if orchestrator or stages depend on them directly
return mock_config
def create_orchestrator_test_asset_rule(name: str, num_file_rules: int = 1) -> mock.MagicMock:
asset_rule = mock.MagicMock(spec=AssetRule)
asset_rule.name = name
asset_rule.id = uuid.uuid4()
asset_rule.source_path = Path(f"/fake/source/{name}") # Using Path object
asset_rule.file_rules = [mock.MagicMock(spec=FileRule) for _ in range(num_file_rules)]
asset_rule.enabled = True
asset_rule.map_types = {} # Initialize as dict
asset_rule.material_name_scheme = "{asset_name}"
asset_rule.texture_name_scheme = "{asset_name}_{map_type}"
asset_rule.output_path_scheme = "{source_name}/{asset_name}"
# ... other necessary AssetRule fields ...
return asset_rule
def create_orchestrator_test_source_rule(name: str, num_assets: int = 1, asset_names: list = None) -> mock.MagicMock:
source_rule = mock.MagicMock(spec=SourceRule)
source_rule.name = name
source_rule.id = uuid.uuid4()
if asset_names:
source_rule.assets = [create_orchestrator_test_asset_rule(an) for an in asset_names]
else:
source_rule.assets = [create_orchestrator_test_asset_rule(f"Asset_{i+1}_in_{name}") for i in range(num_assets)]
source_rule.enabled = True
source_rule.source_path = Path(f"/fake/source_root/{name}") # Using Path object
# ... other necessary SourceRule fields ...
return source_rule
# --- Test Cases for PipelineOrchestrator.process_source_rule() ---
@mock.patch('shutil.rmtree')
@mock.patch('tempfile.mkdtemp')
def test_orchestrator_basic_flow_mock_stages(mock_mkdtemp, mock_rmtree):
mock_mkdtemp.return_value = "/fake/engine_temp_dir_path" # Path for mkdtemp
config = create_orchestrator_test_config()
stage1 = MockPassThroughStage("stage1")
stage2 = MockPassThroughStage("stage2")
orchestrator = PipelineOrchestrator(config_obj=config, stages=[stage1, stage2])
source_rule = create_orchestrator_test_source_rule("MySourceRule", num_assets=2)
asset1_name = source_rule.assets[0].name
asset2_name = source_rule.assets[1].name
# Mock asset_metadata to be updated by stages for status check
# The MockPassThroughStage already sets a 'status' = "Processed" if not skipped/failed
# and adds '{stage_name}_executed' = True to asset_metadata.
results = orchestrator.process_source_rule(
source_rule, Path("/ws"), Path("/out"), False, "inc_val_123", "sha_val_abc"
)
assert stage1.execute_call_count == 2 # Called for each asset
assert stage2.execute_call_count == 2 # Called for each asset
assert asset1_name in results['processed']
assert asset2_name in results['processed']
assert not results['skipped']
assert not results['failed']
# Verify context modifications by stages
for i in range(2): # For each asset
# Stage 1 context checks
s1_context_asset = stage1.contexts_called_with[i]
assert s1_context_asset.asset_metadata.get('stage1_executed') is True
assert s1_context_asset.asset_metadata.get('stage2_executed') is None # Stage 2 not yet run for this asset
# Stage 2 context checks
s2_context_asset = stage2.contexts_called_with[i]
assert s2_context_asset.asset_metadata.get('stage1_executed') is True # From stage 1
assert s2_context_asset.asset_metadata.get('stage2_executed') is True
assert s2_context_asset.asset_metadata.get('status') == "Processed"
mock_mkdtemp.assert_called_once()
# The orchestrator creates a subdirectory within the mkdtemp path
expected_temp_path = Path(mock_mkdtemp.return_value) / source_rule.id.hex
mock_rmtree.assert_called_once_with(expected_temp_path, ignore_errors=True)
@mock.patch('shutil.rmtree')
@mock.patch('tempfile.mkdtemp')
def test_orchestrator_asset_skipping_by_stage(mock_mkdtemp, mock_rmtree):
mock_mkdtemp.return_value = "/fake/engine_temp_dir_path_skip"
config = create_orchestrator_test_config()
skipper_stage = MockPassThroughStage("skipper_stage") # This stage will set skip_asset = True
stage_after_skip = MockPassThroughStage("stage_after_skip")
orchestrator = PipelineOrchestrator(config_obj=config, stages=[skipper_stage, stage_after_skip])
source_rule = create_orchestrator_test_source_rule("SkipSourceRule", num_assets=1)
asset_to_skip_name = source_rule.assets[0].name
results = orchestrator.process_source_rule(
source_rule, Path("/ws_skip"), Path("/out_skip"), False, "inc_skip", "sha_skip"
)
assert skipper_stage.execute_call_count == 1 # Called for the asset
assert stage_after_skip.execute_call_count == 0 # Not called because asset was skipped
assert asset_to_skip_name in results['skipped']
assert not results['processed']
assert not results['failed']
# Verify skip reason in context if needed (MockPassThroughStage stores contexts)
skipped_context = skipper_stage.contexts_called_with[0]
assert skipped_context.status_flags['skip_asset'] is True
assert skipped_context.status_flags['skip_reason'] == "Skipped by skipper_stage"
mock_mkdtemp.assert_called_once()
expected_temp_path = Path(mock_mkdtemp.return_value) / source_rule.id.hex
mock_rmtree.assert_called_once_with(expected_temp_path, ignore_errors=True)
@mock.patch('shutil.rmtree')
@mock.patch('tempfile.mkdtemp')
def test_orchestrator_no_assets_in_source_rule(mock_mkdtemp, mock_rmtree):
mock_mkdtemp.return_value = "/fake/engine_temp_dir_no_assets"
config = create_orchestrator_test_config()
stage1 = MockPassThroughStage("stage1_no_assets")
orchestrator = PipelineOrchestrator(config_obj=config, stages=[stage1])
source_rule = create_orchestrator_test_source_rule("NoAssetSourceRule", num_assets=0)
results = orchestrator.process_source_rule(
source_rule, Path("/ws_no_assets"), Path("/out_no_assets"), False, "inc_no", "sha_no"
)
assert stage1.execute_call_count == 0
assert not results['processed']
assert not results['skipped']
assert not results['failed']
# mkdtemp should still be called for the source rule processing, even if no assets
mock_mkdtemp.assert_called_once()
expected_temp_path = Path(mock_mkdtemp.return_value) / source_rule.id.hex
mock_rmtree.assert_called_once_with(expected_temp_path, ignore_errors=True)
@mock.patch('shutil.rmtree')
@mock.patch('tempfile.mkdtemp')
def test_orchestrator_error_during_stage_execution(mock_mkdtemp, mock_rmtree):
mock_mkdtemp.return_value = "/fake/engine_temp_dir_error"
config = create_orchestrator_test_config()
error_stage = MockPassThroughStage("error_stage") # This stage will raise an error
stage_after_error = MockPassThroughStage("stage_after_error")
orchestrator = PipelineOrchestrator(config_obj=config, stages=[error_stage, stage_after_error])
# Test with two assets, one fails, one processes (if orchestrator continues)
# The current orchestrator's process_asset is per asset, so an error in one
# should not stop processing of other assets in the same source_rule.
source_rule = create_orchestrator_test_source_rule("ErrorSourceRule", asset_names=["AssetFails", "AssetSucceeds"])
asset_fails_name = source_rule.assets[0].name
asset_succeeds_name = source_rule.assets[1].name
# Make only the first asset's processing trigger the error
original_execute = error_stage.execute
def error_execute_side_effect(context: AssetProcessingContext):
if context.asset_rule.name == asset_fails_name:
# The MockPassThroughStage is already configured to raise ValueError for "error_stage"
# but we need to ensure it's only for the first asset.
# We can achieve this by modifying the stage_name temporarily or by checking asset_rule.name
# For simplicity, let's assume the mock stage's error logic is fine,
# and we just need to check the outcome.
# The error_stage will raise ValueError("Simulated error in error_stage")
# The orchestrator's _process_single_asset catches generic Exception.
return original_execute(context) # This will call the erroring logic
else:
# For the second asset, make it pass through without error
context.asset_metadata[f'{error_stage.stage_name}_executed'] = True
context.asset_metadata['status'] = "Processed"
return context
error_stage.execute = mock.MagicMock(side_effect=error_execute_side_effect)
# stage_after_error should still be called for the successful asset
results = orchestrator.process_source_rule(
source_rule, Path("/ws_error"), Path("/out_error"), False, "inc_err", "sha_err"
)
assert error_stage.execute.call_count == 2 # Called for both assets
# stage_after_error is only called for the asset that didn't fail in error_stage
assert stage_after_error.execute_call_count == 1
assert asset_fails_name in results['failed']
assert asset_succeeds_name in results['processed']
assert not results['skipped']
# Verify the context of the failed asset
failed_context = None
for ctx in error_stage.contexts_called_with:
if ctx.asset_rule.name == asset_fails_name:
failed_context = ctx
break
assert failed_context is not None
assert failed_context.status_flags['asset_failed'] is True
assert "Simulated error in error_stage" in failed_context.status_flags['failure_reason']
# Verify the context of the successful asset after stage_after_error
successful_context_after_s2 = None
for ctx in stage_after_error.contexts_called_with:
if ctx.asset_rule.name == asset_succeeds_name:
successful_context_after_s2 = ctx
break
assert successful_context_after_s2 is not None
assert successful_context_after_s2.asset_metadata.get('error_stage_executed') is True # from the non-erroring path
assert successful_context_after_s2.asset_metadata.get('stage_after_error_executed') is True
assert successful_context_after_s2.asset_metadata.get('status') == "Processed"
mock_mkdtemp.assert_called_once()
expected_temp_path = Path(mock_mkdtemp.return_value) / source_rule.id.hex
mock_rmtree.assert_called_once_with(expected_temp_path, ignore_errors=True)
@mock.patch('shutil.rmtree')
@mock.patch('tempfile.mkdtemp')
def test_orchestrator_asset_processing_context_initialization(mock_mkdtemp, mock_rmtree):
mock_engine_temp_dir = "/fake/engine_temp_dir_context_init"
mock_mkdtemp.return_value = mock_engine_temp_dir
config = create_orchestrator_test_config()
mock_stage = MockPassThroughStage("context_check_stage")
orchestrator = PipelineOrchestrator(config_obj=config, stages=[mock_stage])
source_rule = create_orchestrator_test_source_rule("ContextSourceRule", num_assets=1)
asset_rule = source_rule.assets[0]
workspace_path = Path("/ws_context")
output_base_path = Path("/out_context")
incrementing_value = "inc_context_123"
sha5_value = "sha_context_abc"
orchestrator.process_source_rule(
source_rule, workspace_path, output_base_path, False, incrementing_value, sha5_value
)
assert mock_stage.execute_call_count == 1
# Retrieve the context passed to the mock stage
captured_context = mock_stage.contexts_called_with[0]
assert captured_context.source_rule == source_rule
assert captured_context.asset_rule == asset_rule
assert captured_context.workspace_path == workspace_path
# engine_temp_dir for the asset is a sub-directory of the source_rule's temp dir
# which itself is a sub-directory of the main engine_temp_dir from mkdtemp
expected_source_rule_temp_dir = Path(mock_engine_temp_dir) / source_rule.id.hex
expected_asset_temp_dir = expected_source_rule_temp_dir / asset_rule.id.hex
assert captured_context.engine_temp_dir == expected_asset_temp_dir
assert captured_context.output_base_path == output_base_path
assert captured_context.config_obj == config
assert captured_context.incrementing_value == incrementing_value
assert captured_context.sha5_value == sha5_value
# Check initial state of other context fields
assert captured_context.asset_metadata == {} # Should be empty initially for an asset
assert captured_context.status_flags == {} # Should be empty initially
assert captured_context.shared_data == {} # Should be empty initially
assert captured_context.current_files == [] # Should be empty initially
mock_mkdtemp.assert_called_once()
mock_rmtree.assert_called_once_with(expected_source_rule_temp_dir, ignore_errors=True)
@mock.patch('shutil.rmtree')
@mock.patch('tempfile.mkdtemp')
def test_orchestrator_temp_dir_override_from_config(mock_mkdtemp, mock_rmtree):
# This test verifies that if config.general_settings.temp_dir_override is set,
# mkdtemp is NOT called, and the override path is used and cleaned up.
config = create_orchestrator_test_config()
override_temp_path_str = "/override/temp/path"
config.general_settings.temp_dir_override = override_temp_path_str
stage1 = MockPassThroughStage("stage_temp_override")
orchestrator = PipelineOrchestrator(config_obj=config, stages=[stage1])
source_rule = create_orchestrator_test_source_rule("TempOverrideRule", num_assets=1)
asset_rule = source_rule.assets[0]
results = orchestrator.process_source_rule(
source_rule, Path("/ws_override"), Path("/out_override"), False, "inc_override", "sha_override"
)
assert stage1.execute_call_count == 1
assert asset_rule.name in results['processed']
mock_mkdtemp.assert_not_called() # mkdtemp should not be called due to override
# The orchestrator should create its source-rule specific subdir within the override
expected_source_rule_temp_dir_in_override = Path(override_temp_path_str) / source_rule.id.hex
# Verify the context passed to the stage uses the overridden path structure
captured_context = stage1.contexts_called_with[0]
expected_asset_temp_dir_in_override = expected_source_rule_temp_dir_in_override / asset_rule.id.hex
assert captured_context.engine_temp_dir == expected_asset_temp_dir_in_override
# rmtree should be called on the source_rule's directory within the override path
mock_rmtree.assert_called_once_with(expected_source_rule_temp_dir_in_override, ignore_errors=True)
@mock.patch('shutil.rmtree')
@mock.patch('tempfile.mkdtemp')
def test_orchestrator_disabled_asset_rule_is_skipped(mock_mkdtemp, mock_rmtree):
mock_mkdtemp.return_value = "/fake/engine_temp_dir_disabled_asset"
config = create_orchestrator_test_config()
stage1 = MockPassThroughStage("stage_disabled_check")
orchestrator = PipelineOrchestrator(config_obj=config, stages=[stage1])
source_rule = create_orchestrator_test_source_rule("DisabledAssetSourceRule", asset_names=["EnabledAsset", "DisabledAsset"])
enabled_asset = source_rule.assets[0]
disabled_asset = source_rule.assets[1]
disabled_asset.enabled = False # Disable this asset rule
results = orchestrator.process_source_rule(
source_rule, Path("/ws_disabled"), Path("/out_disabled"), False, "inc_dis", "sha_dis"
)
assert stage1.execute_call_count == 1 # Only called for the enabled asset
assert enabled_asset.name in results['processed']
assert disabled_asset.name in results['skipped']
assert not results['failed']
# Verify context for the processed asset
assert stage1.contexts_called_with[0].asset_rule.name == enabled_asset.name
# Verify skip reason for the disabled asset (this is set by the orchestrator itself)
# The orchestrator's _process_single_asset checks asset_rule.enabled
# We need to inspect the results dictionary for the skip reason if it's stored there,
# or infer it. The current structure of `results` doesn't store detailed skip reasons directly,
# but the test ensures it's in the 'skipped' list.
# For a more detailed check, one might need to adjust how results are reported or mock deeper.
# For now, confirming it's in 'skipped' and stage1 wasn't called for it is sufficient.
mock_mkdtemp.assert_called_once()
expected_temp_path = Path(mock_mkdtemp.return_value) / source_rule.id.hex
mock_rmtree.assert_called_once_with(expected_temp_path, ignore_errors=True)