diff --git a/processing/pipeline/stages/metadata_finalization_save.py b/processing/pipeline/stages/metadata_finalization_save.py index f2adb70..fc23833 100644 --- a/processing/pipeline/stages/metadata_finalization_save.py +++ b/processing/pipeline/stages/metadata_finalization_save.py @@ -41,7 +41,7 @@ class MetadataFinalizationAndSaveStage(ProcessingStage): # Check Skip Flag if context.status_flags.get('skip_asset'): context.asset_metadata['status'] = "Skipped" - context.asset_metadata['processing_end_time'] = datetime.datetime.now().isoformat() + # context.asset_metadata['processing_end_time'] = datetime.datetime.now().isoformat() context.asset_metadata['notes'] = context.status_flags.get('skip_reason', 'Skipped early in pipeline') logger.info( f"Asset '{asset_name_for_log}': Marked as skipped. Reason: {context.asset_metadata['notes']}" @@ -51,7 +51,7 @@ class MetadataFinalizationAndSaveStage(ProcessingStage): # However, if we are here, asset_metadata IS initialized. # A. Finalize Metadata - context.asset_metadata['processing_end_time'] = datetime.datetime.now().isoformat() + # context.asset_metadata['processing_end_time'] = datetime.datetime.now().isoformat() # Determine final status (if not already set to Skipped) if context.asset_metadata.get('status') != "Skipped": @@ -115,8 +115,8 @@ class MetadataFinalizationAndSaveStage(ProcessingStage): restructured_processed_maps[map_key] = new_map_entry # Assign the restructured details. Note: 'processed_map_details' (singular 'map') is the key in asset_metadata. - context.asset_metadata['processed_map_details'] = restructured_processed_maps - context.asset_metadata['merged_map_details'] = getattr(context, 'merged_maps_details', {}) + # context.asset_metadata['processed_map_details'] = restructured_processed_maps + # context.asset_metadata['merged_map_details'] = getattr(context, 'merged_maps_details', {}) # (Optional) Add a list of all temporary files # context.asset_metadata['temporary_files'] = getattr(context, 'temporary_files', []) # Assuming this is populated elsewhere @@ -203,6 +203,9 @@ class MetadataFinalizationAndSaveStage(ProcessingStage): return [make_serializable(i) for i in data] return data + # final_output_files is populated by OutputOrganizationStage and might be desired. + # The 'maps' structure is now the primary source for map file paths in metadata. + # context.asset_metadata.pop('final_output_files', None) serializable_metadata = make_serializable(context.asset_metadata) with open(metadata_save_path, 'w') as f: diff --git a/processing/pipeline/stages/output_organization.py b/processing/pipeline/stages/output_organization.py index f032eae..e299f3a 100644 --- a/processing/pipeline/stages/output_organization.py +++ b/processing/pipeline/stages/output_organization.py @@ -61,8 +61,10 @@ class OutputOrganizationStage(ProcessingStage): if saved_files_info and isinstance(saved_files_info, list) and len(saved_files_info) > 0: logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(saved_files_info)} variants for map key '{processed_map_key}' (map type: {base_map_type}) from SaveVariantsStage.") - map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {}) - map_metadata_entry['map_type'] = base_map_type + # Use base_map_type (e.g., "COL") as the key for the map entry + map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(base_map_type, {}) + # map_type is now the key, so no need to store it inside the entry + # map_metadata_entry['map_type'] = base_map_type map_metadata_entry.setdefault('variant_paths', {}) # Initialize if not present processed_any_variant_successfully = False @@ -182,11 +184,15 @@ class OutputOrganizationStage(ProcessingStage): details['final_output_path'] = str(final_path) # Update asset_metadata for metadata.json - map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {}) - map_metadata_entry['map_type'] = base_map_type - map_metadata_entry['path'] = str(Path(relative_dir_path_str) / Path(output_filename)) # Store relative path - if 'variant_paths' in map_metadata_entry: # Clean up variant paths if present from previous runs - del map_metadata_entry['variant_paths'] + # Use base_map_type (e.g., "COL") as the key for the map entry + map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(base_map_type, {}) + # map_type is now the key, so no need to store it inside the entry + # map_metadata_entry['map_type'] = base_map_type + # Store single path in variant_paths, keyed by its resolution string + map_metadata_entry.setdefault('variant_paths', {})[resolution_str] = str(Path(relative_dir_path_str) / Path(output_filename)) + # Remove old cleanup logic, as variant_paths is now the standard + # if 'variant_paths' in map_metadata_entry: + # del map_metadata_entry['variant_paths'] except Exception as e: logger.error(f"Asset '{asset_name_for_log}': Failed to copy {temp_file_path} for map key '{processed_map_key}'. Error: {e}", exc_info=True)