Rusfort/Asset-Frameworker
1
0
Fork 0
Code Issues 24 Pull Requests Actions Packages Projects 3 Releases Wiki Activity

Metadata reformatting

Browse Source
This commit is contained in:
Rusfort 2025-05-13 09:15:43 +02:00
parent f800bb25a9
commit 1cd81cb87a
2 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
processing/pipeline/stages/metadata_finalization_save.py
View File

@ -41,7 +41,7 @@ class MetadataFinalizationAndSaveStage(ProcessingStage):
# Check Skip Flag
if context.status_flags.get('skip_asset'):
context.asset_metadata['status'] = "Skipped"
context.asset_metadata['processing_end_time'] = datetime.datetime.now().isoformat()
# context.asset_metadata['processing_end_time'] = datetime.datetime.now().isoformat()
context.asset_metadata['notes'] = context.status_flags.get('skip_reason', 'Skipped early in pipeline')
logger.info(
f"Asset '{asset_name_for_log}': Marked as skipped. Reason: {context.asset_metadata['notes']}"
@ -51,7 +51,7 @@ class MetadataFinalizationAndSaveStage(ProcessingStage):
# However, if we are here, asset_metadata IS initialized.
# A. Finalize Metadata
context.asset_metadata['processing_end_time'] = datetime.datetime.now().isoformat()
# context.asset_metadata['processing_end_time'] = datetime.datetime.now().isoformat()
# Determine final status (if not already set to Skipped)
if context.asset_metadata.get('status') != "Skipped":
@ -115,8 +115,8 @@ class MetadataFinalizationAndSaveStage(ProcessingStage):
restructured_processed_maps[map_key] = new_map_entry
# Assign the restructured details. Note: 'processed_map_details' (singular 'map') is the key in asset_metadata.
context.asset_metadata['processed_map_details'] = restructured_processed_maps
context.asset_metadata['merged_map_details'] = getattr(context, 'merged_maps_details', {})
# context.asset_metadata['processed_map_details'] = restructured_processed_maps
# context.asset_metadata['merged_map_details'] = getattr(context, 'merged_maps_details', {})
# (Optional) Add a list of all temporary files
# context.asset_metadata['temporary_files'] = getattr(context, 'temporary_files', []) # Assuming this is populated elsewhere
@ -203,6 +203,9 @@ class MetadataFinalizationAndSaveStage(ProcessingStage):
return [make_serializable(i) for i in data]
return data
# final_output_files is populated by OutputOrganizationStage and might be desired.
# The 'maps' structure is now the primary source for map file paths in metadata.
# context.asset_metadata.pop('final_output_files', None)
serializable_metadata = make_serializable(context.asset_metadata)
with open(metadata_save_path, 'w') as f:

20
processing/pipeline/stages/output_organization.py
View File

@ -61,8 +61,10 @@ class OutputOrganizationStage(ProcessingStage):
if saved_files_info and isinstance(saved_files_info, list) and len(saved_files_info) > 0:
logger.debug(f"Asset '{asset_name_for_log}': Organizing {len(saved_files_info)} variants for map key '{processed_map_key}' (map type: {base_map_type}) from SaveVariantsStage.")
map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {})
map_metadata_entry['map_type'] = base_map_type
# Use base_map_type (e.g., "COL") as the key for the map entry
map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(base_map_type, {})
# map_type is now the key, so no need to store it inside the entry
# map_metadata_entry['map_type'] = base_map_type
map_metadata_entry.setdefault('variant_paths', {}) # Initialize if not present
processed_any_variant_successfully = False
@ -182,11 +184,15 @@ class OutputOrganizationStage(ProcessingStage):
details['final_output_path'] = str(final_path)
# Update asset_metadata for metadata.json
map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(processed_map_key, {})
map_metadata_entry['map_type'] = base_map_type
map_metadata_entry['path'] = str(Path(relative_dir_path_str) / Path(output_filename)) # Store relative path
if 'variant_paths' in map_metadata_entry: # Clean up variant paths if present from previous runs
del map_metadata_entry['variant_paths']
# Use base_map_type (e.g., "COL") as the key for the map entry
map_metadata_entry = context.asset_metadata.setdefault('maps', {}).setdefault(base_map_type, {})
# map_type is now the key, so no need to store it inside the entry
# map_metadata_entry['map_type'] = base_map_type
# Store single path in variant_paths, keyed by its resolution string
map_metadata_entry.setdefault('variant_paths', {})[resolution_str] = str(Path(relative_dir_path_str) / Path(output_filename))
# Remove old cleanup logic, as variant_paths is now the standard
# if 'variant_paths' in map_metadata_entry:
# del map_metadata_entry['variant_paths']
except Exception as e:
logger.error(f"Asset '{asset_name_for_log}': Failed to copy {temp_file_path} for map key '{processed_map_key}'. Error: {e}", exc_info=True)