Asset-Frameworker/configuration.py

582 lines
26 KiB
Python

import json
import os
from pathlib import Path
import logging
import re
log = logging.getLogger(__name__)
BASE_DIR = Path(__file__).parent
APP_SETTINGS_PATH = BASE_DIR / "config" / "app_settings.json"
LLM_SETTINGS_PATH = BASE_DIR / "config" / "llm_settings.json"
PRESETS_DIR = BASE_DIR / "Presets"
class ConfigurationError(Exception):
"""Custom exception for configuration loading errors."""
pass
def _get_base_map_type(target_map_string: str) -> str:
"""Extracts the base map type (e.g., 'COL') from a potentially numbered string ('COL-1')."""
# Use regex to find the leading alphabetical part
match = re.match(r"([a-zA-Z]+)", target_map_string)
if match:
return match.group(1).upper()
# Fallback if no number suffix or unexpected format
return target_map_string.upper()
def _fnmatch_to_regex(pattern: str) -> str:
"""
Converts an fnmatch pattern to a regex pattern string.
Handles basic wildcards (*, ?) and escapes other regex special characters.
"""
i, n = 0, len(pattern)
res = ''
while i < n:
c = pattern[i]
i = i + 1
if c == '*':
res = res + '.*'
elif c == '?':
res = res + '.'
elif c == '[':
j = i
if j < n and pattern[j] == '!':
j = j + 1
if j < n and pattern[j] == ']':
j = j + 1
while j < n and pattern[j] != ']':
j = j + 1
if j >= n:
res = res + '\\['
else:
stuff = pattern[i:j].replace('\\','\\\\')
i = j + 1
if stuff[0] == '!':
stuff = '^' + stuff[1:]
elif stuff[0] == '^':
stuff = '\\' + stuff
res = '%s[%s]' % (res, stuff)
else:
res = res + re.escape(c)
# We want to find the pattern anywhere in the filename for flexibility,
# so don't anchor with ^$ by default. Anchoring might be needed for specific cases.
# Let's return the core pattern and let the caller decide on anchoring if needed.
# For filename matching, we usually want to find the pattern, not match the whole string.
return res
class Configuration:
"""
Loads and provides access to core settings combined with a specific preset.
"""
def __init__(self, preset_name: str):
"""
Loads core config and the specified preset file.
Args:
preset_name: The name of the preset (without .json extension).
Raises:
ConfigurationError: If core config or preset cannot be loaded/validated.
"""
log.debug(f"Initializing Configuration with preset: '{preset_name}'")
self.preset_name = preset_name
self._core_settings: dict = self._load_core_config()
self._llm_settings: dict = self._load_llm_config()
self._preset_settings: dict = self._load_preset(preset_name)
self._validate_configs()
self._compile_regex_patterns()
log.info(f"Configuration loaded successfully using preset: '{self.preset_name}'")
def _compile_regex_patterns(self):
"""Compiles regex patterns from config/preset for faster matching."""
log.debug("Compiling regex patterns from configuration...")
self.compiled_extra_regex: list[re.Pattern] = []
self.compiled_model_regex: list[re.Pattern] = []
self.compiled_bit_depth_regex_map: dict[str, re.Pattern] = {}
# Map: base_map_type -> list of tuples: (compiled_regex, original_keyword, rule_index)
self.compiled_map_keyword_regex: dict[str, list[tuple[re.Pattern, str, int]]] = {}
for pattern in self.move_to_extra_patterns:
try:
regex_str = _fnmatch_to_regex(pattern)
self.compiled_extra_regex.append(re.compile(regex_str, re.IGNORECASE))
except re.error as e:
log.warning(f"Failed to compile 'extra' regex pattern '{pattern}': {e}. Skipping pattern.")
model_patterns = self.asset_category_rules.get('model_patterns', [])
for pattern in model_patterns:
try:
regex_str = _fnmatch_to_regex(pattern)
self.compiled_model_regex.append(re.compile(regex_str, re.IGNORECASE))
except re.error as e:
log.warning(f"Failed to compile 'model' regex pattern '{pattern}': {e}. Skipping pattern.")
for map_type, pattern in self.source_bit_depth_variants.items():
try:
regex_str = _fnmatch_to_regex(pattern)
if pattern.endswith('*'):
regex_str = regex_str.removesuffix('.*')
final_regex_str = regex_str
self.compiled_bit_depth_regex_map[map_type] = re.compile(final_regex_str, re.IGNORECASE)
log.debug(f" Compiled bit depth variant for '{map_type}' as regex (IGNORECASE): {final_regex_str}")
except re.error as e:
log.warning(f"Failed to compile 'bit depth' regex pattern '{pattern}' for map type '{map_type}': {e}. Skipping pattern.")
separator = re.escape(self.source_naming_separator)
from collections import defaultdict
temp_compiled_map_regex = defaultdict(list)
for rule_index, mapping_rule in enumerate(self.map_type_mapping):
if not isinstance(mapping_rule, dict) or \
'target_type' not in mapping_rule or \
'keywords' not in mapping_rule or \
not isinstance(mapping_rule['keywords'], list):
log.warning(f"Skipping invalid map_type_mapping rule at index {rule_index}: {mapping_rule}. Expected dict with 'target_type' and 'keywords' list.")
continue
target_type = mapping_rule['target_type'].upper()
source_keywords = mapping_rule['keywords']
for keyword in source_keywords:
if not isinstance(keyword, str):
log.warning(f"Skipping non-string keyword '{keyword}' in rule {rule_index} for target '{target_type}'.")
continue
try:
kw_regex_part = _fnmatch_to_regex(keyword)
regex_str = rf"(?:^|{separator})({kw_regex_part})(?:$|{separator})"
compiled_regex = re.compile(regex_str, re.IGNORECASE)
temp_compiled_map_regex[target_type].append((compiled_regex, keyword, rule_index))
log.debug(f" Compiled keyword '{keyword}' (rule {rule_index}) for target '{target_type}' as regex: {regex_str}")
except re.error as e:
log.warning(f"Failed to compile map keyword regex '{keyword}' for target type '{target_type}': {e}. Skipping keyword.")
self.compiled_map_keyword_regex = dict(temp_compiled_map_regex)
log.debug(f"Compiled map keyword regex keys: {list(self.compiled_map_keyword_regex.keys())}")
log.debug("Finished compiling regex patterns.")
def _load_core_config(self) -> dict:
"""Loads settings from the core app_settings.json file."""
log.debug(f"Loading core config from: {APP_SETTINGS_PATH}")
if not APP_SETTINGS_PATH.is_file():
raise ConfigurationError(f"Core configuration file not found: {APP_SETTINGS_PATH}")
try:
with open(APP_SETTINGS_PATH, 'r', encoding='utf-8') as f:
settings = json.load(f)
log.debug(f"Core config loaded successfully.")
return settings
except json.JSONDecodeError as e:
raise ConfigurationError(f"Failed to parse core configuration file {APP_SETTINGS_PATH}: Invalid JSON - {e}")
except Exception as e:
raise ConfigurationError(f"Failed to read core configuration file {APP_SETTINGS_PATH}: {e}")
def _load_llm_config(self) -> dict:
"""Loads settings from the llm_settings.json file."""
log.debug(f"Loading LLM config from: {LLM_SETTINGS_PATH}")
if not LLM_SETTINGS_PATH.is_file():
# Log a warning but don't raise an error, allow fallback if possible
log.warning(f"LLM configuration file not found: {LLM_SETTINGS_PATH}. LLM features might be disabled or use defaults.")
return {}
try:
with open(LLM_SETTINGS_PATH, 'r', encoding='utf-8') as f:
settings = json.load(f)
log.debug(f"LLM config loaded successfully.")
return settings
except json.JSONDecodeError as e:
log.error(f"Failed to parse LLM configuration file {LLM_SETTINGS_PATH}: Invalid JSON - {e}")
return {}
except Exception as e:
log.error(f"Failed to read LLM configuration file {LLM_SETTINGS_PATH}: {e}")
return {}
def _load_preset(self, preset_name: str) -> dict:
"""Loads the specified preset JSON file."""
log.debug(f"Loading preset: '{preset_name}' from {PRESETS_DIR}")
if not PRESETS_DIR.is_dir():
raise ConfigurationError(f"Presets directory not found: {PRESETS_DIR}")
preset_file = PRESETS_DIR / f"{preset_name}.json"
if not preset_file.is_file():
raise ConfigurationError(f"Preset file not found: {preset_file}")
try:
with open(preset_file, 'r', encoding='utf-8') as f:
preset_data = json.load(f)
log.debug(f"Preset '{preset_name}' loaded successfully.")
return preset_data
except json.JSONDecodeError as e:
raise ConfigurationError(f"Failed to parse preset file {preset_file}: Invalid JSON - {e}")
except Exception as e:
raise ConfigurationError(f"Failed to read preset file {preset_file}: {e}")
def _validate_configs(self):
"""Performs basic validation checks on loaded settings."""
log.debug("Validating loaded configurations...")
# Preset validation
required_preset_keys = [
"preset_name", "supplier_name", "source_naming", "map_type_mapping",
"asset_category_rules", "archetype_rules", "move_to_extra_patterns"
]
for key in required_preset_keys:
if key not in self._preset_settings:
raise ConfigurationError(f"Preset '{self.preset_name}' is missing required key: '{key}'.")
# Validate map_type_mapping structure (new format)
if not isinstance(self._preset_settings['map_type_mapping'], list):
raise ConfigurationError(f"Preset '{self.preset_name}': 'map_type_mapping' must be a list.")
for index, rule in enumerate(self._preset_settings['map_type_mapping']):
if not isinstance(rule, dict):
raise ConfigurationError(f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' must be a dictionary.")
if 'target_type' not in rule or not isinstance(rule['target_type'], str):
raise ConfigurationError(f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' is missing 'target_type' string.")
valid_file_type_keys = self._core_settings.get('FILE_TYPE_DEFINITIONS', {}).keys()
if rule['target_type'] not in valid_file_type_keys:
raise ConfigurationError(
f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' "
f"has an invalid 'target_type': '{rule['target_type']}'. "
f"Must be one of {list(valid_file_type_keys)}."
)
if 'keywords' not in rule or not isinstance(rule['keywords'], list):
raise ConfigurationError(f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' is missing 'keywords' list.")
for kw_index, keyword in enumerate(rule['keywords']):
if not isinstance(keyword, str):
raise ConfigurationError(f"Preset '{self.preset_name}': Keyword at index {kw_index} in rule {index} ('{rule['target_type']}') must be a string.")
if not isinstance(self._core_settings.get('TARGET_FILENAME_PATTERN'), str):
raise ConfigurationError("Core config 'TARGET_FILENAME_PATTERN' must be a string.")
if not isinstance(self._core_settings.get('OUTPUT_DIRECTORY_PATTERN'), str):
raise ConfigurationError("Core config 'OUTPUT_DIRECTORY_PATTERN' must be a string.")
if not isinstance(self._core_settings.get('OUTPUT_FILENAME_PATTERN'), str):
raise ConfigurationError("Core config 'OUTPUT_FILENAME_PATTERN' must be a string.")
if not isinstance(self._core_settings.get('IMAGE_RESOLUTIONS'), dict):
raise ConfigurationError("Core config 'IMAGE_RESOLUTIONS' must be a dictionary.")
# Validate DEFAULT_ASSET_CATEGORY
valid_asset_type_keys = self._core_settings.get('ASSET_TYPE_DEFINITIONS', {}).keys()
default_asset_category_value = self._core_settings.get('DEFAULT_ASSET_CATEGORY')
if not default_asset_category_value:
raise ConfigurationError("Core config 'DEFAULT_ASSET_CATEGORY' is missing.")
if default_asset_category_value not in valid_asset_type_keys:
raise ConfigurationError(
f"Core config 'DEFAULT_ASSET_CATEGORY' ('{default_asset_category_value}') "
f"is not a valid key in ASSET_TYPE_DEFINITIONS. "
f"Must be one of {list(valid_asset_type_keys)}."
)
if self._llm_settings:
required_llm_keys = [
"llm_predictor_examples", "llm_endpoint_url", "llm_api_key",
"llm_model_name", "llm_temperature", "llm_request_timeout",
"llm_predictor_prompt"
]
for key in required_llm_keys:
if key not in self._llm_settings:
# Log warning instead of raising error to allow partial functionality
log.warning(f"LLM config is missing recommended key: '{key}'. LLM features might not work correctly.")
log.debug("Configuration validation passed.")
@property
def supplier_name(self) -> str:
return self._preset_settings.get('supplier_name', 'DefaultSupplier')
@property
def default_asset_category(self) -> str:
"""Gets the default asset category from core settings."""
# Fallback should align with a valid key, and validation should catch issues.
return self._core_settings.get('DEFAULT_ASSET_CATEGORY', 'Surface')
@property
def target_filename_pattern(self) -> str:
return self._core_settings['TARGET_FILENAME_PATTERN']
@property
def output_directory_pattern(self) -> str:
"""Gets the output directory pattern ONLY from core settings."""
# Default pattern if missing in core settings (should be caught by validation)
default_pattern = "[supplier]/[assetname]"
return self._core_settings.get('OUTPUT_DIRECTORY_PATTERN', default_pattern)
@property
def output_filename_pattern(self) -> str:
"""Gets the output filename pattern ONLY from core settings."""
# Default pattern if missing in core settings (should be caught by validation)
default_pattern = "[assetname]_[maptype]_[resolution].[ext]"
return self._core_settings.get('OUTPUT_FILENAME_PATTERN', default_pattern)
@property
def image_resolutions(self) -> dict[str, int]:
return self._core_settings['IMAGE_RESOLUTIONS']
@property
def map_type_mapping(self) -> list:
return self._preset_settings['map_type_mapping']
@property
def source_naming_separator(self) -> str:
return self._preset_settings.get('source_naming', {}).get('separator', '_')
@property
def source_naming_indices(self) -> dict:
return self._preset_settings.get('source_naming', {}).get('part_indices', {})
@property
def source_glossiness_keywords(self) -> list:
return self._preset_settings.get('source_naming', {}).get('glossiness_keywords', [])
@property
def source_bit_depth_variants(self) -> dict:
return self._preset_settings.get('source_naming', {}).get('bit_depth_variants', {})
@property
def archetype_rules(self) -> list:
return self._preset_settings['archetype_rules']
@property
def asset_category_rules(self) -> dict:
return self._preset_settings['asset_category_rules']
@property
def move_to_extra_patterns(self) -> list:
return self._preset_settings['move_to_extra_patterns']
@property
def extra_files_subdir(self) -> str:
return self._core_settings['EXTRA_FILES_SUBDIR']
@property
def metadata_filename(self) -> str:
return self._core_settings['METADATA_FILENAME']
@property
def calculate_stats_resolution(self) -> str:
return self._core_settings['CALCULATE_STATS_RESOLUTION']
@property
def map_merge_rules(self) -> list:
return self._core_settings['MAP_MERGE_RULES']
@property
def aspect_ratio_decimals(self) -> int:
return self._core_settings['ASPECT_RATIO_DECIMALS']
@property
def temp_dir_prefix(self) -> str:
return self._core_settings['TEMP_DIR_PREFIX']
@property
def jpg_quality(self) -> int:
"""Gets the configured JPG quality level."""
return self._core_settings.get('JPG_QUALITY', 95)
@property
def resolution_threshold_for_jpg(self) -> int:
"""Gets the pixel dimension threshold for using JPG for 8-bit images."""
return self._core_settings.get('RESOLUTION_THRESHOLD_FOR_JPG', 4096)
@property
def respect_variant_map_types(self) -> list:
"""Gets the list of map types that should always respect variant numbering."""
# Ensure it returns a list, even if missing from config.py (though defaults should handle it)
return self._core_settings.get('RESPECT_VARIANT_MAP_TYPES', [])
@property
def force_lossless_map_types(self) -> list:
"""Gets the list of map types that must always be saved losslessly."""
return self._core_settings.get('FORCE_LOSSLESS_MAP_TYPES', [])
def get_bit_depth_rule(self, map_type_input: str) -> str:
"""
Gets the bit depth rule ('respect', 'force_8bit', 'force_16bit') for a given map type identifier.
The map_type_input can be an FTD key (e.g., "MAP_COL") or a suffixed FTD key (e.g., "MAP_COL-1").
"""
if not self._core_settings or 'FILE_TYPE_DEFINITIONS' not in self._core_settings:
log.warning("FILE_TYPE_DEFINITIONS not found in core settings. Cannot determine bit depth rule.")
return "respect"
file_type_definitions = self._core_settings['FILE_TYPE_DEFINITIONS']
# 1. Try direct match with map_type_input as FTD key
definition = file_type_definitions.get(map_type_input)
if definition:
rule = definition.get('bit_depth_rule')
if rule in ['respect', 'force_8bit', 'force_16bit']:
return rule
else:
log.warning(f"FTD key '{map_type_input}' found, but 'bit_depth_rule' is missing or invalid: '{rule}'. Defaulting to 'respect'.")
return "respect"
# 2. Try to derive base FTD key by stripping common variant suffixes
# Regex to remove trailing suffixes like -<digits>, -<alphanum>, _<alphanum>
base_ftd_key_candidate = re.sub(r"(-[\w\d]+|_[\w\d]+)$", "", map_type_input)
if base_ftd_key_candidate != map_type_input:
definition = file_type_definitions.get(base_ftd_key_candidate)
if definition:
rule = definition.get('bit_depth_rule')
if rule in ['respect', 'force_8bit', 'force_16bit']:
log.debug(f"Derived base FTD key '{base_ftd_key_candidate}' from '{map_type_input}' and found bit depth rule: {rule}")
return rule
else:
log.warning(f"Derived base FTD key '{base_ftd_key_candidate}' from '{map_type_input}', but 'bit_depth_rule' is missing/invalid: '{rule}'. Defaulting to 'respect'.")
return "respect"
# If no match found after trying direct and derived keys
log.warning(f"Map type identifier '{map_type_input}' (or its derived base) not found in FILE_TYPE_DEFINITIONS. Defaulting bit depth rule to 'respect'.")
return "respect"
def get_16bit_output_formats(self) -> tuple[str, str]:
"""Gets the primary and fallback format names for 16-bit output."""
primary = self._core_settings.get('OUTPUT_FORMAT_16BIT_PRIMARY', 'png')
fallback = self._core_settings.get('OUTPUT_FORMAT_16BIT_FALLBACK', 'png')
return primary.lower(), fallback.lower()
def get_8bit_output_format(self) -> str:
"""Gets the format name for 8-bit output."""
return self._core_settings.get('OUTPUT_FORMAT_8BIT', 'png').lower()
def get_standard_map_type_aliases(self) -> list[str]:
"""
Derives a sorted list of unique standard map type aliases
from FILE_TYPE_DEFINITIONS.
"""
aliases = set()
file_type_definitions = self._core_settings.get('FILE_TYPE_DEFINITIONS', {})
for _key, definition in file_type_definitions.items():
if isinstance(definition, dict):
standard_type = definition.get('standard_type')
if standard_type and isinstance(standard_type, str) and standard_type.strip():
aliases.add(standard_type)
return sorted(list(aliases))
def get_asset_type_definitions(self) -> dict:
"""Returns the ASSET_TYPE_DEFINITIONS dictionary from core settings."""
return self._core_settings.get('ASSET_TYPE_DEFINITIONS', {})
def get_asset_type_keys(self) -> list:
"""Returns a list of valid asset type keys from core settings."""
return list(self.get_asset_type_definitions().keys())
def get_file_type_definitions_with_examples(self) -> dict:
"""Returns the FILE_TYPE_DEFINITIONS dictionary (including descriptions and examples) from core settings."""
return self._core_settings.get('FILE_TYPE_DEFINITIONS', {})
def get_file_type_keys(self) -> list:
"""Returns a list of valid file type keys from core settings."""
return list(self.get_file_type_definitions_with_examples().keys())
def get_llm_examples(self) -> list:
"""Returns the list of LLM input/output examples from LLM settings."""
# Use empty list as fallback if LLM settings file is missing/invalid
return self._llm_settings.get('llm_predictor_examples', [])
@property
def llm_predictor_prompt(self) -> str:
"""Returns the LLM predictor prompt string from LLM settings."""
return self._llm_settings.get('llm_predictor_prompt', '')
@property
def llm_endpoint_url(self) -> str:
"""Returns the LLM endpoint URL from LLM settings."""
return self._llm_settings.get('llm_endpoint_url', '')
@property
def llm_api_key(self) -> str:
"""Returns the LLM API key from LLM settings."""
return self._llm_settings.get('llm_api_key', '')
@property
def llm_model_name(self) -> str:
"""Returns the LLM model name from LLM settings."""
return self._llm_settings.get('llm_model_name', '')
@property
def llm_temperature(self) -> float:
"""Returns the LLM temperature from LLM settings."""
return self._llm_settings.get('llm_temperature', 0.5)
@property
def llm_request_timeout(self) -> int:
"""Returns the LLM request timeout in seconds from LLM settings."""
return self._llm_settings.get('llm_request_timeout', 120)
@property
def keybind_config(self) -> dict[str, list[str]]:
"""
Processes FILE_TYPE_DEFINITIONS to create a mapping of keybinds
to their associated file type keys.
Example: {'C': ['MAP_COL'], 'R': ['MAP_ROUGH', 'MAP_GLOSS']}
"""
keybinds = {}
file_type_defs = self._core_settings.get('FILE_TYPE_DEFINITIONS', {})
for ftd_key, ftd_value in file_type_defs.items():
if isinstance(ftd_value, dict) and 'keybind' in ftd_value:
key = ftd_value['keybind']
if key not in keybinds:
keybinds[key] = []
keybinds[key].append(ftd_key)
# Ensure toggleable keybinds have their file types in a consistent order if necessary
# For example, for 'R': ['MAP_ROUGH', 'MAP_GLOSS']
# The order from app_settings.json is generally preserved by dict iteration in Python 3.7+
# but explicit sorting could be added if a specific cycle order is critical beyond config file order.
# For now, we rely on the order they appear in the config.
return keybinds
def load_base_config() -> dict:
"""
Loads only the base configuration from app_settings.json.
Does not load presets or perform merging/validation.
"""
if not APP_SETTINGS_PATH.is_file():
log.error(f"Base configuration file not found: {APP_SETTINGS_PATH}")
# Return empty dict or raise a specific error if preferred
# For now, return empty dict to allow GUI to potentially start with defaults
return {}
try:
with open(APP_SETTINGS_PATH, 'r', encoding='utf-8') as f:
settings = json.load(f)
return settings
except json.JSONDecodeError as e:
log.error(f"Failed to parse base configuration file {APP_SETTINGS_PATH}: Invalid JSON - {e}")
return {}
except Exception as e:
log.error(f"Failed to read base configuration file {APP_SETTINGS_PATH}: {e}")
return {}
def save_llm_config(settings_dict: dict):
"""
Saves the provided LLM settings dictionary to llm_settings.json.
"""
log.debug(f"Saving LLM config to: {LLM_SETTINGS_PATH}")
try:
with open(LLM_SETTINGS_PATH, 'w', encoding='utf-8') as f:
json.dump(settings_dict, f, indent=4)
# Use info level for successful save
log.info(f"LLM config saved successfully to {LLM_SETTINGS_PATH}")
except Exception as e:
log.error(f"Failed to save LLM configuration file {LLM_SETTINGS_PATH}: {e}")
# Re-raise as ConfigurationError to signal failure upstream
raise ConfigurationError(f"Failed to save LLM configuration: {e}")
def save_base_config(settings_dict: dict):
"""
Saves the provided settings dictionary to app_settings.json.
"""
log.debug(f"Saving base config to: {APP_SETTINGS_PATH}")
try:
with open(APP_SETTINGS_PATH, 'w', encoding='utf-8') as f:
json.dump(settings_dict, f, indent=4)
log.debug(f"Base config saved successfully.")
except Exception as e:
log.error(f"Failed to save base configuration file {APP_SETTINGS_PATH}: {e}")
raise ConfigurationError(f"Failed to save configuration: {e}")