Rusfort/Asset-Frameworker
1
0
Fork 0
Code Issues 24 Pull Requests Actions Packages Projects 3 Releases Wiki Activity

Implemented Item type priority handling ( DISP16 )

Browse Source
This commit is contained in:
Rusfort 2025-05-15 20:52:58 +02:00
parent ca92c72070
commit b43b2522d7
16 changed files with 537 additions and 254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.roo/mcp.json
View File

@ -38,7 +38,8 @@
"delete_system_pattern_by_id", "delete_system_pattern_by_id",
"get_conport_schema", "get_conport_schema",
"get_recent_activity_summary", "get_recent_activity_summary",
"semantic_search_conport" "semantic_search_conport",
"search_system_patterns_fts"
] ]
} }
} }

0
.roo/rules-orchestrator/rules.md Normal file
View File

14
.roomodes
View File

@ -1,3 +1,15 @@
{ {
"customModes": [] "customModes": [
{
"slug": "Task-Initiator",
"name": "Task Initiator",
"roleDefinition": "You are Task Initiator. Your exclusive function is comprehensive initial context gathering, focusing solely on ConPort data. Do NOT perform other tasks or use direct file system tools for context gathering.",
"customInstructions": "1. First, execute standard initial context setup procedures (as per global ConPort strategy).\n2. Next, if a specific user request is pending, YOU, as Task Initiator, should analyze it and proactively gather relevant information, strictly by querying ConPort. Your process for this is:\n a. Identify the key subject(s) of the request.\n b. Loosely search relevant ConPort data for information or summaries related to these identified subject(s).\n3. After completing both standard setup AND any ConPort-based task-specific gathering, briefly report the overall context status. This report must cover ConPort initialization and summarize any specific information found (or explicitly not found) within ConPort relevant to the user's request.\n4. Then, output `[TASK_INITIATOR_COMPLETE]`.\n5. Finally, to address the user's main request with the context you've gathered (or confirmed is missing from ConPort), use the `switch_mode` tool to transition to the determined most appropriate mode by analysing the initial request. you should ALWAYS finish context-gathering before switching modes.",
"groups": [
"mcp",
"read"
],
"source": "project"
}
]
} }

4
Documentation/01_User_Guide/11_Usage_Autotest.md
View File

@ -34,7 +34,7 @@ The script accepts several command-line arguments to configure the test run. If
* A string to search for within the application logs generated during the test run. If found, matching log lines (with context) will be highlighted. * A string to search for within the application logs generated during the test run. If found, matching log lines (with context) will be highlighted.
* Default: None * Default: None
* `--additional-lines NUM_LINES` (optional): * `--additional-lines NUM_LINES` (optional):
* When using `--search`, this specifies how many lines of context before and after each matching log line should be displayed. * When using `--search`, this specifies how many lines of context before and after each matching log line should be displayed. A good non-zero value is 1-2.
* Default: `0` * Default: `0`
**Example Usage:** **Example Usage:**
@ -81,3 +81,5 @@ When executed, `autotest.py` performs the following steps:
* **Output Directory:** Inspect the contents of the specified output directory to manually verify the processed assets if needed. * **Output Directory:** Inspect the contents of the specified output directory to manually verify the processed assets if needed.
This automated test helps ensure the stability of the core processing logic when driven by GUI-equivalent actions. This automated test helps ensure the stability of the core processing logic when driven by GUI-equivalent actions.
Note: Under some conditions, the autotest will exit with errorcode "3221226505". This has no consequence and can therefor be ignore.

24
Presets/Dinesen.json
View File

@ -1,5 +1,5 @@
{ {
"preset_name": "Dinesen Custom", "preset_name": "Dinesen",
"supplier_name": "Dinesen", "supplier_name": "Dinesen",
"notes": "Preset for standard Poliigon downloads. Prioritizes _xxx16 files. Moves previews etc. to Extra/. Assumes Metal/Rough workflow.", "notes": "Preset for standard Poliigon downloads. Prioritizes _xxx16 files. Moves previews etc. to Extra/. Assumes Metal/Rough workflow.",
"source_naming": { "source_naming": {
@ -10,11 +10,7 @@
}, },
"glossiness_keywords": [ "glossiness_keywords": [
"GLOSS" "GLOSS"
], ]
"bit_depth_variants": {
"NRM": "*_NRM16*",
"DISP": "*_DISP16*"
}
}, },
"move_to_extra_patterns": [ "move_to_extra_patterns": [
"*_Preview*", "*_Preview*",
@ -25,7 +21,8 @@
"*.pdf", "*.pdf",
"*.url", "*.url",
"*.htm*", "*.htm*",
"*_Fabric.*" "*_Fabric.*",
"*_DISP_*METALNESS*"
], ],
"map_type_mapping": [ "map_type_mapping": [
{ {
@ -46,6 +43,11 @@
"NORM*", "NORM*",
"NRM*", "NRM*",
"N" "N"
],
"priority_keywords": [
"*_NRM16*",
"*_NM16*",
"*Normal16*"
] ]
}, },
{ {
@ -75,6 +77,14 @@
"DISP", "DISP",
"HEIGHT", "HEIGHT",
"BUMP" "BUMP"
],
"priority_keywords": [
"*_DISP16*",
"*_DSP16*",
"*DSP16*",
"*DISP16*",
"*Displacement16*",
"*Height16*"
] ]
}, },
{ {

115
Presets/Poliigon.json
View File

@ -10,11 +10,7 @@
}, },
"glossiness_keywords": [ "glossiness_keywords": [
"GLOSS" "GLOSS"
], ]
"bit_depth_variants": {
"NRM": "*_NRM16*",
"DISP": "*_DISP16*"
}
}, },
"move_to_extra_patterns": [ "move_to_extra_patterns": [
"*_Preview*", "*_Preview*",
@ -28,7 +24,114 @@
"*_Fabric.*", "*_Fabric.*",
"*_Albedo*" "*_Albedo*"
], ],
"map_type_mapping": [], "map_type_mapping": [
{
"target_type": "MAP_COL",
"keywords": [
"COLOR*",
"COL",
"COL-*",
"DIFFUSE",
"DIF",
"ALBEDO"
]
},
{
"target_type": "MAP_NRM",
"keywords": [
"NORMAL*",
"NORM*",
"NRM*",
"N"
],
"priority_keywords": [
"*_NRM16*",
"*_NM16*",
"*Normal16*"
]
},
{
"target_type": "MAP_ROUGH",
"keywords": [
"ROUGHNESS",
"ROUGH"
]
},
{
"target_type": "MAP_GLOSS",
"keywords": [
"GLOSS"
]
},
{
"target_type": "MAP_AO",
"keywords": [
"AMBIENTOCCLUSION",
"AO"
]
},
{
"target_type": "MAP_DISP",
"keywords": [
"DISPLACEMENT",
"DISP",
"HEIGHT",
"BUMP"
],
"priority_keywords": [
"*_DISP16*",
"*_DSP16*",
"*DSP16*",
"*DISP16*",
"*Displacement16*",
"*Height16*"
]
},
{
"target_type": "MAP_REFL",
"keywords": [
"REFLECTION",
"REFL",
"SPECULAR",
"SPEC"
]
},
{
"target_type": "MAP_SSS",
"keywords": [
"SSS",
"SUBSURFACE*"
]
},
{
"target_type": "MAP_FUZZ",
"keywords": [
"FUZZ"
]
},
{
"target_type": "MAP_IDMAP",
"keywords": [
"IDMAP"
]
},
{
"target_type": "MAP_MASK",
"keywords": [
"OPAC*",
"TRANSP*",
"MASK*",
"ALPHA*"
]
},
{
"target_type": "MAP_METAL",
"keywords": [
"METAL*",
"METALLIC"
]
}
],
"asset_category_rules": { "asset_category_rules": {
"model_patterns": [ "model_patterns": [
"*.fbx", "*.fbx",

4
TestFiles/Test-BoucleChunky001.json
View File

@ -1,9 +1,9 @@
{ {
"source_rules": [ "source_rules": [
{ {
"input_path": "BoucleChunky001.zip", "input_path": "BoucleChunky001.zip",
"supplier_identifier": "Dinesen", "supplier_identifier": "Dinesen",
"preset_name": null, "preset_name": "Dinesen",
"assets": [ "assets": [
{ {
"asset_name": "BoucleChunky001", "asset_name": "BoucleChunky001",

51
autotest.py
View File

@ -298,15 +298,32 @@ class AutoTester(QObject):
def run_test(self) -> None: def run_test(self) -> None:
"""Orchestrates the test steps.""" """Orchestrates the test steps."""
logger.info("Starting test run...") # Load expected rules first to potentially get the preset name
self._load_expected_rules() # Moved here
if not self.expected_rules_data: # Ensure rules were loaded if not self.expected_rules_data: # Ensure rules were loaded
logger.error("Expected rules not loaded. Aborting test.") logger.error("Expected rules not loaded. Aborting test.")
self.cleanup_and_exit(success=False) self.cleanup_and_exit(success=False)
return return
# Determine preset to use: from expected rules if available, else from CLI args
preset_to_use = self.cli_args.preset # Default
if self.expected_rules_data.get("source_rules") and \
isinstance(self.expected_rules_data["source_rules"], list) and \
len(self.expected_rules_data["source_rules"]) > 0 and \
isinstance(self.expected_rules_data["source_rules"][0], dict) and \
self.expected_rules_data["source_rules"][0].get("preset_name"):
preset_to_use = self.expected_rules_data["source_rules"][0]["preset_name"]
logger.info(f"Overriding preset with value from expected_rules.json: '{preset_to_use}'")
else:
logger.info(f"Using preset from CLI arguments: '{preset_to_use}' (this was self.cli_args.preset)")
# If preset_to_use is still self.cli_args.preset, ensure it's logged correctly
# The variable preset_to_use will hold the correct value to be used throughout.
logger.info("Starting test run...") # Moved after preset_to_use definition
# Add a specific summary log for essential context # Add a specific summary log for essential context
logger.info(f"Autotest Context: Input='{self.cli_args.zipfile.name}', Preset='{self.cli_args.preset}', Output='{self.cli_args.outputdir}'") # This now correctly uses preset_to_use
logger.info(f"Autotest Context: Input='{self.cli_args.zipfile.name}', Preset='{preset_to_use}', Output='{self.cli_args.outputdir}'")
# Step 1: Load ZIP # Step 1: Load ZIP
self.test_step = "LOADING_ZIP" self.test_step = "LOADING_ZIP"
@ -326,20 +343,25 @@ class AutoTester(QObject):
# Step 2: Select Preset # Step 2: Select Preset
self.test_step = "SELECTING_PRESET" self.test_step = "SELECTING_PRESET"
logger.info(f"Step 2: Selecting preset: {self.cli_args.preset}") # KEEP INFO - Passes filter # Use preset_to_use (which is now correctly defined earlier)
logger.info(f"Step 2: Selecting preset: {preset_to_use}") # KEEP INFO - Passes filter
# The print statement below already uses preset_to_use, which is good.
print(f"DEBUG: Attempting to select preset: '{preset_to_use}' (derived from expected: {preset_to_use == self.expected_rules_data.get('source_rules',[{}])[0].get('preset_name') if self.expected_rules_data.get('source_rules') else 'N/A'}, cli_arg: {self.cli_args.preset})")
preset_found = False preset_found = False
preset_list_widget = self.main_window.preset_editor_widget.editor_preset_list preset_list_widget = self.main_window.preset_editor_widget.editor_preset_list
for i in range(preset_list_widget.count()): for i in range(preset_list_widget.count()):
item = preset_list_widget.item(i) item = preset_list_widget.item(i)
if item and item.text() == self.cli_args.preset: if item and item.text() == preset_to_use: # Use preset_to_use
preset_list_widget.setCurrentItem(item) preset_list_widget.setCurrentItem(item)
logger.debug(f"Preset '{self.cli_args.preset}' selected.") logger.debug(f"Preset '{preset_to_use}' selected.")
print(f"DEBUG: Successfully selected preset '{item.text()}' in GUI.")
preset_found = True preset_found = True
break break
if not preset_found: if not preset_found:
logger.error(f"Preset '{self.cli_args.preset}' not found in the list.") logger.error(f"Preset '{preset_to_use}' not found in the list.")
available_presets = [preset_list_widget.item(i).text() for i in range(preset_list_widget.count())] available_presets = [preset_list_widget.item(i).text() for i in range(preset_list_widget.count())]
logger.debug(f"Available presets: {available_presets}") logger.debug(f"Available presets: {available_presets}")
print(f"DEBUG: Failed to find preset '{preset_to_use}'. Available: {available_presets}")
self.cleanup_and_exit(success=False) self.cleanup_and_exit(success=False)
return return
@ -449,8 +471,6 @@ class AutoTester(QObject):
else: else:
logger.warning("Log console or output widget not found. Cannot retrieve logs.") logger.warning("Log console or output widget not found. Cannot retrieve logs.")
self._process_and_display_logs(all_logs_text)
logger.info("Log analysis completed.")
# Final Step # Final Step
logger.info("Test run completed successfully.") # KEEP INFO - Passes filter logger.info("Test run completed successfully.") # KEEP INFO - Passes filter
@ -523,7 +543,7 @@ class AutoTester(QObject):
comparable_sources_list.append({ comparable_sources_list.append({
"input_path": Path(source_rule_obj.input_path).name, # Use only the filename "input_path": Path(source_rule_obj.input_path).name, # Use only the filename
"supplier_identifier": source_rule_obj.supplier_identifier, "supplier_identifier": source_rule_obj.supplier_identifier,
"preset_name": source_rule_obj.preset_name, "preset_name": source_rule_obj.preset_name, # This is the actual preset name from the SourceRule object
"assets": comparable_asset_list "assets": comparable_asset_list
}) })
logger.debug("Conversion to comparable dictionary finished.") logger.debug("Conversion to comparable dictionary finished.")
@ -569,6 +589,8 @@ class AutoTester(QObject):
if not self._compare_list_of_rules(actual_value, expected_value, "FileRule", current_context, "file_path"): if not self._compare_list_of_rules(actual_value, expected_value, "FileRule", current_context, "file_path"):
item_match = False item_match = False
else: # Regular field comparison else: # Regular field comparison
if key == "preset_name":
print(f"DEBUG: Comparing preset_name: Actual='{actual_value}', Expected='{expected_value}' for {item_type_name} ({current_context})")
if actual_value != expected_value: if actual_value != expected_value:
# Handle None vs "None" string for preset_name specifically if it's a common issue # Handle None vs "None" string for preset_name specifically if it's a common issue
if key == "preset_name" and actual_value is None and expected_value == "None": if key == "preset_name" and actual_value is None and expected_value == "None":
@ -602,6 +624,11 @@ class AutoTester(QObject):
list_match = False # Count mismatch is an error list_match = False # Count mismatch is an error
# If counts differ, we still try to match what we can to provide more detailed feedback, # If counts differ, we still try to match what we can to provide more detailed feedback,
# but the overall list_match will remain False. # but the overall list_match will remain False.
if item_type_name == "FileRule":
print(f"DEBUG: FileRule count mismatch for {parent_context}. Actual: {len(actual_list)}, Expected: {len(expected_list)}")
print(f"DEBUG: Actual FileRule paths: {[item.get(item_key_field) for item in actual_list]}")
print(f"DEBUG: Expected FileRule paths: {[item.get(item_key_field) for item in expected_list]}")
actual_items_map = {item.get(item_key_field): item for item in actual_list if item.get(item_key_field) is not None} actual_items_map = {item.get(item_key_field): item for item in actual_list if item.get(item_key_field) is not None}
@ -765,6 +792,10 @@ class AutoTester(QObject):
def cleanup_and_exit(self, success: bool = True) -> None: def cleanup_and_exit(self, success: bool = True) -> None:
"""Cleans up and exits the application.""" """Cleans up and exits the application."""
# Retrieve logs before clearing the handler
all_logs_text = "" # This variable is not used by _process_and_display_logs anymore, but kept for signature compatibility if needed elsewhere.
self._process_and_display_logs(all_logs_text) # Process and display logs BEFORE clearing the buffer
global autotest_memory_handler global autotest_memory_handler
if autotest_memory_handler: if autotest_memory_handler:
logger.debug("Clearing memory log handler buffer and removing handler.") logger.debug("Clearing memory log handler buffer and removing handler.")

6
config/file_type_definitions.json
View File

@ -195,14 +195,16 @@
"FILE_IGNORE": { "FILE_IGNORE": {
"bit_depth_rule": "", "bit_depth_rule": "",
"color": "#673d35", "color": "#673d35",
"description": "File to be ignored", "description": "File identified to be ignored due to prioritization rules (e.g., a lower bit-depth version when a higher one is present).",
"category": "Ignored",
"examples": [ "examples": [
"Thumbs.db", "Thumbs.db",
".DS_Store" ".DS_Store"
], ],
"is_grayscale": false, "is_grayscale": false,
"keybind": "X", "keybind": "X",
"standard_type": "" "standard_type": "",
"details": {}
} }
} }
} }

101
configuration.py
View File

@ -104,8 +104,8 @@ class Configuration:
Raises: Raises:
ConfigurationError: If core config or preset cannot be loaded/validated. ConfigurationError: If core config or preset cannot be loaded/validated.
""" """
log.debug(f"Initializing Configuration with preset: '{preset_name}'") log.debug(f"Initializing Configuration with preset filename stem: '{preset_name}'")
self.preset_name = preset_name self._preset_filename_stem = preset_name # Store the stem used for loading
# 1. Load core settings # 1. Load core settings
self._core_settings: dict = self._load_core_config() self._core_settings: dict = self._load_core_config()
@ -129,12 +129,16 @@ class Configuration:
self._llm_settings: dict = self._load_llm_config() self._llm_settings: dict = self._load_llm_config()
# 7. Load preset settings (conceptually overrides combined base + user for shared keys) # 7. Load preset settings (conceptually overrides combined base + user for shared keys)
self._preset_settings: dict = self._load_preset(preset_name) self._preset_settings: dict = self._load_preset(self._preset_filename_stem) # Use the stored stem
# Store the actual preset name read from the file content
self.actual_internal_preset_name = self._preset_settings.get("preset_name", self._preset_filename_stem)
log.info(f"Configuration instance: Loaded preset file '{self._preset_filename_stem}.json', internal preset_name is '{self.actual_internal_preset_name}'")
# 8. Validate and compile (after all base/user/preset settings are established) # 8. Validate and compile (after all base/user/preset settings are established)
self._validate_configs() self._validate_configs()
self._compile_regex_patterns() self._compile_regex_patterns()
log.info(f"Configuration loaded successfully using preset: '{self.preset_name}'") log.info(f"Configuration loaded successfully using preset: '{self.actual_internal_preset_name}'") # Changed self.preset_name to self.actual_internal_preset_name
def _compile_regex_patterns(self): def _compile_regex_patterns(self):
@ -143,8 +147,8 @@ class Configuration:
self.compiled_extra_regex: list[re.Pattern] = [] self.compiled_extra_regex: list[re.Pattern] = []
self.compiled_model_regex: list[re.Pattern] = [] self.compiled_model_regex: list[re.Pattern] = []
self.compiled_bit_depth_regex_map: dict[str, re.Pattern] = {} self.compiled_bit_depth_regex_map: dict[str, re.Pattern] = {}
# Map: base_map_type -> list of tuples: (compiled_regex, original_keyword, rule_index) # Map: base_map_type -> list of tuples: (compiled_regex, original_keyword, rule_index, is_priority)
self.compiled_map_keyword_regex: dict[str, list[tuple[re.Pattern, str, int]]] = {} self.compiled_map_keyword_regex: dict[str, list[tuple[re.Pattern, str, int, bool]]] = {}
for pattern in self.move_to_extra_patterns: for pattern in self.move_to_extra_patterns:
try: try:
@ -179,28 +183,53 @@ class Configuration:
for rule_index, mapping_rule in enumerate(self.map_type_mapping): for rule_index, mapping_rule in enumerate(self.map_type_mapping):
if not isinstance(mapping_rule, dict) or \ if not isinstance(mapping_rule, dict) or \
'target_type' not in mapping_rule or \ 'target_type' not in mapping_rule: # Removed 'keywords' check here as it's handled below
'keywords' not in mapping_rule or \ log.warning(f"Skipping invalid map_type_mapping rule at index {rule_index}: {mapping_rule}. Expected dict with 'target_type'.")
not isinstance(mapping_rule['keywords'], list):
log.warning(f"Skipping invalid map_type_mapping rule at index {rule_index}: {mapping_rule}. Expected dict with 'target_type' and 'keywords' list.")
continue continue
target_type = mapping_rule['target_type'].upper() target_type = mapping_rule['target_type'].upper()
source_keywords = mapping_rule['keywords']
# Ensure 'keywords' exists and is a list, default to empty list if not found or not a list
regular_keywords = mapping_rule.get('keywords', [])
if not isinstance(regular_keywords, list):
log.warning(f"Rule {rule_index} for target '{target_type}' has 'keywords' but it's not a list. Treating as empty.")
regular_keywords = []
for keyword in source_keywords: priority_keywords = mapping_rule.get('priority_keywords', []) # Optional, defaults to empty list
if not isinstance(priority_keywords, list):
log.warning(f"Rule {rule_index} for target '{target_type}' has 'priority_keywords' but it's not a list. Treating as empty.")
priority_keywords = []
# Process regular keywords
for keyword in regular_keywords:
if not isinstance(keyword, str): if not isinstance(keyword, str):
log.warning(f"Skipping non-string keyword '{keyword}' in rule {rule_index} for target '{target_type}'.") log.warning(f"Skipping non-string regular keyword '{keyword}' in rule {rule_index} for target '{target_type}'.")
continue continue
try:
kw_regex_part = _fnmatch_to_regex(keyword)
# Ensure the keyword is treated as a whole word or is at the start/end of a segment
regex_str = rf"(?:^|{separator})({kw_regex_part})(?:$|{separator})"
compiled_regex = re.compile(regex_str, re.IGNORECASE)
# Add False for is_priority
temp_compiled_map_regex[target_type].append((compiled_regex, keyword, rule_index, False))
log.debug(f" Compiled regular keyword '{keyword}' (rule {rule_index}) for target '{target_type}' as regex: {regex_str}")
except re.error as e:
log.warning(f"Failed to compile regular map keyword regex '{keyword}' for target type '{target_type}': {e}. Skipping keyword.")
# Process priority keywords
for keyword in priority_keywords:
if not isinstance(keyword, str):
log.warning(f"Skipping non-string priority keyword '{keyword}' in rule {rule_index} for target '{target_type}'.")
continue
try: try:
kw_regex_part = _fnmatch_to_regex(keyword) kw_regex_part = _fnmatch_to_regex(keyword)
regex_str = rf"(?:^|{separator})({kw_regex_part})(?:$|{separator})" regex_str = rf"(?:^|{separator})({kw_regex_part})(?:$|{separator})"
compiled_regex = re.compile(regex_str, re.IGNORECASE) compiled_regex = re.compile(regex_str, re.IGNORECASE)
temp_compiled_map_regex[target_type].append((compiled_regex, keyword, rule_index)) # Add True for is_priority
log.debug(f" Compiled keyword '{keyword}' (rule {rule_index}) for target '{target_type}' as regex: {regex_str}") temp_compiled_map_regex[target_type].append((compiled_regex, keyword, rule_index, True))
log.debug(f" Compiled priority keyword '{keyword}' (rule {rule_index}) for target '{target_type}' as regex: {regex_str}")
except re.error as e: except re.error as e:
log.warning(f"Failed to compile map keyword regex '{keyword}' for target type '{target_type}': {e}. Skipping keyword.") log.warning(f"Failed to compile priority map keyword regex '{keyword}' for target type '{target_type}': {e}. Skipping keyword.")
self.compiled_map_keyword_regex = dict(temp_compiled_map_regex) self.compiled_map_keyword_regex = dict(temp_compiled_map_regex)
log.debug(f"Compiled map keyword regex keys: {list(self.compiled_map_keyword_regex.keys())}") log.debug(f"Compiled map keyword regex keys: {list(self.compiled_map_keyword_regex.keys())}")
@ -343,31 +372,43 @@ class Configuration:
] ]
for key in required_preset_keys: for key in required_preset_keys:
if key not in self._preset_settings: if key not in self._preset_settings:
raise ConfigurationError(f"Preset '{self.preset_name}' is missing required key: '{key}'.") raise ConfigurationError(f"Preset file '{self._preset_filename_stem}.json' (internal name: '{self.actual_internal_preset_name}') is missing required key: '{key}'.")
# Validate map_type_mapping structure (new format) # Validate map_type_mapping structure (new format)
if not isinstance(self._preset_settings['map_type_mapping'], list): if not isinstance(self._preset_settings['map_type_mapping'], list):
raise ConfigurationError(f"Preset '{self.preset_name}': 'map_type_mapping' must be a list.") raise ConfigurationError(f"Preset file '{self._preset_filename_stem}.json': 'map_type_mapping' must be a list.")
for index, rule in enumerate(self._preset_settings['map_type_mapping']): for index, rule in enumerate(self._preset_settings['map_type_mapping']):
if not isinstance(rule, dict): if not isinstance(rule, dict):
raise ConfigurationError(f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' must be a dictionary.") raise ConfigurationError(f"Preset file '{self._preset_filename_stem}.json': Rule at index {index} in 'map_type_mapping' must be a dictionary.")
if 'target_type' not in rule or not isinstance(rule['target_type'], str): if 'target_type' not in rule or not isinstance(rule['target_type'], str):
raise ConfigurationError(f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' is missing 'target_type' string.") raise ConfigurationError(f"Preset file '{self._preset_filename_stem}.json': Rule at index {index} in 'map_type_mapping' is missing 'target_type' string.")
valid_file_type_keys = self._file_type_definitions.keys() valid_file_type_keys = self._file_type_definitions.keys()
if rule['target_type'] not in valid_file_type_keys: if rule['target_type'] not in valid_file_type_keys:
raise ConfigurationError( raise ConfigurationError(
f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' " f"Preset file '{self._preset_filename_stem}.json': Rule at index {index} in 'map_type_mapping' "
f"has an invalid 'target_type': '{rule['target_type']}'. " f"has an invalid 'target_type': '{rule['target_type']}'. "
f"Must be one of {list(valid_file_type_keys)}." f"Must be one of {list(valid_file_type_keys)}."
) )
if 'keywords' not in rule or not isinstance(rule['keywords'], list): # 'keywords' is optional if 'priority_keywords' is present and not empty,
raise ConfigurationError(f"Preset '{self.preset_name}': Rule at index {index} in 'map_type_mapping' is missing 'keywords' list.") # but if 'keywords' IS present, it must be a list of strings.
for kw_index, keyword in enumerate(rule['keywords']): if 'keywords' in rule:
if not isinstance(keyword, str): if not isinstance(rule['keywords'], list):
raise ConfigurationError(f"Preset '{self.preset_name}': Keyword at index {kw_index} in rule {index} ('{rule['target_type']}') must be a string.") raise ConfigurationError(f"Preset file '{self._preset_filename_stem}.json': Rule at index {index} in 'map_type_mapping' has 'keywords' but it's not a list.")
for kw_index, keyword in enumerate(rule['keywords']):
if not isinstance(keyword, str):
raise ConfigurationError(f"Preset file '{self._preset_filename_stem}.json': Keyword at index {kw_index} in rule {index} ('{rule['target_type']}') must be a string.")
elif not ('priority_keywords' in rule and rule['priority_keywords']): # if 'keywords' is not present, 'priority_keywords' must be
raise ConfigurationError(f"Preset file '{self._preset_filename_stem}.json': Rule at index {index} in 'map_type_mapping' must have 'keywords' or non-empty 'priority_keywords'.")
# Validate priority_keywords if present
if 'priority_keywords' in rule:
if not isinstance(rule['priority_keywords'], list):
raise ConfigurationError(f"Preset file '{self._preset_filename_stem}.json': Rule at index {index} in 'map_type_mapping' has 'priority_keywords' but it's not a list.")
for prio_kw_index, prio_keyword in enumerate(rule['priority_keywords']):
if not isinstance(prio_keyword, str):
raise ConfigurationError(f"Preset file '{self._preset_filename_stem}.json': Priority keyword at index {prio_kw_index} in rule {index} ('{rule['target_type']}') must be a string.")
if not isinstance(self._core_settings.get('TARGET_FILENAME_PATTERN'), str): if not isinstance(self._core_settings.get('TARGET_FILENAME_PATTERN'), str):
raise ConfigurationError("Core config 'TARGET_FILENAME_PATTERN' must be a string.") raise ConfigurationError("Core config 'TARGET_FILENAME_PATTERN' must be a string.")
@ -407,6 +448,12 @@ class Configuration:
def supplier_name(self) -> str: def supplier_name(self) -> str:
return self._preset_settings.get('supplier_name', 'DefaultSupplier') return self._preset_settings.get('supplier_name', 'DefaultSupplier')
@property
def internal_display_preset_name(self) -> str:
"""Returns the 'preset_name' field from within the loaded preset JSON,
or falls back to the filename stem if not present."""
return self.actual_internal_preset_name
@property @property
def default_asset_category(self) -> str: def default_asset_category(self) -> str:
"""Gets the default asset category from core settings.""" """Gets the default asset category from core settings."""

BIN
context_portal/conport_vector_data/3712b223-f80b-4c07-b57a-5cf7c8175c86/length.bin
View File

Binary file not shown.

BIN
context_portal/conport_vector_data/chroma.sqlite3
View File

Binary file not shown.

BIN
context_portal/context.db
View File

Binary file not shown.

40
gui/main_window.py
View File

@ -311,7 +311,7 @@ class MainWindow(QMainWindow):
log.info(f"Added {added_count} new asset paths: {newly_added_paths}") log.info(f"Added {added_count} new asset paths: {newly_added_paths}")
self.statusBar().showMessage(f"Added {added_count} asset(s). Updating preview...", 3000) self.statusBar().showMessage(f"Added {added_count} asset(s). Updating preview...", 3000)
mode, selected_preset_text = self.preset_editor_widget.get_selected_preset_mode() mode, selected_display_name, preset_file_path = self.preset_editor_widget.get_selected_preset_mode()
if mode == "llm": if mode == "llm":
log.info(f"LLM Interpretation selected. Preparing LLM prediction for {len(newly_added_paths)} new paths.") log.info(f"LLM Interpretation selected. Preparing LLM prediction for {len(newly_added_paths)} new paths.")
@ -330,8 +330,9 @@ class MainWindow(QMainWindow):
log.info(f"Delegating {len(llm_requests_to_queue)} LLM requests to the handler.") log.info(f"Delegating {len(llm_requests_to_queue)} LLM requests to the handler.")
self.llm_interaction_handler.queue_llm_requests_batch(llm_requests_to_queue) self.llm_interaction_handler.queue_llm_requests_batch(llm_requests_to_queue)
# The handler manages starting its own processing internally. # The handler manages starting its own processing internally.
elif mode == "preset" and selected_preset_text: elif mode == "preset" and selected_display_name and preset_file_path:
log.info(f"Preset '{selected_preset_text}' selected. Triggering prediction for {len(newly_added_paths)} new paths.") preset_name_for_loading = preset_file_path.stem
log.info(f"Preset '{selected_display_name}' (file: {preset_name_for_loading}.json) selected. Triggering prediction for {len(newly_added_paths)} new paths.")
if self.prediction_thread and not self.prediction_thread.isRunning(): if self.prediction_thread and not self.prediction_thread.isRunning():
log.debug("Starting prediction thread from add_input_paths.") log.debug("Starting prediction thread from add_input_paths.")
self.prediction_thread.start() self.prediction_thread.start()
@ -343,7 +344,8 @@ class MainWindow(QMainWindow):
self._source_file_lists[input_path_str] = file_list self._source_file_lists[input_path_str] = file_list
self._pending_predictions.add(input_path_str) self._pending_predictions.add(input_path_str)
log.debug(f"Added '{input_path_str}' to pending predictions. Current pending: {self._pending_predictions}") log.debug(f"Added '{input_path_str}' to pending predictions. Current pending: {self._pending_predictions}")
self.start_prediction_signal.emit(input_path_str, file_list, selected_preset_text) # Pass the filename stem for loading, not the display name
self.start_prediction_signal.emit(input_path_str, file_list, preset_name_for_loading)
else: else:
log.warning(f"Skipping prediction for {input_path_str} due to extraction error.") log.warning(f"Skipping prediction for {input_path_str} due to extraction error.")
elif mode == "placeholder": elif mode == "placeholder":
@ -446,7 +448,12 @@ class MainWindow(QMainWindow):
self.statusBar().showMessage("No assets added to process.", 3000) self.statusBar().showMessage("No assets added to process.", 3000)
return return
mode, selected_preset_name = self.preset_editor_widget.get_selected_preset_mode() # mode, selected_preset_name, preset_file_path are relevant here if processing depends on the *loaded* preset's config
# For now, _on_process_requested uses the rules already in unified_model, which should have been generated
# using the correct preset context. The preset name itself isn't directly used by the processing engine,
# as the SourceRule object already contains the necessary preset-derived information or the preset name string.
# We'll rely on the SourceRule objects in unified_model.get_all_source_rules() to be correct.
# mode, selected_display_name, preset_file_path = self.preset_editor_widget.get_selected_preset_mode()
output_dir_str = settings.get("output_dir") output_dir_str = settings.get("output_dir")
@ -694,7 +701,7 @@ class MainWindow(QMainWindow):
log.error("RuleBasedPredictionHandler not loaded. Cannot update preview.") log.error("RuleBasedPredictionHandler not loaded. Cannot update preview.")
self.statusBar().showMessage("Error: Prediction components not loaded.", 5000) self.statusBar().showMessage("Error: Prediction components not loaded.", 5000)
return return
mode, selected_preset_name = self.preset_editor_widget.get_selected_preset_mode() mode, selected_display_name, preset_file_path = self.preset_editor_widget.get_selected_preset_mode()
if mode == "placeholder": if mode == "placeholder":
log.debug("Update preview called with placeholder preset selected. Showing existing raw inputs (detailed view).") log.debug("Update preview called with placeholder preset selected. Showing existing raw inputs (detailed view).")
@ -749,9 +756,10 @@ class MainWindow(QMainWindow):
# Do not return here; let the function exit normally after handling LLM case. # Do not return here; let the function exit normally after handling LLM case.
# The standard prediction path below will be skipped because mode is 'llm'. # The standard prediction path below will be skipped because mode is 'llm'.
elif mode == "preset" and selected_preset_name: elif mode == "preset" and selected_display_name and preset_file_path:
log.info(f"[{time.time():.4f}] Requesting background preview update for {len(input_paths)} items using Preset='{selected_preset_name}'") preset_name_for_loading = preset_file_path.stem
self.statusBar().showMessage(f"Updating preview for '{selected_preset_name}'...", 0) log.info(f"[{time.time():.4f}] Requesting background preview update for {len(input_paths)} items using Preset Display='{selected_display_name}' (File Stem='{preset_name_for_loading}')")
self.statusBar().showMessage(f"Updating preview for '{selected_display_name}'...", 0)
log.debug("Clearing accumulated rules for new standard preview batch.") log.debug("Clearing accumulated rules for new standard preview batch.")
self._accumulated_rules.clear() self._accumulated_rules.clear()
@ -764,8 +772,8 @@ class MainWindow(QMainWindow):
for input_path_str in input_paths: for input_path_str in input_paths:
file_list = self._extract_file_list(input_path_str) file_list = self._extract_file_list(input_path_str)
if file_list is not None: if file_list is not None:
log.debug(f"[{time.time():.4f}] Emitting start_prediction_signal for: {input_path_str} with {len(file_list)} files.") log.debug(f"[{time.time():.4f}] Emitting start_prediction_signal for: {input_path_str} with {len(file_list)} files, using preset file stem: {preset_name_for_loading}.")
self.start_prediction_signal.emit(input_path_str, file_list, selected_preset_name) self.start_prediction_signal.emit(input_path_str, file_list, preset_name_for_loading) # Pass stem for loading
else: else:
log.warning(f"[{time.time():.4f}] Skipping standard prediction signal for {input_path_str} due to extraction error.") log.warning(f"[{time.time():.4f}] Skipping standard prediction signal for {input_path_str} due to extraction error.")
else: else:
@ -1066,13 +1074,13 @@ class MainWindow(QMainWindow):
log.debug(f"<-- Exiting _handle_prediction_completion for '{input_path}'") log.debug(f"<-- Exiting _handle_prediction_completion for '{input_path}'")
@Slot(str, str) @Slot(str, str, Path) # mode, display_name, file_path (Path can be None)
def _on_preset_selection_changed(self, mode: str, preset_name: str | None): def _on_preset_selection_changed(self, mode: str, display_name: str | None, file_path: Path | None ):
""" """
Handles changes in the preset editor selection (preset, LLM, placeholder). Handles changes in the preset editor selection (preset, LLM, placeholder).
Switches between PresetEditorWidget and LLMEditorWidget. Switches between PresetEditorWidget and LLMEditorWidget.
""" """
log.info(f"Preset selection changed: mode='{mode}', preset_name='{preset_name}'") log.info(f"Preset selection changed: mode='{mode}', display_name='{display_name}', file_path='{file_path}'")
if mode == "llm": if mode == "llm":
log.debug("Switching editor stack to LLM Editor Widget.") log.debug("Switching editor stack to LLM Editor Widget.")
@ -1094,11 +1102,11 @@ class MainWindow(QMainWindow):
self.editor_stack.setCurrentWidget(self.preset_editor_widget.json_editor_container) self.editor_stack.setCurrentWidget(self.preset_editor_widget.json_editor_container)
# The PresetEditorWidget's internal logic handles disabling/clearing the editor fields. # The PresetEditorWidget's internal logic handles disabling/clearing the editor fields.
if mode == "preset" and preset_name: if mode == "preset" and display_name: # Use display_name for window title
# This might be redundant if the editor handles its own title updates on save/load # This might be redundant if the editor handles its own title updates on save/load
# but good for consistency. # but good for consistency.
unsaved = self.preset_editor_widget.editor_unsaved_changes unsaved = self.preset_editor_widget.editor_unsaved_changes
self.setWindowTitle(f"Asset Processor Tool - {preset_name}{'*' if unsaved else ''}") self.setWindowTitle(f"Asset Processor Tool - {display_name}{'*' if unsaved else ''}")
elif mode == "llm": elif mode == "llm":
self.setWindowTitle("Asset Processor Tool - LLM Interpretation") self.setWindowTitle("Asset Processor Tool - LLM Interpretation")
else: else:

335
gui/prediction_handler.py
View File

@ -39,10 +39,9 @@ if not log.hasHandlers():
def classify_files(file_list: List[str], config: Configuration) -> Dict[str, List[Dict[str, Any]]]: def classify_files(file_list: List[str], config: Configuration) -> Dict[str, List[Dict[str, Any]]]:
""" """
Analyzes a list of files based on configuration rules using a two-pass approach Analyzes a list of files based on configuration rules to group them by asset
to group them by asset and determine initial file properties. and determine initial file properties, applying prioritization based on
Pass 1: Identifies and classifies prioritized bit depth variants. 'priority_keywords' in map_type_mapping.
Pass 2: Classifies extras, general maps (downgrading if primary exists), and ignores.
Args: Args:
file_list: List of absolute file paths. file_list: List of absolute file paths.
@ -53,19 +52,21 @@ def classify_files(file_list: List[str], config: Configuration) -> Dict[str, Lis
Example: Example:
{ {
'AssetName1': [ 'AssetName1': [
{'file_path': '/path/to/AssetName1_DISP16.png', 'item_type': 'DISP', 'asset_name': 'AssetName1'}, {'file_path': '/path/to/AssetName1_DISP16.png', 'item_type': 'MAP_DISP', 'asset_name': 'AssetName1'},
{'file_path': '/path/to/AssetName1_DISP.png', 'item_type': 'EXTRA', 'asset_name': 'AssetName1'}, {'file_path': '/path/to/AssetName1_Color.png', 'item_type': 'MAP_COL', 'asset_name': 'AssetName1'}
{'file_path': '/path/to/AssetName1_Color.png', 'item_type': 'COL', 'asset_name': 'AssetName1'}
], ],
# ... other assets # ... other assets
} }
Files marked as "FILE_IGNORE" will also be included in the output.
Returns an empty dict if classification fails or no files are provided. Returns an empty dict if classification fails or no files are provided.
""" """
temp_grouped_files = defaultdict(list) classified_files_info: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
extra_files_to_associate = [] file_matches: Dict[str, List[Tuple[str, int, bool]]] = defaultdict(list) # {file_path: [(target_type, rule_index, is_priority), ...]}
primary_asset_names = set() files_to_ignore: Set[str] = set()
primary_assignments = set()
processed_in_pass1 = set() # --- DEBUG: Log the input file_list ---
log.info(f"DEBUG_ROO_CLASSIFY_INPUT: classify_files received file_list (len={len(file_list)}): {file_list}")
# --- END DEBUG ---
# --- Validation --- # --- Validation ---
if not file_list or not config: if not file_list or not config:
@ -73,20 +74,20 @@ def classify_files(file_list: List[str], config: Configuration) -> Dict[str, Lis
return {} return {}
if not hasattr(config, 'compiled_map_keyword_regex') or not config.compiled_map_keyword_regex: if not hasattr(config, 'compiled_map_keyword_regex') or not config.compiled_map_keyword_regex:
log.warning("Classification skipped: Missing compiled map keyword regex in config.") log.warning("Classification skipped: Missing compiled map keyword regex in config.")
# Proceeding might still classify EXTRA/FILE_IGNORE if those rules exist
if not hasattr(config, 'compiled_extra_regex'): if not hasattr(config, 'compiled_extra_regex'):
log.warning("Configuration object missing 'compiled_extra_regex'. Cannot classify extra files.") log.warning("Configuration object missing 'compiled_extra_regex'. Cannot classify extra files.")
if not hasattr(config, 'compiled_bit_depth_regex_map'): compiled_extra_regex = [] # Provide default to avoid errors
log.warning("Configuration object missing 'compiled_bit_depth_regex_map'. Cannot prioritize bit depth variants.") else:
compiled_extra_regex = getattr(config, 'compiled_extra_regex', [])
compiled_map_regex = getattr(config, 'compiled_map_keyword_regex', {}) compiled_map_regex = getattr(config, 'compiled_map_keyword_regex', {})
compiled_extra_regex = getattr(config, 'compiled_extra_regex', []) # Note: compiled_bit_depth_regex_map is no longer used for primary classification logic here
compiled_bit_depth_regex_map = getattr(config, 'compiled_bit_depth_regex_map', {})
num_map_rules = sum(len(patterns) for patterns in compiled_map_regex.values()) num_map_rules = sum(len(patterns) for patterns in compiled_map_regex.values())
num_extra_rules = len(compiled_extra_regex) num_extra_rules = len(compiled_extra_regex)
num_bit_depth_rules = len(compiled_bit_depth_regex_map)
log.debug(f"Starting classification for {len(file_list)} files using {num_map_rules} map keyword patterns, {num_bit_depth_rules} bit depth patterns, and {num_extra_rules} extra patterns.") log.debug(f"Starting classification for {len(file_list)} files using {num_map_rules} map keyword patterns and {num_extra_rules} extra patterns.")
# --- Asset Name Extraction Helper --- # --- Asset Name Extraction Helper ---
def get_asset_name(f_path: Path, cfg: Configuration) -> str: def get_asset_name(f_path: Path, cfg: Configuration) -> str:
@ -120,155 +121,179 @@ def classify_files(file_list: List[str], config: Configuration) -> Dict[str, Lis
log.warning(f"Asset name extraction resulted in empty string for '{filename}'. Using stem: '{asset_name}'.") log.warning(f"Asset name extraction resulted in empty string for '{filename}'. Using stem: '{asset_name}'.")
return asset_name return asset_name
# --- Pass 1: Prioritized Bit Depth Variants --- # --- Pass 1: Collect all potential matches for each file ---
log.debug("--- Starting Classification Pass 1: Prioritized Variants ---") # For each file, find all map_type_mapping rules it matches (both regular and priority keywords).
# Store the target_type, original rule_index, and whether it was a priority match.
log.debug("--- Starting Classification Pass 1: Collect Potential Matches ---")
file_matches: Dict[str, List[Tuple[str, int, bool]]] = defaultdict(list) # {file_path: [(target_type, rule_index, is_priority), ...]}
files_classified_as_extra: Set[str] = set() # Files already classified as EXTRA
compiled_map_regex = getattr(config, 'compiled_map_keyword_regex', {})
compiled_extra_regex = getattr(config, 'compiled_extra_regex', [])
for file_path_str in file_list: for file_path_str in file_list:
file_path = Path(file_path_str) file_path = Path(file_path_str)
filename = file_path.name filename = file_path.name
asset_name = get_asset_name(file_path, config) asset_name = get_asset_name(file_path, config)
processed = False
for target_type, variant_regex in compiled_bit_depth_regex_map.items(): if "BoucleChunky001" in file_path_str:
match = variant_regex.search(filename) log.info(f"DEBUG_ROO: Processing file: {file_path_str}")
if match:
log.debug(f"PASS 1: File '{filename}' matched PRIORITIZED bit depth variant for type '{target_type}'.")
matched_item_type = target_type
if (asset_name, matched_item_type) in primary_assignments: # Check for EXTRA files first
log.warning(f"PASS 1: Primary assignment ({asset_name}, {matched_item_type}) already exists. File '{filename}' will be handled in Pass 2.")
else:
primary_assignments.add((asset_name, matched_item_type))
log.debug(f" PASS 1: Added primary assignment: ({asset_name}, {matched_item_type})")
primary_asset_names.add(asset_name)
temp_grouped_files[asset_name].append({
'file_path': file_path_str,
'item_type': matched_item_type,
'asset_name': asset_name
})
processed_in_pass1.add(file_path_str)
processed = True
break # Stop checking other variant patterns for this file
log.debug(f"--- Finished Pass 1. Primary assignments made: {primary_assignments} ---")
# --- Pass 2: Extras, General Maps, Ignores ---
log.debug("--- Starting Classification Pass 2: Extras, General Maps, Ignores ---")
for file_path_str in file_list:
if file_path_str in processed_in_pass1:
log.debug(f"PASS 2: Skipping '{Path(file_path_str).name}' (processed in Pass 1).")
continue
file_path = Path(file_path_str)
filename = file_path.name
asset_name = get_asset_name(file_path, config)
is_extra = False is_extra = False
is_map = False
# 1. Check for Extra Files FIRST in Pass 2
for extra_pattern in compiled_extra_regex: for extra_pattern in compiled_extra_regex:
if extra_pattern.search(filename): if extra_pattern.search(filename):
log.debug(f"PASS 2: File '{filename}' matched EXTRA pattern: {extra_pattern.pattern}") if "BoucleChunky001_DISP_1K_METALNESS.png" in filename and extra_pattern.search(filename):
extra_files_to_associate.append((file_path_str, filename)) log.info(f"DEBUG_ROO: EXTRA MATCH: File '{filename}' matched EXTRA pattern: {extra_pattern.pattern}")
log.debug(f"PASS 1: File '{filename}' matched EXTRA pattern: {extra_pattern.pattern}")
# For EXTRA, we assign it directly and don't check map rules for this file
classified_files_info[asset_name].append({
'file_path': file_path_str,
'item_type': "EXTRA",
'asset_name': asset_name
})
files_classified_as_extra.add(file_path_str)
is_extra = True is_extra = True
break break
if is_extra: if "BoucleChunky001_DISP_1K_METALNESS.png" in filename and not is_extra: # after the extra loop
continue log.info(f"DEBUG_ROO: EXTRA CHECK FAILED for {filename}. is_extra: {is_extra}")
# 2. Check for General Map Files in Pass 2 if "BoucleChunky001_DISP_1K_METALNESS.png" in filename and not is_extra:
log.info(f"DEBUG_ROO: EXTRA CHECK FAILED for {filename}. is_extra: {is_extra}")
if is_extra:
continue # Move to the next file
# If not EXTRA, check for MAP matches (collect all potential matches)
for target_type, patterns_list in compiled_map_regex.items(): for target_type, patterns_list in compiled_map_regex.items():
for compiled_regex, original_keyword, rule_index in patterns_list: for compiled_regex, original_keyword, rule_index, is_priority in patterns_list:
match = compiled_regex.search(filename) match = compiled_regex.search(filename)
if match: if match:
try: if "BoucleChunky001" in file_path_str:
# map_type_mapping_list = config.map_type_mapping # Old gloss logic source log.info(f"DEBUG_ROO: PASS 1 MAP MATCH: File '{filename}' matched keyword '{original_keyword}' (priority: {is_priority}) for target type '{target_type}' (Rule Index: {rule_index}).")
# matched_rule_details = map_type_mapping_list[rule_index] # Old gloss logic source log.debug(f" PASS 1: File '{filename}' matched keyword '{original_keyword}' (priority: {is_priority}) for target type '{target_type}' (Rule Index: {rule_index}).")
# is_gloss_flag = matched_rule_details.get('is_gloss_source', False) # Old gloss logic file_matches[file_path_str].append((target_type, rule_index, is_priority))
log.debug(f" PASS 2: Match found! Rule Index: {rule_index}, Keyword: '{original_keyword}', Target: '{target_type}'") # Removed Gloss from log
except Exception as e:
log.exception(f" PASS 2: Error accessing rule details for index {rule_index}: {e}")
# *** Crucial Check: Has a prioritized variant claimed this type? *** log.debug(f"--- Finished Pass 1. Collected matches for {len(file_matches)} files. ---")
if (asset_name, target_type) in primary_assignments:
log.debug(f"PASS 2: File '{filename}' matched '{original_keyword}' for type '{target_type}', but primary already assigned via Pass 1. Classifying as EXTRA.")
matched_item_type = "EXTRA"
# is_gloss_flag = False # Old gloss logic
else:
log.debug(f"PASS 2: File '{filename}' matched '{original_keyword}' for item_type '{target_type}'.")
matched_item_type = target_type
temp_grouped_files[asset_name].append({ # --- Pass 2: Determine Trumped Regular Matches ---
'file_path': file_path_str, # Identify which regular matches are trumped by a priority match for the same rule_index within the asset.
'item_type': matched_item_type, log.debug("--- Starting Classification Pass 2: Determine Trumped Regular Matches ---")
'asset_name': asset_name
})
is_map = True
break
if is_map:
break
# 3. Handle Unmatched Files in Pass 2 (Not Extra, Not Map) trumped_regular_matches: Set[Tuple[str, int]] = set() # Set of (file_path_str, rule_index) pairs that are trumped
if not is_extra and not is_map:
log.debug(f"PASS 2: File '{filename}' did not match any map/extra pattern. Grouping under asset '{asset_name}' as FILE_IGNORE.")
temp_grouped_files[asset_name].append({
'file_path': file_path_str,
'item_type': "FILE_IGNORE",
'asset_name': asset_name
})
log.debug("--- Finished Pass 2 ---") # First, determine which rule_indices have *any* priority match across the entire asset
rule_index_has_priority_match_in_asset: Set[int] = set()
for file_path_str, matches in file_matches.items():
for match_target, match_rule_index, match_is_priority in matches:
if match_is_priority:
rule_index_has_priority_match_in_asset.add(match_rule_index)
# --- Determine Primary Asset Name for Extra Association (using Pass 1 results) --- log.debug(f" Rule indices with priority matches in asset: {sorted(list(rule_index_has_priority_match_in_asset))}")
final_primary_asset_name = None
if primary_asset_names:
primary_map_asset_names_pass1 = [
f_info['asset_name']
for asset_files in temp_grouped_files.values()
for f_info in asset_files
if f_info['asset_name'] in primary_asset_names and (f_info['asset_name'], f_info['item_type']) in primary_assignments
]
if primary_map_asset_names_pass1:
name_counts = Counter(primary_map_asset_names_pass1)
most_common_names = name_counts.most_common()
final_primary_asset_name = most_common_names[0][0]
if len(most_common_names) > 1 and most_common_names[0][1] == most_common_names[1][1]:
tied_names = sorted([name for name, count in most_common_names if count == most_common_names[0][1]])
final_primary_asset_name = tied_names[0]
log.warning(f"Multiple primary asset names tied for most common based on Pass 1: {tied_names}. Using '{final_primary_asset_name}' for associating extra files.")
log.debug(f"Determined primary asset name for extras based on Pass 1 primary maps: '{final_primary_asset_name}'")
else:
log.warning("Primary asset names set (from Pass 1) was populated, but no corresponding groups found. Falling back.")
if not final_primary_asset_name: # Then, for each file, check its matches against the rules that had priority matches
if temp_grouped_files and extra_files_to_associate: for file_path_str in file_list:
fallback_name = sorted(temp_grouped_files.keys())[0] if file_path_str in files_classified_as_extra:
final_primary_asset_name = fallback_name continue
log.warning(f"No primary map files found in Pass 1. Associating extras with first group found alphabetically: '{final_primary_asset_name}'.")
elif extra_files_to_associate: matches_for_this_file = file_matches.get(file_path_str, [])
log.warning(f"Could not determine any asset name to associate {len(extra_files_to_associate)} extra file(s) with. They will be ignored.")
else: # Determine if this file has any priority match for a given rule_index
log.debug("No primary asset name determined (no maps or extras found).") file_has_priority_match_for_rule: Dict[int, bool] = defaultdict(bool)
for match_target, match_rule_index, match_is_priority in matches_for_this_file:
if match_is_priority:
file_has_priority_match_for_rule[match_rule_index] = True
# Determine if this file has any regular match for a given rule_index
file_has_regular_match_for_rule: Dict[int, bool] = defaultdict(bool)
for match_target, match_rule_index, match_is_priority in matches_for_this_file:
if not match_is_priority:
file_has_regular_match_for_rule[match_rule_index] = True
# Identify trumped regular matches for this file
for match_target, match_rule_index, match_is_priority in matches_for_this_file:
if not match_is_priority: # Only consider regular matches
if match_rule_index in rule_index_has_priority_match_in_asset:
# This regular match is for a rule_index that had a priority match somewhere in the asset
if not file_has_priority_match_for_rule[match_rule_index]:
# And this specific file did NOT have a priority match for this rule_index
trumped_regular_matches.add((file_path_str, match_rule_index))
log.debug(f" File '{Path(file_path_str).name}': Regular match for Rule Index {match_rule_index} is trumped.")
if "BoucleChunky001" in file_path_str:
log.info(f"DEBUG_ROO: TRUMPED: File '{Path(file_path_str).name}': Regular match for Rule Index {match_rule_index} (target {match_target}) is trumped.")
if "BoucleChunky001" in file_path_str: # Check if it was actually added by checking the set, or just log if the condition was met
if (file_path_str, match_rule_index) in trumped_regular_matches:
log.info(f"DEBUG_ROO: TRUMPED: File '{Path(file_path_str).name}': Regular match for Rule Index {match_rule_index} (target {match_target}) is trumped.")
# --- Associate Extra Files (collected in Pass 2) --- log.debug(f"--- Finished Pass 2. Identified {len(trumped_regular_matches)} trumped regular matches. ---")
if final_primary_asset_name and extra_files_to_associate:
log.debug(f"Associating {len(extra_files_to_associate)} extra file(s) with primary asset '{final_primary_asset_name}'") # --- Pass 3: Final Assignment & Inter-Entry Resolution ---
for file_path_str, filename in extra_files_to_associate: # Iterate through files, apply ignore rules, and then apply earliest rule wins for remaining valid matches.
if not any(f['file_path'] == file_path_str for f in temp_grouped_files[final_primary_asset_name]): log.debug("--- Starting Classification Pass 3: Final Assignment ---")
temp_grouped_files[final_primary_asset_name].append({
'file_path': file_path_str, final_file_assignments: Dict[str, str] = {} # {file_path: final_item_type}
'item_type': "EXTRA",
'asset_name': final_primary_asset_name
}) for file_path_str in file_list:
# Check if the file was already classified as EXTRA in Pass 1 and added to classified_files_info
if file_path_str in files_classified_as_extra:
log.debug(f" Final Assignment: Skipping '{Path(file_path_str).name}' as it was already classified as EXTRA in Pass 1.")
continue # Skip this file in Pass 3 as it's already handled
asset_name = get_asset_name(Path(file_path_str), config) # Need asset name for the final output structure
# Get valid matches for this file after considering intra-entry priority trumps regular
valid_matches = []
for match_target, match_rule_index, match_is_priority in file_matches.get(file_path_str, []):
if (file_path_str, match_rule_index) not in trumped_regular_matches:
valid_matches.append((match_target, match_rule_index, match_is_priority))
log.debug(f" File '{Path(file_path_str).name}': Valid match - Target: '{match_target}', Rule Index: {match_rule_index}, Priority: {match_is_priority}")
else: else:
log.debug(f"Skipping duplicate association of extra file: {filename}") log.debug(f" File '{Path(file_path_str).name}': Invalid match (trumped by priority) - Target: '{match_target}', Rule Index: {match_rule_index}, Priority: {match_is_priority}")
elif extra_files_to_associate:
pass if "BoucleChunky001" in file_path_str:
log.info(f"DEBUG_ROO: PASS 3 PRE-ASSIGN: File '{Path(file_path_str).name}'. Valid matches: {valid_matches}")
if "BoucleChunky001" in file_path_str:
log.info(f"DEBUG_ROO: PASS 3 PRE-ASSIGN: File '{Path(file_path_str).name}'. Valid matches: {valid_matches}")
final_item_type = "FILE_IGNORE" # Default to ignore if no valid matches
if valid_matches:
# Apply earliest rule wins among valid matches
best_match = min(valid_matches, key=lambda x: x[1]) # Find match with lowest rule_index
final_item_type = best_match[0] # Assign the target_type of the best match
log.debug(f" File '{Path(file_path_str).name}': Best valid match -> Target: '{best_match[0]}', Rule Index: {best_match[1]}. Final type: '{final_item_type}'.")
else:
log.debug(f" File '{Path(file_path_str).name}'': No valid matches after filtering. Final type: '{final_item_type}'.")
if "BoucleChunky001" in file_path_str:
log.info(f"DEBUG_ROO: PASS 3 FINAL ASSIGN: File '{Path(file_path_str).name}' -> Final Type: '{final_item_type}'")
final_file_assignments[file_path_str] = final_item_type
if "BoucleChunky001" in file_path_str:
log.info(f"DEBUG_ROO: PASS 3 FINAL ASSIGN: File '{Path(file_path_str).name}' -> Final Type: '{final_item_type}'")
# Add the file info to the classified_files_info structure
log.info(f"DEBUG_ROO: PASS 3 APPEND: Appending file '{Path(file_path_str).name}' with type '{final_item_type}' to classified_files_info['{asset_name}']")
classified_files_info[asset_name].append({
'file_path': file_path_str,
'item_type': final_item_type,
'asset_name': asset_name
})
log.debug(f" Final Grouping: '{Path(file_path_str).name}' -> '{final_item_type}' (Asset: '{asset_name}')")
log.debug(f"Classification complete. Found {len(temp_grouped_files)} potential assets.") log.debug(f"Classification complete. Found {len(classified_files_info)} potential assets.")
return dict(temp_grouped_files) # Enhanced logging for the content of classified_files_info
boucle_chunky_data = {
key: val for key, val in classified_files_info.items()
if 'BoucleChunky001' in key or any('BoucleChunky001' in (f_info.get('file_path','')) for f_info in val)
}
import json # Make sure json is imported if not already at top of file
log.info(f"DEBUG_ROO: Final classified_files_info for BoucleChunky001 (content): \n{json.dumps(boucle_chunky_data, indent=2)}")
return dict(classified_files_info)
class RuleBasedPredictionHandler(BasePredictionHandler): class RuleBasedPredictionHandler(BasePredictionHandler):
@ -367,7 +392,8 @@ class RuleBasedPredictionHandler(BasePredictionHandler):
source_rule = SourceRule( source_rule = SourceRule(
input_path=input_source_identifier, input_path=input_source_identifier,
supplier_identifier=supplier_identifier, supplier_identifier=supplier_identifier,
preset_name=preset_name # Use the internal display name from the config object
preset_name=config.internal_display_preset_name
) )
asset_rules = [] asset_rules = []
file_type_definitions = config._core_settings.get('FILE_TYPE_DEFINITIONS', {}) file_type_definitions = config._core_settings.get('FILE_TYPE_DEFINITIONS', {})
@ -463,23 +489,22 @@ class RuleBasedPredictionHandler(BasePredictionHandler):
base_item_type = file_info['item_type'] base_item_type = file_info['item_type']
target_asset_name_override = file_info['asset_name'] target_asset_name_override = file_info['asset_name']
final_item_type = base_item_type final_item_type = base_item_type
if not base_item_type.startswith("MAP_") and base_item_type not in ["FILE_IGNORE", "EXTRA", "MODEL"]: # The classification logic now returns the final item_type directly,
final_item_type = f"MAP_{base_item_type}" # including "FILE_IGNORE" and correctly prioritized MAP_ types.
# No need for the old MAP_ prefixing logic here.
if file_type_definitions and final_item_type not in file_type_definitions and base_item_type not in ["FILE_IGNORE", "EXTRA"]: # Validate the final_item_type against definitions, unless it's EXTRA or FILE_IGNORE
log.warning(f"Predicted ItemType '{base_item_type}' (checked as '{final_item_type}') for file '{file_info['file_path']}' is not in FILE_TYPE_DEFINITIONS. Setting to FILE_IGNORE.") if final_item_type not in ["EXTRA", "FILE_IGNORE"] and file_type_definitions and final_item_type not in file_type_definitions:
log.warning(f"Predicted ItemType '{final_item_type}' for file '{file_info['file_path']}' is not in FILE_TYPE_DEFINITIONS. Setting to FILE_IGNORE.")
final_item_type = "FILE_IGNORE" final_item_type = "FILE_IGNORE"
# is_gloss_source_value = file_info.get('is_gloss_source', False) # Removed
file_rule = FileRule( file_rule = FileRule(
file_path=file_info['file_path'], file_path=file_info['file_path'],
item_type=final_item_type, item_type=final_item_type,
item_type_override=final_item_type, item_type_override=final_item_type, # item_type_override defaults to item_type
target_asset_name_override=target_asset_name_override, target_asset_name_override=target_asset_name_override,
output_format_override=None, output_format_override=None,
# is_gloss_source=is_gloss_source_value if isinstance(is_gloss_source_value, bool) else False, # Removed
resolution_override=None, resolution_override=None,
channel_merge_instructions={}, channel_merge_instructions={},
) )
@ -489,6 +514,18 @@ class RuleBasedPredictionHandler(BasePredictionHandler):
source_rule.assets = asset_rules source_rule.assets = asset_rules
source_rules_list.append(source_rule) source_rules_list.append(source_rule)
# DEBUG: Log the structure of the source_rule being emitted
if source_rule and source_rule.assets:
for asset_r_idx, asset_r in enumerate(source_rule.assets):
log.info(f"DEBUG_ROO_EMIT: Source '{input_source_identifier}', Asset {asset_r_idx} ('{asset_r.asset_name}') has {len(asset_r.files)} FileRules.")
for fr_idx, fr in enumerate(asset_r.files):
log.info(f"DEBUG_ROO_EMIT: FR {fr_idx}: Path='{fr.file_path}', Type='{fr.item_type}', TargetAsset='{fr.target_asset_name_override}'")
elif source_rule:
log.info(f"DEBUG_ROO_EMIT: Emitting SourceRule for {input_source_identifier} but it has no assets.")
else:
log.info(f"DEBUG_ROO_EMIT: Attempting to emit for {input_source_identifier}, but source_rule object is None.")
# END DEBUG
except Exception as e: except Exception as e:
log.exception(f"Error building rule hierarchy for source '{input_source_identifier}': {e}") log.exception(f"Error building rule hierarchy for source '{input_source_identifier}': {e}")
raise RuntimeError(f"Error building rule hierarchy: {e}") from e raise RuntimeError(f"Error building rule hierarchy: {e}") from e

80
gui/preset_editor_widget.py
View File

@ -36,8 +36,8 @@ class PresetEditorWidget(QWidget):
# Signal emitted when presets list changes (saved, deleted, new) # Signal emitted when presets list changes (saved, deleted, new)
presets_changed_signal = Signal() presets_changed_signal = Signal()
# Signal emitted when the selected preset (or LLM/Placeholder) changes # Signal emitted when the selected preset (or LLM/Placeholder) changes
# Emits: mode ("preset", "llm", "placeholder"), preset_name (str or None) # Emits: mode ("preset", "llm", "placeholder"), display_name (str or None), file_path (Path or None)
preset_selection_changed_signal = Signal(str, str) preset_selection_changed_signal = Signal(str, str, Path)
def __init__(self, parent=None): def __init__(self, parent=None):
super().__init__(parent) super().__init__(parent)
@ -296,8 +296,22 @@ class PresetEditorWidget(QWidget):
log.warning(msg) log.warning(msg)
else: else:
for preset_path in presets: for preset_path in presets:
item = QListWidgetItem(preset_path.stem) preset_display_name = preset_path.stem # Fallback
item.setData(Qt.ItemDataRole.UserRole, preset_path) try:
with open(preset_path, 'r', encoding='utf-8') as f:
preset_content = json.load(f)
internal_name = preset_content.get("preset_name")
if internal_name and isinstance(internal_name, str) and internal_name.strip():
preset_display_name = internal_name.strip()
else:
log.warning(f"Preset file {preset_path.name} is missing 'preset_name' or it's empty. Using filename stem '{preset_path.stem}' as display name.")
except json.JSONDecodeError:
log.error(f"Failed to parse JSON from {preset_path.name}. Using filename stem '{preset_path.stem}' as display name.")
except Exception as e:
log.error(f"Error reading {preset_path.name}: {e}. Using filename stem '{preset_path.stem}' as display name.")
item = QListWidgetItem(preset_display_name)
item.setData(Qt.ItemDataRole.UserRole, preset_path) # Store the path for loading
self.editor_preset_list.addItem(item) self.editor_preset_list.addItem(item)
log.info(f"Loaded {len(presets)} presets into editor list.") log.info(f"Loaded {len(presets)} presets into editor list.")
@ -525,7 +539,8 @@ class PresetEditorWidget(QWidget):
log.debug(f"PresetEditor: currentItemChanged signal triggered. current: {current_item.text() if current_item else 'None'}") log.debug(f"PresetEditor: currentItemChanged signal triggered. current: {current_item.text() if current_item else 'None'}")
mode = "placeholder" mode = "placeholder"
preset_name = None display_name_to_emit = None # Changed from preset_name
file_path_to_emit = None # New variable for Path
# Check for unsaved changes before proceeding # Check for unsaved changes before proceeding
if self.check_unsaved_changes(): if self.check_unsaved_changes():
@ -540,41 +555,53 @@ class PresetEditorWidget(QWidget):
# Determine mode and preset name based on selection # Determine mode and preset name based on selection
if current_item: if current_item:
item_data = current_item.data(Qt.ItemDataRole.UserRole) item_data = current_item.data(Qt.ItemDataRole.UserRole)
current_display_text = current_item.text() # This is the internal name from populate_presets
if item_data == "__PLACEHOLDER__": if item_data == "__PLACEHOLDER__":
log.debug("Placeholder item selected.") log.debug("Placeholder item selected.")
self._clear_editor() self._clear_editor()
self._set_editor_enabled(False) self._set_editor_enabled(False)
mode = "placeholder" mode = "placeholder"
display_name_to_emit = None
file_path_to_emit = None
self._last_valid_preset_name = None # Clear last valid name self._last_valid_preset_name = None # Clear last valid name
elif item_data == "__LLM__": elif item_data == "__LLM__":
log.debug("LLM Interpretation item selected.") log.debug("LLM Interpretation item selected.")
self._clear_editor() self._clear_editor()
self._set_editor_enabled(False) self._set_editor_enabled(False)
mode = "llm" mode = "llm"
# Keep _last_valid_preset_name as it was display_name_to_emit = None # LLM mode has no specific preset display name
elif isinstance(item_data, Path): file_path_to_emit = None
log.debug(f"Loading preset for editing: {current_item.text()}") # Keep _last_valid_preset_name as it was (it should be the display name)
preset_path = item_data elif isinstance(item_data, Path): # item_data is the Path object for a preset
self._load_preset_for_editing(preset_path) log.debug(f"Loading preset for editing: {current_display_text}")
self._last_valid_preset_name = preset_path.stem preset_file_path_obj = item_data
self._load_preset_for_editing(preset_file_path_obj)
# _last_valid_preset_name should store the display name for delegate use
self._last_valid_preset_name = current_display_text
mode = "preset" mode = "preset"
preset_name = self._last_valid_preset_name display_name_to_emit = current_display_text
else: file_path_to_emit = preset_file_path_obj
else: # Should not happen if list is populated correctly
log.error(f"Invalid data type for preset path: {type(item_data)}. Clearing editor.") log.error(f"Invalid data type for preset path: {type(item_data)}. Clearing editor.")
self._clear_editor() self._clear_editor()
self._set_editor_enabled(False) self._set_editor_enabled(False)
mode = "placeholder" # Treat as placeholder on error mode = "placeholder"
display_name_to_emit = None
file_path_to_emit = None
self._last_valid_preset_name = None self._last_valid_preset_name = None
else: else: # No current_item (e.g., list cleared)
log.debug("No preset selected. Clearing editor.") log.debug("No preset selected. Clearing editor.")
self._clear_editor() self._clear_editor()
self._set_editor_enabled(False) self._set_editor_enabled(False)
mode = "placeholder" mode = "placeholder"
display_name_to_emit = None
file_path_to_emit = None
self._last_valid_preset_name = None self._last_valid_preset_name = None
# Emit the signal regardless of what was selected # Emit the signal with all three arguments
log.debug(f"Emitting preset_selection_changed_signal: mode='{mode}', preset_name='{preset_name}'") log.debug(f"Emitting preset_selection_changed_signal: mode='{mode}', display_name='{display_name_to_emit}', file_path='{file_path_to_emit}'")
self.preset_selection_changed_signal.emit(mode, preset_name) self.preset_selection_changed_signal.emit(mode, display_name_to_emit, file_path_to_emit)
def _gather_editor_data(self) -> dict: def _gather_editor_data(self) -> dict:
"""Gathers data from all editor UI widgets and returns a dictionary.""" """Gathers data from all editor UI widgets and returns a dictionary."""
@ -757,22 +784,25 @@ class PresetEditorWidget(QWidget):
# --- Public Access Methods for MainWindow --- # --- Public Access Methods for MainWindow ---
def get_selected_preset_mode(self) -> tuple[str, str | None]: def get_selected_preset_mode(self) -> tuple[str, str | None, Path | None]:
""" """
Returns the current selection mode and preset name (if applicable). Returns the current selection mode, display name, and file path for loading.
Returns: tuple(mode_string, preset_name_string_or_None) Returns: tuple(mode_string, display_name_string_or_None, file_path_or_None)
mode_string can be "preset", "llm", "placeholder" mode_string can be "preset", "llm", "placeholder"
""" """
current_item = self.editor_preset_list.currentItem() current_item = self.editor_preset_list.currentItem()
if current_item: if current_item:
item_data = current_item.data(Qt.ItemDataRole.UserRole) item_data = current_item.data(Qt.ItemDataRole.UserRole)
display_text = current_item.text() # This is now the internal name
if item_data == "__PLACEHOLDER__": if item_data == "__PLACEHOLDER__":
return "placeholder", None return "placeholder", None, None
elif item_data == "__LLM__": elif item_data == "__LLM__":
return "llm", None return "llm", None, None # LLM mode doesn't have a specific preset file path
elif isinstance(item_data, Path): elif isinstance(item_data, Path):
return "preset", item_data.stem # For a preset, display_text is the internal name, item_data is the Path
return "placeholder", None # Default or if no item selected return "preset", display_text, item_data # Return internal name and path
return "placeholder", None, None # Default or if no item selected
def get_last_valid_preset_name(self) -> str | None: def get_last_valid_preset_name(self) -> str | None:
""" """