58 lines
160 KiB
JSON
58 lines
160 KiB
JSON
{
|
|
"sourceFile": "main.py",
|
|
"activeCommit": 0,
|
|
"commits": [
|
|
{
|
|
"activePatchIndex": 10,
|
|
"patches": [
|
|
{
|
|
"date": 1745226384674,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n"
|
|
},
|
|
{
|
|
"date": 1745226460375,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n@@ -212,10 +212,8 @@\n for i, input_path in enumerate(valid_inputs):\r\n log.debug(f\"Submitting task {i+1}/{len(valid_inputs)} for: {Path(input_path).name}\")\r\n future = executor.submit(\r\n process_single_asset_wrapper,\r\n-:start_line:217\r\n--------\r\n input_path,\r\n preset_name,\r\n output_dir_for_processor,\r\n overwrite,\r\n"
|
|
},
|
|
{
|
|
"date": 1745226476001,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n@@ -70,10 +70,10 @@\n if cores:\r\n default_workers = max(1, cores // 2)\r\n # Cap default workers? Maybe not necessary, let user decide via flag.\r\n # default_workers = min(default_workers, 8) # Example cap\r\n- except NotImplementedError:\r\n- log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n+ except NotImplementedError:\r\n+ log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n \r\n parser = argparse.ArgumentParser(\r\n description=\"Process asset files (ZIPs or folders) into a standardized library format using presets.\",\r\n formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message\r\n@@ -215,10 +215,9 @@\n process_single_asset_wrapper,\r\n input_path,\r\n preset_name,\r\n output_dir_for_processor,\r\n- overwrite,\r\n- args.verbose # Pass the verbose flag\r\n+ overwrite\r\n )\r\n futures[future] = input_path # Store future -> input_path mapping\r\n \r\n # Process completed futures\r\n"
|
|
},
|
|
{
|
|
"date": 1745226486051,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n@@ -62,18 +62,20 @@\n def setup_arg_parser():\r\n \"\"\"Sets up and returns the command-line argument parser.\"\"\"\r\n # Determine a sensible default worker count\r\n default_workers = 1\r\n+:start_line:66\r\n+-------\r\n try:\r\n # Use half the cores, but at least 1, max maybe 8-16? Depends on task nature.\r\n # Let's try max(1, os.cpu_count() // 2)\r\n cores = os.cpu_count()\r\n if cores:\r\n default_workers = max(1, cores // 2)\r\n # Cap default workers? Maybe not necessary, let user decide via flag.\r\n # default_workers = min(default_workers, 8) # Example cap\r\n- except NotImplementedError:\r\n- log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n+ except NotImplementedError:\r\n+ log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n \r\n parser = argparse.ArgumentParser(\r\n description=\"Process asset files (ZIPs or folders) into a standardized library format using presets.\",\r\n formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message\r\n"
|
|
},
|
|
{
|
|
"date": 1745226507748,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n@@ -62,20 +62,18 @@\n def setup_arg_parser():\r\n \"\"\"Sets up and returns the command-line argument parser.\"\"\"\r\n # Determine a sensible default worker count\r\n default_workers = 1\r\n-:start_line:66\r\n--------\r\n try:\r\n # Use half the cores, but at least 1, max maybe 8-16? Depends on task nature.\r\n # Let's try max(1, os.cpu_count() // 2)\r\n cores = os.cpu_count()\r\n if cores:\r\n default_workers = max(1, cores // 2)\r\n # Cap default workers? Maybe not necessary, let user decide via flag.\r\n # default_workers = min(default_workers, 8) # Example cap\r\n- except NotImplementedError:\r\n- log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n+ except NotImplementedError:\r\n+ log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n \r\n parser = argparse.ArgumentParser(\r\n description=\"Process asset files (ZIPs or folders) into a standardized library format using presets.\",\r\n formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message\r\n@@ -308,9 +306,9 @@\n log.error(\"Output directory not specified with -o and OUTPUT_BASE_DIR not found or empty in config.py. Exiting.\")\r\n sys.exit(1)\r\n log.info(f\"Using default output directory from config.py: {output_dir_str}\")\r\n except Exception as e:\r\n- log.error(f\"Could not read OUTPUT_BASE_DIR from config.py: {e}\")\r\n+ log.error(f\"Could not read OUTPUT_BASE_BASE_DIR from config.py: {e}\")\r\n sys.exit(1)\r\n \r\n # --- Resolve Output Path (Handles Relative Paths Explicitly) ---\r\n output_path_obj: Path\r\n"
|
|
},
|
|
{
|
|
"date": 1745226689896,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n@@ -0,0 +1,397 @@\n+# main.py\r\n+\r\n+import argparse\r\n+import sys\r\n+import time\r\n+import os\r\n+import logging\r\n+from pathlib import Path\r\n+from concurrent.futures import ProcessPoolExecutor, as_completed\r\n+import platform # To potentially adjust worker count defaults\r\n+from typing import List, Dict, Tuple, Optional # Added for type hinting\r\n+\r\n+# --- Assuming classes are in sibling files ---\r\n+try:\r\n+ from configuration import Configuration, ConfigurationError\r\n+ from asset_processor import AssetProcessor, AssetProcessingError\r\n+ import config as core_config_module # <<< IMPORT config.py HERE\r\n+except ImportError as e:\r\n+ # Provide a more helpful error message if imports fail\r\n+ script_dir = Path(__file__).parent.resolve()\r\n+ print(f\"ERROR: Failed to import necessary classes: {e}\")\r\n+ print(f\"Ensure 'configuration.py' and 'asset_processor.py' exist in the directory:\")\r\n+ print(f\" {script_dir}\")\r\n+ print(\"Or that the directory is included in your PYTHONPATH.\")\r\n+ sys.exit(1)\r\n+\r\n+# --- Setup Logging ---\r\n+# Keep setup_logging as is, it's called by main() or potentially monitor.py\r\n+def setup_logging(verbose: bool):\r\n+ \"\"\"Configures logging for the application.\"\"\"\r\n+ log_level = logging.DEBUG if verbose else logging.INFO\r\n+ log_format = '%(asctime)s [%(levelname)-8s] %(name)s: %(message)s'\r\n+ date_format = '%Y-%m-%d %H:%M:%S'\r\n+\r\n+ # Configure root logger\r\n+ # Remove existing handlers to avoid duplication if re-run in same session\r\n+ for handler in logging.root.handlers[:]:\r\n+ logging.root.removeHandler(handler)\r\n+\r\n+ logging.basicConfig(\r\n+ level=log_level,\r\n+ format=log_format,\r\n+ datefmt=date_format,\r\n+ handlers=[\r\n+ logging.StreamHandler(sys.stdout) # Log to console\r\n+ # Optional: Add FileHandler for persistent logs\r\n+ # logging.FileHandler(\"asset_processor.log\", mode='a', encoding='utf-8')\r\n+ ]\r\n+ )\r\n+ # Get logger specifically for this main script\r\n+ log = logging.getLogger(__name__) # or use 'main'\r\n+ log.info(f\"Logging level set to: {logging.getLevelName(log_level)}\")\r\n+ # Suppress overly verbose messages from libraries if needed (e.g., cv2)\r\n+ # logging.getLogger('cv2').setLevel(logging.WARNING)\r\n+\r\n+# Use module-level logger after configuration\r\n+log = logging.getLogger(__name__)\r\n+\r\n+\r\n+# --- Argument Parser Setup ---\r\n+# Keep setup_arg_parser as is, it's only used when running main.py directly\r\n+def setup_arg_parser():\r\n+ \"\"\"Sets up and returns the command-line argument parser.\"\"\"\r\n+ # Determine a sensible default worker count\r\n+ default_workers = 1\r\n+ try:\r\n+ # Use half the cores, but at least 1, max maybe 8-16? Depends on task nature.\r\n+ # Let's try max(1, os.cpu_count() // 2)\r\n+ cores = os.cpu_count()\r\n+ if cores:\r\n+ default_workers = max(1, cores // 2)\r\n+ # Cap default workers? Maybe not necessary, let user decide via flag.\r\n+ # default_workers = min(default_workers, 8) # Example cap\r\n+ except NotImplementedError:\r\n+ log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n+\r\n+ parser = argparse.ArgumentParser(\r\n+ description=\"Process asset files (ZIPs or folders) into a standardized library format using presets.\",\r\n+ formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message\r\n+ )\r\n+ parser.add_argument(\r\n+ \"input_paths\",\r\n+ metavar=\"INPUT_PATH\",\r\n+ type=str,\r\n+ nargs='+', # Requires one or more input paths\r\n+ help=\"Path(s) to the input ZIP file(s) or folder(s) containing assets.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-p\", \"--preset\",\r\n+ type=str,\r\n+ required=True,\r\n+ help=\"Name of the configuration preset (e.g., 'poliigon') located in the 'presets' directory (without .json extension).\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-o\", \"--output-dir\",\r\n+ type=str,\r\n+ required=False, # No longer required\r\n+ default=None, # Default is None, will check core_config later\r\n+ help=\"Override the default base output directory defined in config.py.\" # Updated help\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-w\", \"--workers\",\r\n+ type=int,\r\n+ default=default_workers,\r\n+ help=\"Maximum number of assets to process concurrently in parallel processes.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-v\", \"--verbose\",\r\n+ action=\"store_true\", # Makes it a flag, value is True if present\r\n+ help=\"Enable detailed DEBUG level logging for troubleshooting.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"--overwrite\",\r\n+ action=\"store_true\",\r\n+ help=\"Force reprocessing and overwrite existing output asset folders if they exist.\"\r\n+ )\r\n+ # Potential future flags:\r\n+ # parser.add_argument(\"--log-file\", type=str, default=None, help=\"Path to save log output to a file.\")\r\n+ return parser\r\n+\r\n+\r\n+# --- Worker Function ---\r\n+# Keep process_single_asset_wrapper as is, it's called by the processing pool\r\n+def process_single_asset_wrapper(input_path_str: str, preset_name: str, output_dir_str: str, overwrite: bool) -> Tuple[str, str, Optional[str]]:\r\n+ \"\"\"\r\n+ Wrapper function for processing a single asset in a separate process.\r\n+ Handles instantiation of Configuration and AssetProcessor, passes the overwrite flag, and catches errors.\r\n+ Returns: (input_path_str, status_string [\"processed\", \"skipped\", \"failed\"], error_message_or_None)\r\n+ \"\"\"\r\n+ # Note: Logging setup might need re-initialization in child processes on some platforms\r\n+ # if file handlers or complex configurations are used. Stdout usually works.\r\n+ # Removed forced DEBUG logging for worker\r\n+ worker_log = logging.getLogger(f\"Worker_{os.getpid()}\") # Log with worker PID\r\n+ # Initial log message moved inside try block for better status reporting\r\n+\r\n+ try:\r\n+ worker_log.info(f\"Starting processing attempt for: {Path(input_path_str).name}\")\r\n+ # Each worker needs its own Configuration instance based on the preset name\r\n+ config = Configuration(preset_name)\r\n+ # Ensure output_dir_str is treated as the absolute base path\r\n+ output_base_path = Path(output_dir_str)\r\n+ input_path = Path(input_path_str)\r\n+\r\n+ # Pass the overwrite flag to the AssetProcessor\r\n+ processor = AssetProcessor(input_path, config, output_base_path, overwrite=overwrite)\r\n+ status = processor.process() # <<< Execute the main processing pipeline and get status\r\n+\r\n+ # Log based on status\r\n+ if status == \"skipped\":\r\n+ worker_log.info(f\"Worker skipped: {input_path.name}\")\r\n+ elif status == \"processed\":\r\n+ worker_log.info(f\"Worker finished processing: {input_path.name}\")\r\n+ # Note: Failures within process() should raise exceptions caught below\r\n+\r\n+ return (input_path_str, status, None) # Return the status string\r\n+\r\n+ except (ConfigurationError, AssetProcessingError) as e:\r\n+ worker_log.error(f\"Failed: {Path(input_path_str).name} - {type(e).__name__}: {e}\")\r\n+ return (input_path_str, \"failed\", f\"{type(e).__name__}: {e}\") # Return \"failed\" status\r\n+ except Exception as e:\r\n+ # Catch any other unexpected errors originating from the worker process\r\n+ # Use exc_info=True to log the full traceback from the worker\r\n+ worker_log.exception(f\"Unexpected failure processing {Path(input_path_str).name}: {e}\")\r\n+ return (input_path_str, \"failed\", f\"Unexpected Worker Error: {e}\") # Return \"failed\" status\r\n+\r\n+\r\n+# --- Core Processing Function ---\r\n+def run_processing(\r\n+ valid_inputs: List[str],\r\n+ preset_name: str,\r\n+ output_dir_for_processor: str,\r\n+ overwrite: bool,\r\n+ num_workers: int\r\n+) -> Dict:\r\n+ \"\"\"\r\n+ Executes the core asset processing logic using a process pool.\r\n+\r\n+ Args:\r\n+ valid_inputs: List of validated input file/directory paths (strings).\r\n+ preset_name: Name of the preset to use.\r\n+ output_dir_for_processor: Absolute path string for the output base directory.\r\n+ overwrite: Boolean flag to force reprocessing.\r\n+ num_workers: Maximum number of worker processes.\r\n+\r\n+ Returns:\r\n+ A dictionary containing processing results:\r\n+ {\r\n+ \"processed\": int,\r\n+ \"skipped\": int,\r\n+ \"failed\": int,\r\n+ \"results_list\": List[Tuple[str, str, Optional[str]]] # (input_path, status, error_msg)\r\n+ }\r\n+ \"\"\"\r\n+ log.info(f\"Processing {len(valid_inputs)} asset(s) using preset '{preset_name}' with up to {num_workers} worker(s)...\")\r\n+ results_list = []\r\n+ successful_processed_count = 0\r\n+ skipped_count = 0\r\n+ failed_count = 0\r\n+\r\n+ # Ensure at least one worker\r\n+ num_workers = max(1, num_workers)\r\n+\r\n+ # Using ProcessPoolExecutor is generally good if AssetProcessor tasks are CPU-bound.\r\n+ # If tasks are mostly I/O bound, ThreadPoolExecutor might be sufficient.\r\n+ # Important: Ensure Configuration and AssetProcessor are \"pickleable\".\r\n+ try:\r\n+ with ProcessPoolExecutor(max_workers=num_workers) as executor:\r\n+ # Create futures\r\n+ futures = {}\r\n+ log.debug(f\"Submitting {len(valid_inputs)} tasks...\")\r\n+ # Removed the 1-second delay for potentially faster submission in non-CLI use\r\n+ for i, input_path in enumerate(valid_inputs):\r\n+ log.debug(f\"Submitting task {i+1}/{len(valid_inputs)} for: {Path(input_path).name}\")\r\n+ future = executor.submit(\r\n+ process_single_asset_wrapper,\r\n+ input_path,\r\n+ preset_name,\r\n+ output_dir_for_processor,\r\n+ overwrite\r\n+ )\r\n+ futures[future] = input_path # Store future -> input_path mapping\r\n+\r\n+ # Process completed futures\r\n+ for i, future in enumerate(as_completed(futures), 1):\r\n+ input_path = futures[future]\r\n+ asset_name = Path(input_path).name\r\n+ log.info(f\"--- [{i}/{len(valid_inputs)}] Worker finished attempt for: {asset_name} ---\")\r\n+ try:\r\n+ # Get result tuple: (input_path_str, status_string, error_message_or_None)\r\n+ result_tuple = future.result()\r\n+ results_list.append(result_tuple)\r\n+ input_path_res, status, err_msg = result_tuple\r\n+\r\n+ # Increment counters based on status\r\n+ if status == \"processed\":\r\n+ successful_processed_count += 1\r\n+ elif status == \"skipped\":\r\n+ skipped_count += 1\r\n+ elif status == \"failed\":\r\n+ failed_count += 1\r\n+ else: # Should not happen, but log as warning/failure\r\n+ log.warning(f\"Unknown status '{status}' received for {asset_name}. Counting as failed.\")\r\n+ failed_count += 1\r\n+\r\n+ except Exception as e:\r\n+ # Catch errors if the future itself fails (e.g., worker process crashed hard)\r\n+ log.exception(f\"Critical worker failure for {asset_name}: {e}\")\r\n+ results_list.append((input_path, \"failed\", f\"Worker process crashed: {e}\"))\r\n+ failed_count += 1 # Count crashes as failures\r\n+\r\n+ except Exception as pool_exc:\r\n+ log.exception(f\"An error occurred with the process pool: {pool_exc}\")\r\n+ # Re-raise or handle as appropriate for the calling context (monitor.py)\r\n+ # For now, log and return current counts\r\n+ return {\r\n+ \"processed\": successful_processed_count,\r\n+ \"skipped\": skipped_count,\r\n+ \"failed\": failed_count + (len(valid_inputs) - len(results_list)), # Count unprocessed as failed\r\n+ \"results_list\": results_list,\r\n+ \"pool_error\": str(pool_exc) # Add pool error info\r\n+ }\r\n+\r\n+ return {\r\n+ \"processed\": successful_processed_count,\r\n+ \"skipped\": skipped_count,\r\n+ \"failed\": failed_count,\r\n+ \"results_list\": results_list\r\n+ }\r\n+\r\n+\r\n+# --- Main Execution (for CLI usage) ---\r\n+def main():\r\n+ \"\"\"Parses arguments, sets up logging, runs processing, and reports summary.\"\"\"\r\n+ parser = setup_arg_parser()\r\n+ args = parser.parse_args()\r\n+\r\n+ # Setup logging based on verbosity argument *before* logging status messages\r\n+ setup_logging(args.verbose)\r\n+\r\n+ start_time = time.time()\r\n+ log.info(\"Asset Processor Script Started (CLI Mode)\")\r\n+\r\n+ # --- Validate Input Paths ---\r\n+ valid_inputs = []\r\n+ for p_str in args.input_paths:\r\n+ p = Path(p_str)\r\n+ if p.exists():\r\n+ if p.is_dir() or (p.is_file() and p.suffix.lower() == '.zip'):\r\n+ valid_inputs.append(p_str) # Store the original string path\r\n+ else:\r\n+ log.warning(f\"Input is not a directory or .zip, skipping: {p_str}\")\r\n+ else:\r\n+ log.warning(f\"Input path not found, skipping: {p_str}\")\r\n+\r\n+ if not valid_inputs:\r\n+ log.error(\"No valid input paths found. Exiting.\")\r\n+ sys.exit(1) # Exit with error code\r\n+\r\n+ # --- Determine Output Directory ---\r\n+ output_dir_str = args.output_dir # Get value from args (might be None)\r\n+ if not output_dir_str:\r\n+ log.debug(\"Output directory not specified via -o, reading default from config.py.\")\r\n+ try:\r\n+ output_dir_str = getattr(core_config_module, 'OUTPUT_BASE_DIR', None)\r\n+ if not output_dir_str:\r\n+ log.error(\"Output directory not specified with -o and OUTPUT_BASE_DIR not found or empty in config.py. Exiting.\")\r\n+ sys.exit(1)\r\n+ log.info(f\"Using default output directory from config.py: {output_dir_str}\")\r\n+ except Exception as e:\r\n+ log.error(f\"Could not read OUTPUT_BASE_BASE_DIR from config.py: {e}\")\r\n+ sys.exit(1)\r\n+\r\n+ # --- Resolve Output Path (Handles Relative Paths Explicitly) ---\r\n+ output_path_obj: Path\r\n+ if os.path.isabs(output_dir_str):\r\n+ output_path_obj = Path(output_dir_str)\r\n+ log.info(f\"Using absolute output directory: {output_path_obj}\")\r\n+ else:\r\n+ # Path() interprets relative paths against CWD by default\r\n+ output_path_obj = Path(output_dir_str)\r\n+ log.info(f\"Using relative output directory '{output_dir_str}'. Resolved against CWD to: {output_path_obj.resolve()}\")\r\n+\r\n+ # --- Validate and Setup Output Directory ---\r\n+ try:\r\n+ # Resolve to ensure we have an absolute path for consistency and creation\r\n+ resolved_output_dir = output_path_obj.resolve()\r\n+ log.info(f\"Ensuring output directory exists: {resolved_output_dir}\")\r\n+ resolved_output_dir.mkdir(parents=True, exist_ok=True)\r\n+ # Use the resolved absolute path string for the processor\r\n+ output_dir_for_processor = str(resolved_output_dir)\r\n+ except Exception as e:\r\n+ log.error(f\"Cannot create or access output directory '{resolved_output_dir}': {e}\", exc_info=True)\r\n+ sys.exit(1)\r\n+\r\n+ # --- Check Preset Existence (Basic Check) ---\r\n+ preset_dir = Path(__file__).parent / \"presets\"\r\n+ preset_file = preset_dir / f\"{args.preset}.json\"\r\n+ if not preset_file.is_file():\r\n+ log.error(f\"Preset file not found: {preset_file}\")\r\n+ log.error(f\"Ensure a file named '{args.preset}.json' exists in the directory: {preset_dir.resolve()}\")\r\n+ sys.exit(1)\r\n+\r\n+ # --- Execute Processing via the new function ---\r\n+ processing_results = run_processing(\r\n+ valid_inputs=valid_inputs,\r\n+ preset_name=args.preset,\r\n+ output_dir_for_processor=output_dir_for_processor,\r\n+ overwrite=args.overwrite,\r\n+ num_workers=args.workers\r\n+ )\r\n+\r\n+ # --- Report Summary ---\r\n+ duration = time.time() - start_time\r\n+ successful_processed_count = processing_results[\"processed\"]\r\n+ skipped_count = processing_results[\"skipped\"]\r\n+ failed_count = processing_results[\"failed\"]\r\n+ results_list = processing_results[\"results_list\"]\r\n+\r\n+ log.info(\"=\" * 40)\r\n+ log.info(\"Processing Summary\")\r\n+ log.info(f\" Duration: {duration:.2f} seconds\")\r\n+ log.info(f\" Assets Attempted: {len(valid_inputs)}\")\r\n+ log.info(f\" Successfully Processed: {successful_processed_count}\")\r\n+ log.info(f\" Skipped (Already Existed): {skipped_count}\")\r\n+ log.info(f\" Failed: {failed_count}\")\r\n+\r\n+ if processing_results.get(\"pool_error\"):\r\n+ log.error(f\" Process Pool Error: {processing_results['pool_error']}\")\r\n+ # Ensure failed count reflects pool error if it happened\r\n+ if failed_count == 0 and successful_processed_count == 0 and skipped_count == 0:\r\n+ failed_count = len(valid_inputs) # Assume all failed if pool died early\r\n+\r\n+ exit_code = 0\r\n+ if failed_count > 0:\r\n+ log.warning(\"Failures occurred:\")\r\n+ # Iterate through results to show specific errors for failed items\r\n+ for input_path, status, err_msg in results_list:\r\n+ if status == \"failed\":\r\n+ log.warning(f\" - {Path(input_path).name}: {err_msg}\")\r\n+ exit_code = 1 # Exit with error code if failures occurred\r\n+ else:\r\n+ # Consider skipped assets as a form of success for the overall run exit code\r\n+ if successful_processed_count > 0 or skipped_count > 0:\r\n+ log.info(\"All assets processed or skipped successfully.\")\r\n+ exit_code = 0 # Exit code 0 indicates success (including skips)\r\n+ else:\r\n+ # This case might happen if all inputs were invalid initially\r\n+ log.warning(\"No assets were processed, skipped, or failed (check input validation logs).\")\r\n+ exit_code = 0 # Still exit 0 as the script itself didn't crash\r\n+\r\n+ sys.exit(exit_code)\r\n+\r\n+\r\n+if __name__ == \"__main__\":\r\n+ # This ensures the main() function runs only when the script is executed directly\r\n+ # Important for multiprocessing to work correctly on some platforms (like Windows)\r\n+ main()\n\\ No newline at end of file\n"
|
|
},
|
|
{
|
|
"date": 1745226706422,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n@@ -0,0 +1,408 @@\n+# main.py\r\n+\r\n+import argparse\r\n+import sys\r\n+import time\r\n+import os\r\n+import logging\r\n+from pathlib import Path\r\n+from concurrent.futures import ProcessPoolExecutor, as_completed\r\n+import platform # To potentially adjust worker count defaults\r\n+from typing import List, Dict, Tuple, Optional # Added for type hinting\r\n+\r\n+# --- Assuming classes are in sibling files ---\r\n+try:\r\n+ from configuration import Configuration, ConfigurationError\r\n+ from asset_processor import AssetProcessor, AssetProcessingError\r\n+ import config as core_config_module # <<< IMPORT config.py HERE\r\n+except ImportError as e:\r\n+ # Provide a more helpful error message if imports fail\r\n+ script_dir = Path(__file__).parent.resolve()\r\n+ print(f\"ERROR: Failed to import necessary classes: {e}\")\r\n+ print(f\"Ensure 'configuration.py' and 'asset_processor.py' exist in the directory:\")\r\n+ print(f\" {script_dir}\")\r\n+ print(\"Or that the directory is included in your PYTHONPATH.\")\r\n+ sys.exit(1)\r\n+\r\n+# --- Setup Logging ---\r\n+# Keep setup_logging as is, it's called by main() or potentially monitor.py\r\n+def setup_logging(verbose: bool):\r\n+ \"\"\"Configures logging for the application.\"\"\"\r\n+ log_level = logging.DEBUG if verbose else logging.INFO\r\n+ log_format = '%(asctime)s [%(levelname)-8s] %(name)s: %(message)s'\r\n+ date_format = '%Y-%m-%d %H:%M:%S'\r\n+\r\n+ # Configure root logger\r\n+ # Remove existing handlers to avoid duplication if re-run in same session\r\n+ for handler in logging.root.handlers[:]:\r\n+ logging.root.removeHandler(handler)\r\n+\r\n+ logging.basicConfig(\r\n+ level=log_level,\r\n+ format=log_format,\r\n+ datefmt=date_format,\r\n+ handlers=[\r\n+ logging.StreamHandler(sys.stdout) # Log to console\r\n+ # Optional: Add FileHandler for persistent logs\r\n+ # logging.FileHandler(\"asset_processor.log\", mode='a', encoding='utf-8')\r\n+ ]\r\n+ )\r\n+ # Get logger specifically for this main script\r\n+ log = logging.getLogger(__name__) # or use 'main'\r\n+ log.info(f\"Logging level set to: {logging.getLevelName(log_level)}\")\r\n+ # Suppress overly verbose messages from libraries if needed (e.g., cv2)\r\n+ # logging.getLogger('cv2').setLevel(logging.WARNING)\r\n+\r\n+# Use module-level logger after configuration\r\n+log = logging.getLogger(__name__)\r\n+\r\n+\r\n+# --- Argument Parser Setup ---\r\n+# Keep setup_arg_parser as is, it's only used when running main.py directly\r\n+def setup_arg_parser():\r\n+ \"\"\"Sets up and returns the command-line argument parser.\"\"\"\r\n+ # Determine a sensible default worker count\r\n+ default_workers = 1\r\n+ try:\r\n+ # Use half the cores, but at least 1, max maybe 8-16? Depends on task nature.\r\n+ # Let's try max(1, os.cpu_count() // 2)\r\n+ cores = os.cpu_count()\r\n+ if cores:\r\n+ default_workers = max(1, cores // 2)\r\n+ # Cap default workers? Maybe not necessary, let user decide via flag.\r\n+ # default_workers = min(default_workers, 8) # Example cap\r\n+ except NotImplementedError:\r\n+ log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n+\r\n+ parser = argparse.ArgumentParser(\r\n+ description=\"Process asset files (ZIPs or folders) into a standardized library format using presets.\",\r\n+ formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message\r\n+ )\r\n+ parser.add_argument(\r\n+ \"input_paths\",\r\n+ metavar=\"INPUT_PATH\",\r\n+ type=str,\r\n+ nargs='+', # Requires one or more input paths\r\n+ help=\"Path(s) to the input ZIP file(s) or folder(s) containing assets.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-p\", \"--preset\",\r\n+ type=str,\r\n+ required=True,\r\n+ help=\"Name of the configuration preset (e.g., 'poliigon') located in the 'presets' directory (without .json extension).\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-o\", \"--output-dir\",\r\n+ type=str,\r\n+ required=False, # No longer required\r\n+ default=None, # Default is None, will check core_config later\r\n+ help=\"Override the default base output directory defined in config.py.\" # Updated help\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-w\", \"--workers\",\r\n+ type=int,\r\n+ default=default_workers,\r\n+ help=\"Maximum number of assets to process concurrently in parallel processes.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-v\", \"--verbose\",\r\n+ action=\"store_true\", # Makes it a flag, value is True if present\r\n+ help=\"Enable detailed DEBUG level logging for troubleshooting.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"--overwrite\",\r\n+ action=\"store_true\",\r\n+ help=\"Force reprocessing and overwrite existing output asset folders if they exist.\"\r\n+ )\r\n+ # Potential future flags:\r\n+ # parser.add_argument(\"--log-file\", type=str, default=None, help=\"Path to save log output to a file.\")\r\n+ return parser\r\n+\r\n+\r\n+# --- Worker Function ---\r\n+def process_single_asset_wrapper(input_path_str: str, preset_name: str, output_dir_str: str, overwrite: bool, verbose: bool) -> Tuple[str, str, Optional[str]]:\r\n+ \"\"\"\r\n+ Wrapper function for processing a single asset in a separate process.\r\n+ Handles instantiation of Configuration and AssetProcessor, passes the overwrite flag, and catches errors.\r\n+ Ensures logging is configured for the worker process.\r\n+ Returns: (input_path_str, status_string [\"processed\", \"skipped\", \"failed\"], error_message_or_None)\r\n+ \"\"\"\r\n+ # Explicitly configure logging for this worker process\r\n+ worker_log = logging.getLogger(f\"Worker_{os.getpid()}\") # Log with worker PID\r\n+ # Check if root logger already has handlers (might happen in some environments)\r\n+ if not logging.root.handlers:\r\n+ # Basic config if no handlers are set up (should be handled by main, but safety)\r\n+ logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)-8s] %(name)s: %(message)s')\r\n+ # Set the level for this worker's logger based on the verbose flag\r\n+ worker_log.setLevel(logging.DEBUG if verbose else logging.INFO)\r\n+ # Also ensure the root logger level is set if verbose\r\n+ if verbose:\r\n+ logging.root.setLevel(logging.DEBUG)\r\n+\r\n+\r\n+ try:\r\n+ worker_log.info(f\"Starting processing attempt for: {Path(input_path_str).name}\")\r\n+ # Each worker needs its own Configuration instance based on the preset name\r\n+ config = Configuration(preset_name)\r\n+ # Ensure output_dir_str is treated as the absolute base path\r\n+ output_base_path = Path(output_dir_str)\r\n+ input_path = Path(input_path_str)\r\n+\r\n+ # Pass the overwrite flag to the AssetProcessor\r\n+ processor = AssetProcessor(input_path, config, output_base_path, overwrite=overwrite)\r\n+ status = processor.process() # <<< Execute the main processing pipeline and get status\r\n+\r\n+ # Log based on status\r\n+ if status == \"skipped\":\r\n+ worker_log.info(f\"Worker skipped: {input_path.name}\")\r\n+ elif status == \"processed\":\r\n+ worker_log.info(f\"Worker finished processing: {input_path.name}\")\r\n+ # Note: Failures within process() should raise exceptions caught below\r\n+\r\n+ return (input_path_str, status, None) # Return the status string\r\n+\r\n+ except (ConfigurationError, AssetProcessingError) as e:\r\n+ worker_log.error(f\"Failed: {Path(input_path_str).name} - {type(e).__name__}: {e}\")\r\n+ return (input_path_str, \"failed\", f\"{type(e).__name__}: {e}\") # Return \"failed\" status\r\n+ except Exception as e:\r\n+ # Catch any other unexpected errors originating from the worker process\r\n+ # Use exc_info=True to log the full traceback from the worker\r\n+ worker_log.exception(f\"Unexpected failure processing {Path(input_path_str).name}: {e}\")\r\n+ return (input_path_str, \"failed\", f\"Unexpected Worker Error: {e}\") # Return \"failed\" status\r\n+\r\n+\r\n+# --- Core Processing Function ---\r\n+def run_processing(\r\n+ valid_inputs: List[str],\r\n+ preset_name: str,\r\n+ output_dir_for_processor: str,\r\n+ overwrite: bool,\r\n+ num_workers: int,\r\n+ verbose: bool # Add verbose parameter here\r\n+) -> Dict:\r\n+ \"\"\"\r\n+ Executes the core asset processing logic using a process pool.\r\n+\r\n+ Args:\r\n+ valid_inputs: List of validated input file/directory paths (strings).\r\n+ preset_name: Name of the preset to use.\r\n+ output_dir_for_processor: Absolute path string for the output base directory.\r\n+ overwrite: Boolean flag to force reprocessing.\r\n+ num_workers: Maximum number of worker processes.\r\n+ verbose: Boolean flag for verbose logging.\r\n+\r\n+ Returns:\r\n+ A dictionary containing processing results:\r\n+ {\r\n+ \"processed\": int,\r\n+ \"skipped\": int,\r\n+ \"failed\": int,\r\n+ \"results_list\": List[Tuple[str, str, Optional[str]]] # (input_path, status, error_msg)\r\n+ }\r\n+ \"\"\"\r\n+ log.info(f\"Processing {len(valid_inputs)} asset(s) using preset '{preset_name}' with up to {num_workers} worker(s)...\")\r\n+ results_list = []\r\n+ successful_processed_count = 0\r\n+ skipped_count = 0\r\n+ failed_count = 0\r\n+\r\n+ # Ensure at least one worker\r\n+ num_workers = max(1, num_workers)\r\n+\r\n+ # Using ProcessPoolExecutor is generally good if AssetProcessor tasks are CPU-bound.\r\n+ # If tasks are mostly I/O bound, ThreadPoolExecutor might be sufficient.\r\n+ # Important: Ensure Configuration and AssetProcessor are \"pickleable\".\r\n+ try:\r\n+ with ProcessPoolExecutor(max_workers=num_workers) as executor:\r\n+ # Create futures\r\n+ futures = {}\r\n+ log.debug(f\"Submitting {len(valid_inputs)} tasks...\")\r\n+ # Removed the 1-second delay for potentially faster submission in non-CLI use\r\n+ for i, input_path in enumerate(valid_inputs):\r\n+ log.debug(f\"Submitting task {i+1}/{len(valid_inputs)} for: {Path(input_path).name}\")\r\n+ future = executor.submit(\r\n+ process_single_asset_wrapper,\r\n+ input_path,\r\n+ preset_name,\r\n+ output_dir_for_processor,\r\n+ overwrite,\r\n+ verbose # Pass the verbose flag\r\n+ )\r\n+ futures[future] = input_path # Store future -> input_path mapping\r\n+\r\n+ # Process completed futures\r\n+ for i, future in enumerate(as_completed(futures), 1):\r\n+ input_path = futures[future]\r\n+ asset_name = Path(input_path).name\r\n+ log.info(f\"--- [{i}/{len(valid_inputs)}] Worker finished attempt for: {asset_name} ---\")\r\n+ try:\r\n+ # Get result tuple: (input_path_str, status_string, error_message_or_None)\r\n+ result_tuple = future.result()\r\n+ results_list.append(result_tuple)\r\n+ input_path_res, status, err_msg = result_tuple\r\n+\r\n+ # Increment counters based on status\r\n+ if status == \"processed\":\r\n+ successful_processed_count += 1\r\n+ elif status == \"skipped\":\r\n+ skipped_count += 1\r\n+ elif status == \"failed\":\r\n+ failed_count += 1\r\n+ else: # Should not happen, but log as warning/failure\r\n+ log.warning(f\"Unknown status '{status}' received for {asset_name}. Counting as failed.\")\r\n+ failed_count += 1\r\n+\r\n+ except Exception as e:\r\n+ # Catch errors if the future itself fails (e.g., worker process crashed hard)\r\n+ log.exception(f\"Critical worker failure for {asset_name}: {e}\")\r\n+ results_list.append((input_path, \"failed\", f\"Worker process crashed: {e}\"))\r\n+ failed_count += 1 # Count crashes as failures\r\n+\r\n+ except Exception as pool_exc:\r\n+ log.exception(f\"An error occurred with the process pool: {pool_exc}\")\r\n+ # Re-raise or handle as appropriate for the calling context (monitor.py)\r\n+ # For now, log and return current counts\r\n+ return {\r\n+ \"processed\": successful_processed_count,\r\n+ \"skipped\": skipped_count,\r\n+ \"failed\": failed_count + (len(valid_inputs) - len(results_list)), # Count unprocessed as failed\r\n+ \"results_list\": results_list,\r\n+ \"pool_error\": str(pool_exc) # Add pool error info\r\n+ }\r\n+\r\n+ return {\r\n+ \"processed\": successful_processed_count,\r\n+ \"skipped\": skipped_count,\r\n+ \"failed\": failed_count,\r\n+ \"results_list\": results_list\r\n+ }\r\n+\r\n+\r\n+# --- Main Execution (for CLI usage) ---\r\n+def main():\r\n+ \"\"\"Parses arguments, sets up logging, runs processing, and reports summary.\"\"\"\r\n+ parser = setup_arg_parser()\r\n+ args = parser.parse_args()\r\n+\r\n+ # Setup logging based on verbosity argument *before* logging status messages\r\n+ setup_logging(args.verbose)\r\n+\r\n+ start_time = time.time()\r\n+ log.info(\"Asset Processor Script Started (CLI Mode)\")\r\n+\r\n+ # --- Validate Input Paths ---\r\n+ valid_inputs = []\r\n+ for p_str in args.input_paths:\r\n+ p = Path(p_str)\r\n+ if p.exists():\r\n+ if p.is_dir() or (p.is_file() and p.suffix.lower() == '.zip'):\r\n+ valid_inputs.append(p_str) # Store the original string path\r\n+ else:\r\n+ log.warning(f\"Input is not a directory or .zip, skipping: {p_str}\")\r\n+ else:\r\n+ log.warning(f\"Input path not found, skipping: {p_str}\")\r\n+\r\n+ if not valid_inputs:\r\n+ log.error(\"No valid input paths found. Exiting.\")\r\n+ sys.exit(1) # Exit with error code\r\n+\r\n+ # --- Determine Output Directory ---\r\n+ output_dir_str = args.output_dir # Get value from args (might be None)\r\n+ if not output_dir_str:\r\n+ log.debug(\"Output directory not specified via -o, reading default from config.py.\")\r\n+ try:\r\n+ output_dir_str = getattr(core_config_module, 'OUTPUT_BASE_DIR', None)\r\n+ if not output_dir_str:\r\n+ log.error(\"Output directory not specified with -o and OUTPUT_BASE_DIR not found or empty in config.py. Exiting.\")\r\n+ sys.exit(1)\r\n+ log.info(f\"Using default output directory from config.py: {output_dir_str}\")\r\n+ except Exception as e:\r\n+ log.error(f\"Could not read OUTPUT_BASE_DIR from config.py: {e}\")\r\n+ sys.exit(1)\r\n+\r\n+ # --- Resolve Output Path (Handles Relative Paths Explicitly) ---\r\n+ output_path_obj: Path\r\n+ if os.path.isabs(output_dir_str):\r\n+ output_path_obj = Path(output_dir_str)\r\n+ log.info(f\"Using absolute output directory: {output_path_obj}\")\r\n+ else:\r\n+ # Path() interprets relative paths against CWD by default\r\n+ output_path_obj = Path(output_dir_str)\r\n+ log.info(f\"Using relative output directory '{output_dir_str}'. Resolved against CWD to: {output_path_obj.resolve()}\")\r\n+\r\n+ # --- Validate and Setup Output Directory ---\r\n+ try:\r\n+ # Resolve to ensure we have an absolute path for consistency and creation\r\n+ resolved_output_dir = output_path_obj.resolve()\r\n+ log.info(f\"Ensuring output directory exists: {resolved_output_dir}\")\r\n+ resolved_output_dir.mkdir(parents=True, exist_ok=True)\r\n+ # Use the resolved absolute path string for the processor\r\n+ output_dir_for_processor = str(resolved_output_dir)\r\n+ except Exception as e:\r\n+ log.error(f\"Cannot create or access output directory '{resolved_output_dir}': {e}\", exc_info=True)\r\n+ sys.exit(1)\r\n+\r\n+ # --- Check Preset Existence (Basic Check) ---\r\n+ preset_dir = Path(__file__).parent / \"presets\"\r\n+ preset_file = preset_dir / f\"{args.preset}.json\"\r\n+ if not preset_file.is_file():\r\n+ log.error(f\"Preset file not found: {preset_file}\")\r\n+ log.error(f\"Ensure a file named '{args.preset}.json' exists in the directory: {preset_dir.resolve()}\")\r\n+ sys.exit(1)\r\n+\r\n+ # --- Execute Processing via the new function ---\r\n+ processing_results = run_processing(\r\n+ valid_inputs=valid_inputs,\r\n+ preset_name=args.preset,\r\n+ output_dir_for_processor=output_dir_for_processor,\r\n+ overwrite=args.overwrite,\r\n+ num_workers=args.workers,\r\n+ verbose=args.verbose # Pass the verbose flag\r\n+ )\r\n+\r\n+ # --- Report Summary ---\r\n+ duration = time.time() - start_time\r\n+ successful_processed_count = processing_results[\"processed\"]\r\n+ skipped_count = processing_results[\"skipped\"]\r\n+ failed_count = processing_results[\"failed\"]\r\n+ results_list = processing_results[\"results_list\"]\r\n+\r\n+ log.info(\"=\" * 40)\r\n+ log.info(\"Processing Summary\")\r\n+ log.info(f\" Duration: {duration:.2f} seconds\")\r\n+ log.info(f\" Assets Attempted: {len(valid_inputs)}\")\r\n+ log.info(f\" Successfully Processed: {successful_processed_count}\")\r\n+ log.info(f\" Skipped (Already Existed): {skipped_count}\")\r\n+ log.info(f\" Failed: {failed_count}\")\r\n+\r\n+ if processing_results.get(\"pool_error\"):\r\n+ log.error(f\" Process Pool Error: {processing_results['pool_error']}\")\r\n+ # Ensure failed count reflects pool error if it happened\r\n+ if failed_count == 0 and successful_processed_count == 0 and skipped_count == 0:\r\n+ failed_count = len(valid_inputs) # Assume all failed if pool died early\r\n+\r\n+ exit_code = 0\r\n+ if failed_count > 0:\r\n+ log.warning(\"Failures occurred:\")\r\n+ # Iterate through results to show specific errors for failed items\r\n+ for input_path, status, err_msg in results_list:\r\n+ if status == \"failed\":\r\n+ log.warning(f\" - {Path(input_path).name}: {err_msg}\")\r\n+ exit_code = 1 # Exit with error code if failures occurred\r\n+ else:\r\n+ # Consider skipped assets as a form of success for the overall run exit code\r\n+ if successful_processed_count > 0 or skipped_count > 0:\r\n+ log.info(\"All assets processed or skipped successfully.\")\r\n+ exit_code = 0 # Exit code 0 indicates success (including skips)\r\n+ else:\r\n+ # This case might happen if all inputs were invalid initially\r\n+ log.warning(\"No assets were processed, skipped, or failed (check input validation logs).\")\r\n+ exit_code = 0 # Still exit 0 as the script itself didn't crash\r\n+\r\n+ sys.exit(exit_code)\r\n+\r\n+\r\n+if __name__ == \"__main__\":\r\n+ # This ensures the main() function runs only when the script is executed directly\r\n+ # Important for multiprocessing to work correctly on some platforms (like Windows)\r\n+ main()\n\\ No newline at end of file\n"
|
|
},
|
|
{
|
|
"date": 1745261929518,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n@@ -0,0 +1,420 @@\n+# main.py\r\n+\r\n+import argparse\r\n+import sys\r\n+import time\r\n+import os\r\n+import logging\r\n+from pathlib import Path\r\n+from concurrent.futures import ProcessPoolExecutor, as_completed\r\n+import platform # To potentially adjust worker count defaults\r\n+from typing import List, Dict, Tuple, Optional # Added for type hinting\r\n+\r\n+# --- Assuming classes are in sibling files ---\r\n+try:\r\n+ from configuration import Configuration, ConfigurationError\r\n+ from asset_processor import AssetProcessor, AssetProcessingError\r\n+ import config as core_config_module # <<< IMPORT config.py HERE\r\n+except ImportError as e:\r\n+ # Provide a more helpful error message if imports fail\r\n+ script_dir = Path(__file__).parent.resolve()\r\n+ print(f\"ERROR: Failed to import necessary classes: {e}\")\r\n+ print(f\"Ensure 'configuration.py' and 'asset_processor.py' exist in the directory:\")\r\n+ print(f\" {script_dir}\")\r\n+ print(\"Or that the directory is included in your PYTHONPATH.\")\r\n+ sys.exit(1)\r\n+\r\n+# --- Setup Logging ---\r\n+# Keep setup_logging as is, it's called by main() or potentially monitor.py\r\n+def setup_logging(verbose: bool):\r\n+ \"\"\"Configures logging for the application.\"\"\"\r\n+ log_level = logging.DEBUG if verbose else logging.INFO\r\n+ log_format = '%(asctime)s [%(levelname)-8s] %(name)s: %(message)s'\r\n+ date_format = '%Y-%m-%d %H:%M:%S'\r\n+\r\n+ # Configure root logger\r\n+ # Remove existing handlers to avoid duplication if re-run in same session\r\n+ for handler in logging.root.handlers[:]:\r\n+ logging.root.removeHandler(handler)\r\n+\r\n+ logging.basicConfig(\r\n+ level=log_level,\r\n+ format=log_format,\r\n+ datefmt=date_format,\r\n+ handlers=[\r\n+ logging.StreamHandler(sys.stdout) # Log to console\r\n+ # Optional: Add FileHandler for persistent logs\r\n+ # logging.FileHandler(\"asset_processor.log\", mode='a', encoding='utf-8')\r\n+ ]\r\n+ )\r\n+ # Get logger specifically for this main script\r\n+ log = logging.getLogger(__name__) # or use 'main'\r\n+ log.info(f\"Logging level set to: {logging.getLevelName(log_level)}\")\r\n+ # Suppress overly verbose messages from libraries if needed (e.g., cv2)\r\n+ # logging.getLogger('cv2').setLevel(logging.WARNING)\r\n+\r\n+# Use module-level logger after configuration\r\n+log = logging.getLogger(__name__)\r\n+\r\n+\r\n+# --- Argument Parser Setup ---\r\n+# Keep setup_arg_parser as is, it's only used when running main.py directly\r\n+def setup_arg_parser():\r\n+ \"\"\"Sets up and returns the command-line argument parser.\"\"\"\r\n+ # Determine a sensible default worker count\r\n+ default_workers = 1\r\n+ try:\r\n+ # Use half the cores, but at least 1, max maybe 8-16? Depends on task nature.\r\n+ # Let's try max(1, os.cpu_count() // 2)\r\n+ cores = os.cpu_count()\r\n+ if cores:\r\n+ default_workers = max(1, cores // 2)\r\n+ # Cap default workers? Maybe not necessary, let user decide via flag.\r\n+ # default_workers = min(default_workers, 8) # Example cap\r\n+ except NotImplementedError:\r\n+ log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n+\r\n+ parser = argparse.ArgumentParser(\r\n+ description=\"Process asset files (ZIPs or folders) into a standardized library format using presets.\",\r\n+ formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message\r\n+ )\r\n+ parser.add_argument(\r\n+ \"input_paths\",\r\n+ metavar=\"INPUT_PATH\",\r\n+ type=str,\r\n+ nargs='+', # Requires one or more input paths\r\n+ help=\"Path(s) to the input ZIP file(s) or folder(s) containing assets.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-p\", \"--preset\",\r\n+ type=str,\r\n+ required=True,\r\n+ help=\"Name of the configuration preset (e.g., 'poliigon') located in the 'presets' directory (without .json extension).\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-o\", \"--output-dir\",\r\n+ type=str,\r\n+ required=False, # No longer required\r\n+ default=None, # Default is None, will check core_config later\r\n+ help=\"Override the default base output directory defined in config.py.\" # Updated help\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-w\", \"--workers\",\r\n+ type=int,\r\n+ default=default_workers,\r\n+ help=\"Maximum number of assets to process concurrently in parallel processes.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"-v\", \"--verbose\",\r\n+ action=\"store_true\", # Makes it a flag, value is True if present\r\n+ help=\"Enable detailed DEBUG level logging for troubleshooting.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"--overwrite\",\r\n+ action=\"store_true\",\r\n+ help=\"Force reprocessing and overwrite existing output asset folders if they exist.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"--nodegroup-blend\",\r\n+ type=str,\r\n+ default=None,\r\n+ help=\"Path to the .blend file for creating/updating node groups. Overrides config.py default.\"\r\n+ )\r\n+ parser.add_argument(\r\n+ \"--materials-blend\",\r\n+ type=str,\r\n+ default=None,\r\n+ help=\"Path to the .blend file for creating/updating materials. Overrides config.py default.\"\r\n+ )\r\n+ # Potential future flags:\r\n+ # parser.add_argument(\"--log-file\", type=str, default=None, help=\"Path to save log output to a file.\")\r\n+ return parser\r\n+\r\n+\r\n+# --- Worker Function ---\r\n+def process_single_asset_wrapper(input_path_str: str, preset_name: str, output_dir_str: str, overwrite: bool, verbose: bool) -> Tuple[str, str, Optional[str]]:\r\n+ \"\"\"\r\n+ Wrapper function for processing a single asset in a separate process.\r\n+ Handles instantiation of Configuration and AssetProcessor, passes the overwrite flag, and catches errors.\r\n+ Ensures logging is configured for the worker process.\r\n+ Returns: (input_path_str, status_string [\"processed\", \"skipped\", \"failed\"], error_message_or_None)\r\n+ \"\"\"\r\n+ # Explicitly configure logging for this worker process\r\n+ worker_log = logging.getLogger(f\"Worker_{os.getpid()}\") # Log with worker PID\r\n+ # Check if root logger already has handlers (might happen in some environments)\r\n+ if not logging.root.handlers:\r\n+ # Basic config if no handlers are set up (should be handled by main, but safety)\r\n+ logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)-8s] %(name)s: %(message)s')\r\n+ # Set the level for this worker's logger based on the verbose flag\r\n+ worker_log.setLevel(logging.DEBUG if verbose else logging.INFO)\r\n+ # Also ensure the root logger level is set if verbose\r\n+ if verbose:\r\n+ logging.root.setLevel(logging.DEBUG)\r\n+\r\n+\r\n+ try:\r\n+ worker_log.info(f\"Starting processing attempt for: {Path(input_path_str).name}\")\r\n+ # Each worker needs its own Configuration instance based on the preset name\r\n+ config = Configuration(preset_name)\r\n+ # Ensure output_dir_str is treated as the absolute base path\r\n+ output_base_path = Path(output_dir_str)\r\n+ input_path = Path(input_path_str)\r\n+\r\n+ # Pass the overwrite flag to the AssetProcessor\r\n+ processor = AssetProcessor(input_path, config, output_base_path, overwrite=overwrite)\r\n+ status = processor.process() # <<< Execute the main processing pipeline and get status\r\n+\r\n+ # Log based on status\r\n+ if status == \"skipped\":\r\n+ worker_log.info(f\"Worker skipped: {input_path.name}\")\r\n+ elif status == \"processed\":\r\n+ worker_log.info(f\"Worker finished processing: {input_path.name}\")\r\n+ # Note: Failures within process() should raise exceptions caught below\r\n+\r\n+ return (input_path_str, status, None) # Return the status string\r\n+\r\n+ except (ConfigurationError, AssetProcessingError) as e:\r\n+ worker_log.error(f\"Failed: {Path(input_path_str).name} - {type(e).__name__}: {e}\")\r\n+ return (input_path_str, \"failed\", f\"{type(e).__name__}: {e}\") # Return \"failed\" status\r\n+ except Exception as e:\r\n+ # Catch any other unexpected errors originating from the worker process\r\n+ # Use exc_info=True to log the full traceback from the worker\r\n+ worker_log.exception(f\"Unexpected failure processing {Path(input_path_str).name}: {e}\")\r\n+ return (input_path_str, \"failed\", f\"Unexpected Worker Error: {e}\") # Return \"failed\" status\r\n+\r\n+\r\n+# --- Core Processing Function ---\r\n+def run_processing(\r\n+ valid_inputs: List[str],\r\n+ preset_name: str,\r\n+ output_dir_for_processor: str,\r\n+ overwrite: bool,\r\n+ num_workers: int,\r\n+ verbose: bool # Add verbose parameter here\r\n+) -> Dict:\r\n+ \"\"\"\r\n+ Executes the core asset processing logic using a process pool.\r\n+\r\n+ Args:\r\n+ valid_inputs: List of validated input file/directory paths (strings).\r\n+ preset_name: Name of the preset to use.\r\n+ output_dir_for_processor: Absolute path string for the output base directory.\r\n+ overwrite: Boolean flag to force reprocessing.\r\n+ num_workers: Maximum number of worker processes.\r\n+ verbose: Boolean flag for verbose logging.\r\n+\r\n+ Returns:\r\n+ A dictionary containing processing results:\r\n+ {\r\n+ \"processed\": int,\r\n+ \"skipped\": int,\r\n+ \"failed\": int,\r\n+ \"results_list\": List[Tuple[str, str, Optional[str]]] # (input_path, status, error_msg)\r\n+ }\r\n+ \"\"\"\r\n+ log.info(f\"Processing {len(valid_inputs)} asset(s) using preset '{preset_name}' with up to {num_workers} worker(s)...\")\r\n+ results_list = []\r\n+ successful_processed_count = 0\r\n+ skipped_count = 0\r\n+ failed_count = 0\r\n+\r\n+ # Ensure at least one worker\r\n+ num_workers = max(1, num_workers)\r\n+\r\n+ # Using ProcessPoolExecutor is generally good if AssetProcessor tasks are CPU-bound.\r\n+ # If tasks are mostly I/O bound, ThreadPoolExecutor might be sufficient.\r\n+ # Important: Ensure Configuration and AssetProcessor are \"pickleable\".\r\n+ try:\r\n+ with ProcessPoolExecutor(max_workers=num_workers) as executor:\r\n+ # Create futures\r\n+ futures = {}\r\n+ log.debug(f\"Submitting {len(valid_inputs)} tasks...\")\r\n+ # Removed the 1-second delay for potentially faster submission in non-CLI use\r\n+ for i, input_path in enumerate(valid_inputs):\r\n+ log.debug(f\"Submitting task {i+1}/{len(valid_inputs)} for: {Path(input_path).name}\")\r\n+ future = executor.submit(\r\n+ process_single_asset_wrapper,\r\n+ input_path,\r\n+ preset_name,\r\n+ output_dir_for_processor,\r\n+ overwrite,\r\n+ verbose # Pass the verbose flag\r\n+ )\r\n+ futures[future] = input_path # Store future -> input_path mapping\r\n+\r\n+ # Process completed futures\r\n+ for i, future in enumerate(as_completed(futures), 1):\r\n+ input_path = futures[future]\r\n+ asset_name = Path(input_path).name\r\n+ log.info(f\"--- [{i}/{len(valid_inputs)}] Worker finished attempt for: {asset_name} ---\")\r\n+ try:\r\n+ # Get result tuple: (input_path_str, status_string, error_message_or_None)\r\n+ result_tuple = future.result()\r\n+ results_list.append(result_tuple)\r\n+ input_path_res, status, err_msg = result_tuple\r\n+\r\n+ # Increment counters based on status\r\n+ if status == \"processed\":\r\n+ successful_processed_count += 1\r\n+ elif status == \"skipped\":\r\n+ skipped_count += 1\r\n+ elif status == \"failed\":\r\n+ failed_count += 1\r\n+ else: # Should not happen, but log as warning/failure\r\n+ log.warning(f\"Unknown status '{status}' received for {asset_name}. Counting as failed.\")\r\n+ failed_count += 1\r\n+\r\n+ except Exception as e:\r\n+ # Catch errors if the future itself fails (e.g., worker process crashed hard)\r\n+ log.exception(f\"Critical worker failure for {asset_name}: {e}\")\r\n+ results_list.append((input_path, \"failed\", f\"Worker process crashed: {e}\"))\r\n+ failed_count += 1 # Count crashes as failures\r\n+\r\n+ except Exception as pool_exc:\r\n+ log.exception(f\"An error occurred with the process pool: {pool_exc}\")\r\n+ # Re-raise or handle as appropriate for the calling context (monitor.py)\r\n+ # For now, log and return current counts\r\n+ return {\r\n+ \"processed\": successful_processed_count,\r\n+ \"skipped\": skipped_count,\r\n+ \"failed\": failed_count + (len(valid_inputs) - len(results_list)), # Count unprocessed as failed\r\n+ \"results_list\": results_list,\r\n+ \"pool_error\": str(pool_exc) # Add pool error info\r\n+ }\r\n+\r\n+ return {\r\n+ \"processed\": successful_processed_count,\r\n+ \"skipped\": skipped_count,\r\n+ \"failed\": failed_count,\r\n+ \"results_list\": results_list\r\n+ }\r\n+\r\n+\r\n+# --- Main Execution (for CLI usage) ---\r\n+def main():\r\n+ \"\"\"Parses arguments, sets up logging, runs processing, and reports summary.\"\"\"\r\n+ parser = setup_arg_parser()\r\n+ args = parser.parse_args()\r\n+\r\n+ # Setup logging based on verbosity argument *before* logging status messages\r\n+ setup_logging(args.verbose)\r\n+\r\n+ start_time = time.time()\r\n+ log.info(\"Asset Processor Script Started (CLI Mode)\")\r\n+\r\n+ # --- Validate Input Paths ---\r\n+ valid_inputs = []\r\n+ for p_str in args.input_paths:\r\n+ p = Path(p_str)\r\n+ if p.exists():\r\n+ if p.is_dir() or (p.is_file() and p.suffix.lower() == '.zip'):\r\n+ valid_inputs.append(p_str) # Store the original string path\r\n+ else:\r\n+ log.warning(f\"Input is not a directory or .zip, skipping: {p_str}\")\r\n+ else:\r\n+ log.warning(f\"Input path not found, skipping: {p_str}\")\r\n+\r\n+ if not valid_inputs:\r\n+ log.error(\"No valid input paths found. Exiting.\")\r\n+ sys.exit(1) # Exit with error code\r\n+\r\n+ # --- Determine Output Directory ---\r\n+ output_dir_str = args.output_dir # Get value from args (might be None)\r\n+ if not output_dir_str:\r\n+ log.debug(\"Output directory not specified via -o, reading default from config.py.\")\r\n+ try:\r\n+ output_dir_str = getattr(core_config_module, 'OUTPUT_BASE_DIR', None)\r\n+ if not output_dir_str:\r\n+ log.error(\"Output directory not specified with -o and OUTPUT_BASE_DIR not found or empty in config.py. Exiting.\")\r\n+ sys.exit(1)\r\n+ log.info(f\"Using default output directory from config.py: {output_dir_str}\")\r\n+ except Exception as e:\r\n+ log.error(f\"Could not read OUTPUT_BASE_DIR from config.py: {e}\")\r\n+ sys.exit(1)\r\n+\r\n+ # --- Resolve Output Path (Handles Relative Paths Explicitly) ---\r\n+ output_path_obj: Path\r\n+ if os.path.isabs(output_dir_str):\r\n+ output_path_obj = Path(output_dir_str)\r\n+ log.info(f\"Using absolute output directory: {output_path_obj}\")\r\n+ else:\r\n+ # Path() interprets relative paths against CWD by default\r\n+ output_path_obj = Path(output_dir_str)\r\n+ log.info(f\"Using relative output directory '{output_dir_str}'. Resolved against CWD to: {output_path_obj.resolve()}\")\r\n+\r\n+ # --- Validate and Setup Output Directory ---\r\n+ try:\r\n+ # Resolve to ensure we have an absolute path for consistency and creation\r\n+ resolved_output_dir = output_path_obj.resolve()\r\n+ log.info(f\"Ensuring output directory exists: {resolved_output_dir}\")\r\n+ resolved_output_dir.mkdir(parents=True, exist_ok=True)\r\n+ # Use the resolved absolute path string for the processor\r\n+ output_dir_for_processor = str(resolved_output_dir)\r\n+ except Exception as e:\r\n+ log.error(f\"Cannot create or access output directory '{resolved_output_dir}': {e}\", exc_info=True)\r\n+ sys.exit(1)\r\n+\r\n+ # --- Check Preset Existence (Basic Check) ---\r\n+ preset_dir = Path(__file__).parent / \"presets\"\r\n+ preset_file = preset_dir / f\"{args.preset}.json\"\r\n+ if not preset_file.is_file():\r\n+ log.error(f\"Preset file not found: {preset_file}\")\r\n+ log.error(f\"Ensure a file named '{args.preset}.json' exists in the directory: {preset_dir.resolve()}\")\r\n+ sys.exit(1)\r\n+\r\n+ # --- Execute Processing via the new function ---\r\n+ processing_results = run_processing(\r\n+ valid_inputs=valid_inputs,\r\n+ preset_name=args.preset,\r\n+ output_dir_for_processor=output_dir_for_processor,\r\n+ overwrite=args.overwrite,\r\n+ num_workers=args.workers,\r\n+ verbose=args.verbose # Pass the verbose flag\r\n+ )\r\n+\r\n+ # --- Report Summary ---\r\n+ duration = time.time() - start_time\r\n+ successful_processed_count = processing_results[\"processed\"]\r\n+ skipped_count = processing_results[\"skipped\"]\r\n+ failed_count = processing_results[\"failed\"]\r\n+ results_list = processing_results[\"results_list\"]\r\n+\r\n+ log.info(\"=\" * 40)\r\n+ log.info(\"Processing Summary\")\r\n+ log.info(f\" Duration: {duration:.2f} seconds\")\r\n+ log.info(f\" Assets Attempted: {len(valid_inputs)}\")\r\n+ log.info(f\" Successfully Processed: {successful_processed_count}\")\r\n+ log.info(f\" Skipped (Already Existed): {skipped_count}\")\r\n+ log.info(f\" Failed: {failed_count}\")\r\n+\r\n+ if processing_results.get(\"pool_error\"):\r\n+ log.error(f\" Process Pool Error: {processing_results['pool_error']}\")\r\n+ # Ensure failed count reflects pool error if it happened\r\n+ if failed_count == 0 and successful_processed_count == 0 and skipped_count == 0:\r\n+ failed_count = len(valid_inputs) # Assume all failed if pool died early\r\n+\r\n+ exit_code = 0\r\n+ if failed_count > 0:\r\n+ log.warning(\"Failures occurred:\")\r\n+ # Iterate through results to show specific errors for failed items\r\n+ for input_path, status, err_msg in results_list:\r\n+ if status == \"failed\":\r\n+ log.warning(f\" - {Path(input_path).name}: {err_msg}\")\r\n+ exit_code = 1 # Exit with error code if failures occurred\r\n+ else:\r\n+ # Consider skipped assets as a form of success for the overall run exit code\r\n+ if successful_processed_count > 0 or skipped_count > 0:\r\n+ log.info(\"All assets processed or skipped successfully.\")\r\n+ exit_code = 0 # Exit code 0 indicates success (including skips)\r\n+ else:\r\n+ # This case might happen if all inputs were invalid initially\r\n+ log.warning(\"No assets were processed, skipped, or failed (check input validation logs).\")\r\n+ exit_code = 0 # Still exit 0 as the script itself didn't crash\r\n+\r\n+ sys.exit(exit_code)\r\n+\r\n+\r\n+if __name__ == \"__main__\":\r\n+ # This ensures the main() function runs only when the script is executed directly\r\n+ # Important for multiprocessing to work correctly on some platforms (like Windows)\r\n+ main()\n\\ No newline at end of file\n"
|
|
},
|
|
{
|
|
"date": 1745262048320,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n@@ -7,8 +7,10 @@\n import logging\r\n from pathlib import Path\r\n from concurrent.futures import ProcessPoolExecutor, as_completed\r\n import platform # To potentially adjust worker count defaults\r\n+import subprocess # <<< ADDED IMPORT\r\n+import shutil # <<< ADDED IMPORT\r\n from typing import List, Dict, Tuple, Optional # Added for type hinting\r\n \r\n # --- Assuming classes are in sibling files ---\r\n try:\r\n@@ -289,414 +291,73 @@\n \"results_list\": results_list\r\n }\r\n \r\n \r\n-# --- Main Execution (for CLI usage) ---\r\n-def main():\r\n- \"\"\"Parses arguments, sets up logging, runs processing, and reports summary.\"\"\"\r\n- parser = setup_arg_parser()\r\n- args = parser.parse_args()\r\n-\r\n- # Setup logging based on verbosity argument *before* logging status messages\r\n- setup_logging(args.verbose)\r\n-\r\n- start_time = time.time()\r\n- log.info(\"Asset Processor Script Started (CLI Mode)\")\r\n-\r\n- # --- Validate Input Paths ---\r\n- valid_inputs = []\r\n- for p_str in args.input_paths:\r\n- p = Path(p_str)\r\n- if p.exists():\r\n- if p.is_dir() or (p.is_file() and p.suffix.lower() == '.zip'):\r\n- valid_inputs.append(p_str) # Store the original string path\r\n- else:\r\n- log.warning(f\"Input is not a directory or .zip, skipping: {p_str}\")\r\n- else:\r\n- log.warning(f\"Input path not found, skipping: {p_str}\")\r\n-\r\n- if not valid_inputs:\r\n- log.error(\"No valid input paths found. Exiting.\")\r\n- sys.exit(1) # Exit with error code\r\n-\r\n- # --- Determine Output Directory ---\r\n- output_dir_str = args.output_dir # Get value from args (might be None)\r\n- if not output_dir_str:\r\n- log.debug(\"Output directory not specified via -o, reading default from config.py.\")\r\n- try:\r\n- output_dir_str = getattr(core_config_module, 'OUTPUT_BASE_DIR', None)\r\n- if not output_dir_str:\r\n- log.error(\"Output directory not specified with -o and OUTPUT_BASE_DIR not found or empty in config.py. Exiting.\")\r\n- sys.exit(1)\r\n- log.info(f\"Using default output directory from config.py: {output_dir_str}\")\r\n- except Exception as e:\r\n- log.error(f\"Could not read OUTPUT_BASE_DIR from config.py: {e}\")\r\n- sys.exit(1)\r\n-\r\n- # --- Resolve Output Path (Handles Relative Paths Explicitly) ---\r\n- output_path_obj: Path\r\n- if os.path.isabs(output_dir_str):\r\n- output_path_obj = Path(output_dir_str)\r\n- log.info(f\"Using absolute output directory: {output_path_obj}\")\r\n- else:\r\n- # Path() interprets relative paths against CWD by default\r\n- output_path_obj = Path(output_dir_str)\r\n- log.info(f\"Using relative output directory '{output_dir_str}'. Resolved against CWD to: {output_path_obj.resolve()}\")\r\n-\r\n- # --- Validate and Setup Output Directory ---\r\n- try:\r\n- # Resolve to ensure we have an absolute path for consistency and creation\r\n- resolved_output_dir = output_path_obj.resolve()\r\n- log.info(f\"Ensuring output directory exists: {resolved_output_dir}\")\r\n- resolved_output_dir.mkdir(parents=True, exist_ok=True)\r\n- # Use the resolved absolute path string for the processor\r\n- output_dir_for_processor = str(resolved_output_dir)\r\n- except Exception as e:\r\n- log.error(f\"Cannot create or access output directory '{resolved_output_dir}': {e}\", exc_info=True)\r\n- sys.exit(1)\r\n-\r\n- # --- Check Preset Existence (Basic Check) ---\r\n- preset_dir = Path(__file__).parent / \"presets\"\r\n- preset_file = preset_dir / f\"{args.preset}.json\"\r\n- if not preset_file.is_file():\r\n- log.error(f\"Preset file not found: {preset_file}\")\r\n- log.error(f\"Ensure a file named '{args.preset}.json' exists in the directory: {preset_dir.resolve()}\")\r\n- sys.exit(1)\r\n-\r\n- # --- Execute Processing via the new function ---\r\n- processing_results = run_processing(\r\n- valid_inputs=valid_inputs,\r\n- preset_name=args.preset,\r\n- output_dir_for_processor=output_dir_for_processor,\r\n- overwrite=args.overwrite,\r\n- num_workers=args.workers,\r\n- verbose=args.verbose # Pass the verbose flag\r\n- )\r\n-\r\n- # --- Report Summary ---\r\n- duration = time.time() - start_time\r\n- successful_processed_count = processing_results[\"processed\"]\r\n- skipped_count = processing_results[\"skipped\"]\r\n- failed_count = processing_results[\"failed\"]\r\n- results_list = processing_results[\"results_list\"]\r\n-\r\n- log.info(\"=\" * 40)\r\n- log.info(\"Processing Summary\")\r\n- log.info(f\" Duration: {duration:.2f} seconds\")\r\n- log.info(f\" Assets Attempted: {len(valid_inputs)}\")\r\n- log.info(f\" Successfully Processed: {successful_processed_count}\")\r\n- log.info(f\" Skipped (Already Existed): {skipped_count}\")\r\n- log.info(f\" Failed: {failed_count}\")\r\n-\r\n- if processing_results.get(\"pool_error\"):\r\n- log.error(f\" Process Pool Error: {processing_results['pool_error']}\")\r\n- # Ensure failed count reflects pool error if it happened\r\n- if failed_count == 0 and successful_processed_count == 0 and skipped_count == 0:\r\n- failed_count = len(valid_inputs) # Assume all failed if pool died early\r\n-\r\n- exit_code = 0\r\n- if failed_count > 0:\r\n- log.warning(\"Failures occurred:\")\r\n- # Iterate through results to show specific errors for failed items\r\n- for input_path, status, err_msg in results_list:\r\n- if status == \"failed\":\r\n- log.warning(f\" - {Path(input_path).name}: {err_msg}\")\r\n- exit_code = 1 # Exit with error code if failures occurred\r\n- else:\r\n- # Consider skipped assets as a form of success for the overall run exit code\r\n- if successful_processed_count > 0 or skipped_count > 0:\r\n- log.info(\"All assets processed or skipped successfully.\")\r\n- exit_code = 0 # Exit code 0 indicates success (including skips)\r\n- else:\r\n- # This case might happen if all inputs were invalid initially\r\n- log.warning(\"No assets were processed, skipped, or failed (check input validation logs).\")\r\n- exit_code = 0 # Still exit 0 as the script itself didn't crash\r\n-\r\n- sys.exit(exit_code)\r\n-\r\n-\r\n-if __name__ == \"__main__\":\r\n- # This ensures the main() function runs only when the script is executed directly\r\n- # Important for multiprocessing to work correctly on some platforms (like Windows)\r\n- main()\n-# main.py\r\n-\r\n-import argparse\r\n-import sys\r\n-import time\r\n-import os\r\n-import logging\r\n-from pathlib import Path\r\n-from concurrent.futures import ProcessPoolExecutor, as_completed\r\n-import platform # To potentially adjust worker count defaults\r\n-from typing import List, Dict, Tuple, Optional # Added for type hinting\r\n-\r\n-# --- Assuming classes are in sibling files ---\r\n-try:\r\n- from configuration import Configuration, ConfigurationError\r\n- from asset_processor import AssetProcessor, AssetProcessingError\r\n- import config as core_config_module # <<< IMPORT config.py HERE\r\n-except ImportError as e:\r\n- # Provide a more helpful error message if imports fail\r\n- script_dir = Path(__file__).parent.resolve()\r\n- print(f\"ERROR: Failed to import necessary classes: {e}\")\r\n- print(f\"Ensure 'configuration.py' and 'asset_processor.py' exist in the directory:\")\r\n- print(f\" {script_dir}\")\r\n- print(\"Or that the directory is included in your PYTHONPATH.\")\r\n- sys.exit(1)\r\n-\r\n-# --- Setup Logging ---\r\n-# Keep setup_logging as is, it's called by main() or potentially monitor.py\r\n-def setup_logging(verbose: bool):\r\n- \"\"\"Configures logging for the application.\"\"\"\r\n- log_level = logging.DEBUG if verbose else logging.INFO\r\n- log_format = '%(asctime)s [%(levelname)-8s] %(name)s: %(message)s'\r\n- date_format = '%Y-%m-%d %H:%M:%S'\r\n-\r\n- # Configure root logger\r\n- # Remove existing handlers to avoid duplication if re-run in same session\r\n- for handler in logging.root.handlers[:]:\r\n- logging.root.removeHandler(handler)\r\n-\r\n- logging.basicConfig(\r\n- level=log_level,\r\n- format=log_format,\r\n- datefmt=date_format,\r\n- handlers=[\r\n- logging.StreamHandler(sys.stdout) # Log to console\r\n- # Optional: Add FileHandler for persistent logs\r\n- # logging.FileHandler(\"asset_processor.log\", mode='a', encoding='utf-8')\r\n- ]\r\n- )\r\n- # Get logger specifically for this main script\r\n- log = logging.getLogger(__name__) # or use 'main'\r\n- log.info(f\"Logging level set to: {logging.getLevelName(log_level)}\")\r\n- # Suppress overly verbose messages from libraries if needed (e.g., cv2)\r\n- # logging.getLogger('cv2').setLevel(logging.WARNING)\r\n-\r\n-# Use module-level logger after configuration\r\n-log = logging.getLogger(__name__)\r\n-\r\n-\r\n-# --- Argument Parser Setup ---\r\n-# Keep setup_arg_parser as is, it's only used when running main.py directly\r\n-def setup_arg_parser():\r\n- \"\"\"Sets up and returns the command-line argument parser.\"\"\"\r\n- # Determine a sensible default worker count\r\n- default_workers = 1\r\n- try:\r\n- # Use half the cores, but at least 1, max maybe 8-16? Depends on task nature.\r\n- # Let's try max(1, os.cpu_count() // 2)\r\n- cores = os.cpu_count()\r\n- if cores:\r\n- default_workers = max(1, cores // 2)\r\n- # Cap default workers? Maybe not necessary, let user decide via flag.\r\n- # default_workers = min(default_workers, 8) # Example cap\r\n- except NotImplementedError:\r\n- log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n-\r\n- parser = argparse.ArgumentParser(\r\n- description=\"Process asset files (ZIPs or folders) into a standardized library format using presets.\",\r\n- formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message\r\n- )\r\n- parser.add_argument(\r\n- \"input_paths\",\r\n- metavar=\"INPUT_PATH\",\r\n- type=str,\r\n- nargs='+', # Requires one or more input paths\r\n- help=\"Path(s) to the input ZIP file(s) or folder(s) containing assets.\"\r\n- )\r\n- parser.add_argument(\r\n- \"-p\", \"--preset\",\r\n- type=str,\r\n- required=True,\r\n- help=\"Name of the configuration preset (e.g., 'poliigon') located in the 'presets' directory (without .json extension).\"\r\n- )\r\n- parser.add_argument(\r\n- \"-o\", \"--output-dir\",\r\n- type=str,\r\n- required=False, # No longer required\r\n- default=None, # Default is None, will check core_config later\r\n- help=\"Override the default base output directory defined in config.py.\" # Updated help\r\n- )\r\n- parser.add_argument(\r\n- \"-w\", \"--workers\",\r\n- type=int,\r\n- default=default_workers,\r\n- help=\"Maximum number of assets to process concurrently in parallel processes.\"\r\n- )\r\n- parser.add_argument(\r\n- \"-v\", \"--verbose\",\r\n- action=\"store_true\", # Makes it a flag, value is True if present\r\n- help=\"Enable detailed DEBUG level logging for troubleshooting.\"\r\n- )\r\n- parser.add_argument(\r\n- \"--overwrite\",\r\n- action=\"store_true\",\r\n- help=\"Force reprocessing and overwrite existing output asset folders if they exist.\"\r\n- )\r\n- # Potential future flags:\r\n- # parser.add_argument(\"--log-file\", type=str, default=None, help=\"Path to save log output to a file.\")\r\n- return parser\r\n-\r\n-\r\n-# --- Worker Function ---\r\n-def process_single_asset_wrapper(input_path_str: str, preset_name: str, output_dir_str: str, overwrite: bool, verbose: bool) -> Tuple[str, str, Optional[str]]:\r\n+# --- Blender Script Execution Helper ---\r\n+def run_blender_script(blender_exe_path: str, blend_file_path: str, python_script_path: str, asset_root_dir: str):\r\n \"\"\"\r\n- Wrapper function for processing a single asset in a separate process.\r\n- Handles instantiation of Configuration and AssetProcessor, passes the overwrite flag, and catches errors.\r\n- Ensures logging is configured for the worker process.\r\n- Returns: (input_path_str, status_string [\"processed\", \"skipped\", \"failed\"], error_message_or_None)\r\n- \"\"\"\r\n- # Explicitly configure logging for this worker process\r\n- worker_log = logging.getLogger(f\"Worker_{os.getpid()}\") # Log with worker PID\r\n- # Check if root logger already has handlers (might happen in some environments)\r\n- if not logging.root.handlers:\r\n- # Basic config if no handlers are set up (should be handled by main, but safety)\r\n- logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)-8s] %(name)s: %(message)s')\r\n- # Set the level for this worker's logger based on the verbose flag\r\n- worker_log.setLevel(logging.DEBUG if verbose else logging.INFO)\r\n- # Also ensure the root logger level is set if verbose\r\n- if verbose:\r\n- logging.root.setLevel(logging.DEBUG)\r\n+ Executes a Python script within Blender in the background.\r\n \r\n-\r\n- try:\r\n- worker_log.info(f\"Starting processing attempt for: {Path(input_path_str).name}\")\r\n- # Each worker needs its own Configuration instance based on the preset name\r\n- config = Configuration(preset_name)\r\n- # Ensure output_dir_str is treated as the absolute base path\r\n- output_base_path = Path(output_dir_str)\r\n- input_path = Path(input_path_str)\r\n-\r\n- # Pass the overwrite flag to the AssetProcessor\r\n- processor = AssetProcessor(input_path, config, output_base_path, overwrite=overwrite)\r\n- status = processor.process() # <<< Execute the main processing pipeline and get status\r\n-\r\n- # Log based on status\r\n- if status == \"skipped\":\r\n- worker_log.info(f\"Worker skipped: {input_path.name}\")\r\n- elif status == \"processed\":\r\n- worker_log.info(f\"Worker finished processing: {input_path.name}\")\r\n- # Note: Failures within process() should raise exceptions caught below\r\n-\r\n- return (input_path_str, status, None) # Return the status string\r\n-\r\n- except (ConfigurationError, AssetProcessingError) as e:\r\n- worker_log.error(f\"Failed: {Path(input_path_str).name} - {type(e).__name__}: {e}\")\r\n- return (input_path_str, \"failed\", f\"{type(e).__name__}: {e}\") # Return \"failed\" status\r\n- except Exception as e:\r\n- # Catch any other unexpected errors originating from the worker process\r\n- # Use exc_info=True to log the full traceback from the worker\r\n- worker_log.exception(f\"Unexpected failure processing {Path(input_path_str).name}: {e}\")\r\n- return (input_path_str, \"failed\", f\"Unexpected Worker Error: {e}\") # Return \"failed\" status\r\n-\r\n-\r\n-# --- Core Processing Function ---\r\n-def run_processing(\r\n- valid_inputs: List[str],\r\n- preset_name: str,\r\n- output_dir_for_processor: str,\r\n- overwrite: bool,\r\n- num_workers: int,\r\n- verbose: bool # Add verbose parameter here\r\n-) -> Dict:\r\n- \"\"\"\r\n- Executes the core asset processing logic using a process pool.\r\n-\r\n Args:\r\n- valid_inputs: List of validated input file/directory paths (strings).\r\n- preset_name: Name of the preset to use.\r\n- output_dir_for_processor: Absolute path string for the output base directory.\r\n- overwrite: Boolean flag to force reprocessing.\r\n- num_workers: Maximum number of worker processes.\r\n- verbose: Boolean flag for verbose logging.\r\n+ blender_exe_path: Path to the Blender executable.\r\n+ blend_file_path: Path to the .blend file to open.\r\n+ python_script_path: Path to the Python script to execute within Blender.\r\n+ asset_root_dir: Path to the processed asset library root directory (passed to the script).\r\n \r\n Returns:\r\n- A dictionary containing processing results:\r\n- {\r\n- \"processed\": int,\r\n- \"skipped\": int,\r\n- \"failed\": int,\r\n- \"results_list\": List[Tuple[str, str, Optional[str]]] # (input_path, status, error_msg)\r\n- }\r\n+ True if the script executed successfully (return code 0), False otherwise.\r\n \"\"\"\r\n- log.info(f\"Processing {len(valid_inputs)} asset(s) using preset '{preset_name}' with up to {num_workers} worker(s)...\")\r\n- results_list = []\r\n- successful_processed_count = 0\r\n- skipped_count = 0\r\n- failed_count = 0\r\n+ log.info(f\"Attempting to run Blender script: {Path(python_script_path).name} on {Path(blend_file_path).name}\")\r\n \r\n- # Ensure at least one worker\r\n- num_workers = max(1, num_workers)\r\n+ # Ensure paths are absolute strings for subprocess\r\n+ blender_exe_path = str(Path(blender_exe_path).resolve())\r\n+ blend_file_path = str(Path(blend_file_path).resolve())\r\n+ python_script_path = str(Path(python_script_path).resolve())\r\n+ asset_root_dir = str(Path(asset_root_dir).resolve())\r\n \r\n- # Using ProcessPoolExecutor is generally good if AssetProcessor tasks are CPU-bound.\r\n- # If tasks are mostly I/O bound, ThreadPoolExecutor might be sufficient.\r\n- # Important: Ensure Configuration and AssetProcessor are \"pickleable\".\r\n- try:\r\n- with ProcessPoolExecutor(max_workers=num_workers) as executor:\r\n- # Create futures\r\n- futures = {}\r\n- log.debug(f\"Submitting {len(valid_inputs)} tasks...\")\r\n- # Removed the 1-second delay for potentially faster submission in non-CLI use\r\n- for i, input_path in enumerate(valid_inputs):\r\n- log.debug(f\"Submitting task {i+1}/{len(valid_inputs)} for: {Path(input_path).name}\")\r\n- future = executor.submit(\r\n- process_single_asset_wrapper,\r\n- input_path,\r\n- preset_name,\r\n- output_dir_for_processor,\r\n- overwrite,\r\n- verbose # Pass the verbose flag\r\n- )\r\n- futures[future] = input_path # Store future -> input_path mapping\r\n+ # Construct the command arguments\r\n+ # -b: Run in background (no UI)\r\n+ # -S: Save the file after running the script\r\n+ # --python: Execute the specified Python script\r\n+ # --: Separator, arguments after this are passed to the Python script's sys.argv\r\n+ command = [\r\n+ blender_exe_path,\r\n+ \"-b\", # Run in background\r\n+ blend_file_path,\r\n+ \"--python\", python_script_path,\r\n+ \"--\", # Pass subsequent arguments to the script\r\n+ asset_root_dir,\r\n+ \"-S\" # Save the blend file after script execution\r\n+ ]\r\n \r\n- # Process completed futures\r\n- for i, future in enumerate(as_completed(futures), 1):\r\n- input_path = futures[future]\r\n- asset_name = Path(input_path).name\r\n- log.info(f\"--- [{i}/{len(valid_inputs)}] Worker finished attempt for: {asset_name} ---\")\r\n- try:\r\n- # Get result tuple: (input_path_str, status_string, error_message_or_None)\r\n- result_tuple = future.result()\r\n- results_list.append(result_tuple)\r\n- input_path_res, status, err_msg = result_tuple\r\n+ log.debug(f\"Executing Blender command: {' '.join(command)}\") # Log the command for debugging\r\n \r\n- # Increment counters based on status\r\n- if status == \"processed\":\r\n- successful_processed_count += 1\r\n- elif status == \"skipped\":\r\n- skipped_count += 1\r\n- elif status == \"failed\":\r\n- failed_count += 1\r\n- else: # Should not happen, but log as warning/failure\r\n- log.warning(f\"Unknown status '{status}' received for {asset_name}. Counting as failed.\")\r\n- failed_count += 1\r\n+ try:\r\n+ # Execute the command\r\n+ # capture_output=True captures stdout and stderr\r\n+ # text=True decodes stdout/stderr as text\r\n+ # check=False prevents raising CalledProcessError on non-zero exit codes\r\n+ result = subprocess.run(command, capture_output=True, text=True, check=False)\r\n \r\n- except Exception as e:\r\n- # Catch errors if the future itself fails (e.g., worker process crashed hard)\r\n- log.exception(f\"Critical worker failure for {asset_name}: {e}\")\r\n- results_list.append((input_path, \"failed\", f\"Worker process crashed: {e}\"))\r\n- failed_count += 1 # Count crashes as failures\r\n+ # Log results\r\n+ log.info(f\"Blender script '{Path(python_script_path).name}' finished with exit code: {result.returncode}\")\r\n+ if result.stdout:\r\n+ log.debug(f\"Blender stdout:\\n{result.stdout.strip()}\")\r\n+ if result.stderr:\r\n+ # Log stderr as warning or error depending on return code\r\n+ if result.returncode != 0:\r\n+ log.error(f\"Blender stderr:\\n{result.stderr.strip()}\")\r\n+ else:\r\n+ log.warning(f\"Blender stderr (Return Code 0):\\n{result.stderr.strip()}\") # Log stderr even on success as scripts might print warnings\r\n \r\n- except Exception as pool_exc:\r\n- log.exception(f\"An error occurred with the process pool: {pool_exc}\")\r\n- # Re-raise or handle as appropriate for the calling context (monitor.py)\r\n- # For now, log and return current counts\r\n- return {\r\n- \"processed\": successful_processed_count,\r\n- \"skipped\": skipped_count,\r\n- \"failed\": failed_count + (len(valid_inputs) - len(results_list)), # Count unprocessed as failed\r\n- \"results_list\": results_list,\r\n- \"pool_error\": str(pool_exc) # Add pool error info\r\n- }\r\n+ return result.returncode == 0\r\n \r\n- return {\r\n- \"processed\": successful_processed_count,\r\n- \"skipped\": skipped_count,\r\n- \"failed\": failed_count,\r\n- \"results_list\": results_list\r\n- }\r\n+ except FileNotFoundError:\r\n+ log.error(f\"Blender executable not found at: {blender_exe_path}\")\r\n+ return False\r\n+ except Exception as e:\r\n+ log.exception(f\"An unexpected error occurred while running Blender script '{Path(python_script_path).name}': {e}\")\r\n+ return False\r\n \r\n \r\n # --- Main Execution (for CLI usage) ---\r\n def main():\r\n@@ -818,802 +479,134 @@\n # This case might happen if all inputs were invalid initially\r\n log.warning(\"No assets were processed, skipped, or failed (check input validation logs).\")\r\n exit_code = 0 # Still exit 0 as the script itself didn't crash\r\n \r\n- sys.exit(exit_code)\r\n+ # --- Blender Script Execution (Optional) ---\r\n+ run_nodegroups = False\r\n+ run_materials = False\r\n+ nodegroup_blend_path = None\r\n+ materials_blend_path = None\r\n+ blender_exe = None\r\n \r\n-\r\n-if __name__ == \"__main__\":\r\n- # This ensures the main() function runs only when the script is executed directly\r\n- # Important for multiprocessing to work correctly on some platforms (like Windows)\r\n- main()\n-# main.py\r\n-\r\n-import argparse\r\n-import sys\r\n-import time\r\n-import os\r\n-import logging\r\n-from pathlib import Path\r\n-from concurrent.futures import ProcessPoolExecutor, as_completed\r\n-import platform # To potentially adjust worker count defaults\r\n-from typing import List, Dict, Tuple, Optional # Added for type hinting\r\n-\r\n-# --- Assuming classes are in sibling files ---\r\n-try:\r\n- from configuration import Configuration, ConfigurationError\r\n- from asset_processor import AssetProcessor, AssetProcessingError\r\n- import config as core_config_module # <<< IMPORT config.py HERE\r\n-except ImportError as e:\r\n- # Provide a more helpful error message if imports fail\r\n- script_dir = Path(__file__).parent.resolve()\r\n- print(f\"ERROR: Failed to import necessary classes: {e}\")\r\n- print(f\"Ensure 'configuration.py' and 'asset_processor.py' exist in the directory:\")\r\n- print(f\" {script_dir}\")\r\n- print(\"Or that the directory is included in your PYTHONPATH.\")\r\n- sys.exit(1)\r\n-\r\n-# --- Setup Logging ---\r\n-# Keep setup_logging as is, it's called by main() or potentially monitor.py\r\n-def setup_logging(verbose: bool):\r\n- \"\"\"Configures logging for the application.\"\"\"\r\n- log_level = logging.DEBUG if verbose else logging.INFO\r\n- log_format = '%(asctime)s [%(levelname)-8s] %(name)s: %(message)s'\r\n- date_format = '%Y-%m-%d %H:%M:%S'\r\n-\r\n- # Configure root logger\r\n- # Remove existing handlers to avoid duplication if re-run in same session\r\n- for handler in logging.root.handlers[:]:\r\n- logging.root.removeHandler(handler)\r\n-\r\n- logging.basicConfig(\r\n- level=log_level,\r\n- format=log_format,\r\n- datefmt=date_format,\r\n- handlers=[\r\n- logging.StreamHandler(sys.stdout) # Log to console\r\n- # Optional: Add FileHandler for persistent logs\r\n- # logging.FileHandler(\"asset_processor.log\", mode='a', encoding='utf-8')\r\n- ]\r\n- )\r\n- # Get logger specifically for this main script\r\n- log = logging.getLogger(__name__) # or use 'main'\r\n- log.info(f\"Logging level set to: {logging.getLevelName(log_level)}\")\r\n- # Suppress overly verbose messages from libraries if needed (e.g., cv2)\r\n- # logging.getLogger('cv2').setLevel(logging.WARNING)\r\n-\r\n-# Use module-level logger after configuration\r\n-log = logging.getLogger(__name__)\r\n-\r\n-\r\n-# --- Argument Parser Setup ---\r\n-# Keep setup_arg_parser as is, it's only used when running main.py directly\r\n-def setup_arg_parser():\r\n- \"\"\"Sets up and returns the command-line argument parser.\"\"\"\r\n- # Determine a sensible default worker count\r\n- default_workers = 1\r\n+ # 1. Find Blender Executable\r\n try:\r\n- # Use half the cores, but at least 1, max maybe 8-16? Depends on task nature.\r\n- # Let's try max(1, os.cpu_count() // 2)\r\n- cores = os.cpu_count()\r\n- if cores:\r\n- default_workers = max(1, cores // 2)\r\n- # Cap default workers? Maybe not necessary, let user decide via flag.\r\n- # default_workers = min(default_workers, 8) # Example cap\r\n- except NotImplementedError:\r\n- log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n+ blender_exe_config = getattr(core_config_module, 'BLENDER_EXECUTABLE_PATH', None)\r\n+ if blender_exe_config:\r\n+ # Check if the path in config exists\r\n+ if Path(blender_exe_config).is_file():\r\n+ blender_exe = str(Path(blender_exe_config).resolve())\r\n+ log.info(f\"Using Blender executable from config: {blender_exe}\")\r\n+ else:\r\n+ # Try finding it in PATH if config path is invalid\r\n+ log.warning(f\"Blender path in config not found: '{blender_exe_config}'. Trying to find 'blender' in PATH.\")\r\n+ blender_exe = shutil.which(\"blender\")\r\n+ if blender_exe:\r\n+ log.info(f\"Found Blender executable in PATH: {blender_exe}\")\r\n+ else:\r\n+ log.warning(\"Could not find 'blender' in system PATH.\")\r\n+ else:\r\n+ # Try finding it in PATH if not set in config\r\n+ log.info(\"BLENDER_EXECUTABLE_PATH not set in config. Trying to find 'blender' in PATH.\")\r\n+ blender_exe = shutil.which(\"blender\")\r\n+ if blender_exe:\r\n+ log.info(f\"Found Blender executable in PATH: {blender_exe}\")\r\n+ else:\r\n+ log.warning(\"Could not find 'blender' in system PATH.\")\r\n \r\n- parser = argparse.ArgumentParser(\r\n- description=\"Process asset files (ZIPs or folders) into a standardized library format using presets.\",\r\n- formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message\r\n- )\r\n- parser.add_argument(\r\n- \"input_paths\",\r\n- metavar=\"INPUT_PATH\",\r\n- type=str,\r\n- nargs='+', # Requires one or more input paths\r\n- help=\"Path(s) to the input ZIP file(s) or folder(s) containing assets.\"\r\n- )\r\n- parser.add_argument(\r\n- \"-p\", \"--preset\",\r\n- type=str,\r\n- required=True,\r\n- help=\"Name of the configuration preset (e.g., 'poliigon') located in the 'presets' directory (without .json extension).\"\r\n- )\r\n- parser.add_argument(\r\n- \"-o\", \"--output-dir\",\r\n- type=str,\r\n- required=False, # No longer required\r\n- default=None, # Default is None, will check core_config later\r\n- help=\"Override the default base output directory defined in config.py.\" # Updated help\r\n- )\r\n- parser.add_argument(\r\n- \"-w\", \"--workers\",\r\n- type=int,\r\n- default=default_workers,\r\n- help=\"Maximum number of assets to process concurrently in parallel processes.\"\r\n- )\r\n- parser.add_argument(\r\n- \"-v\", \"--verbose\",\r\n- action=\"store_true\", # Makes it a flag, value is True if present\r\n- help=\"Enable detailed DEBUG level logging for troubleshooting.\"\r\n- )\r\n- parser.add_argument(\r\n- \"--overwrite\",\r\n- action=\"store_true\",\r\n- help=\"Force reprocessing and overwrite existing output asset folders if they exist.\"\r\n- )\r\n- # Potential future flags:\r\n- # parser.add_argument(\"--log-file\", type=str, default=None, help=\"Path to save log output to a file.\")\r\n- return parser\r\n+ if not blender_exe:\r\n+ log.warning(\"Blender executable not found or configured. Skipping Blender script execution.\")\r\n \r\n-\r\n-# --- Worker Function ---\r\n-# Keep process_single_asset_wrapper as is, it's called by the processing pool\r\n-def process_single_asset_wrapper(input_path_str: str, preset_name: str, output_dir_str: str, overwrite: bool) -> Tuple[str, str, Optional[str]]:\r\n- \"\"\"\r\n- Wrapper function for processing a single asset in a separate process.\r\n- Handles instantiation of Configuration and AssetProcessor, passes the overwrite flag, and catches errors.\r\n- Returns: (input_path_str, status_string [\"processed\", \"skipped\", \"failed\"], error_message_or_None)\r\n- \"\"\"\r\n- # Note: Logging setup might need re-initialization in child processes on some platforms\r\n- # if file handlers or complex configurations are used. Stdout usually works.\r\n- # Removed forced DEBUG logging for worker\r\n- worker_log = logging.getLogger(f\"Worker_{os.getpid()}\") # Log with worker PID\r\n- # Initial log message moved inside try block for better status reporting\r\n-\r\n- try:\r\n- worker_log.info(f\"Starting processing attempt for: {Path(input_path_str).name}\")\r\n- # Each worker needs its own Configuration instance based on the preset name\r\n- config = Configuration(preset_name)\r\n- # Ensure output_dir_str is treated as the absolute base path\r\n- output_base_path = Path(output_dir_str)\r\n- input_path = Path(input_path_str)\r\n-\r\n- # Pass the overwrite flag to the AssetProcessor\r\n- processor = AssetProcessor(input_path, config, output_base_path, overwrite=overwrite)\r\n- status = processor.process() # <<< Execute the main processing pipeline and get status\r\n-\r\n- # Log based on status\r\n- if status == \"skipped\":\r\n- worker_log.info(f\"Worker skipped: {input_path.name}\")\r\n- elif status == \"processed\":\r\n- worker_log.info(f\"Worker finished processing: {input_path.name}\")\r\n- # Note: Failures within process() should raise exceptions caught below\r\n-\r\n- return (input_path_str, status, None) # Return the status string\r\n-\r\n- except (ConfigurationError, AssetProcessingError) as e:\r\n- worker_log.error(f\"Failed: {Path(input_path_str).name} - {type(e).__name__}: {e}\")\r\n- return (input_path_str, \"failed\", f\"{type(e).__name__}: {e}\") # Return \"failed\" status\r\n except Exception as e:\r\n- # Catch any other unexpected errors originating from the worker process\r\n- # Use exc_info=True to log the full traceback from the worker\r\n- worker_log.exception(f\"Unexpected failure processing {Path(input_path_str).name}: {e}\")\r\n- return (input_path_str, \"failed\", f\"Unexpected Worker Error: {e}\") # Return \"failed\" status\r\n+ log.error(f\"Error checking Blender executable path: {e}\")\r\n+ blender_exe = None # Ensure it's None on error\r\n \r\n-\r\n-# --- Core Processing Function ---\r\n-def run_processing(\r\n- valid_inputs: List[str],\r\n- preset_name: str,\r\n- output_dir_for_processor: str,\r\n- overwrite: bool,\r\n- num_workers: int\r\n-) -> Dict:\r\n- \"\"\"\r\n- Executes the core asset processing logic using a process pool.\r\n-\r\n- Args:\r\n- valid_inputs: List of validated input file/directory paths (strings).\r\n- preset_name: Name of the preset to use.\r\n- output_dir_for_processor: Absolute path string for the output base directory.\r\n- overwrite: Boolean flag to force reprocessing.\r\n- num_workers: Maximum number of worker processes.\r\n-\r\n- Returns:\r\n- A dictionary containing processing results:\r\n- {\r\n- \"processed\": int,\r\n- \"skipped\": int,\r\n- \"failed\": int,\r\n- \"results_list\": List[Tuple[str, str, Optional[str]]] # (input_path, status, error_msg)\r\n- }\r\n- \"\"\"\r\n- log.info(f\"Processing {len(valid_inputs)} asset(s) using preset '{preset_name}' with up to {num_workers} worker(s)...\")\r\n- results_list = []\r\n- successful_processed_count = 0\r\n- skipped_count = 0\r\n- failed_count = 0\r\n-\r\n- # Ensure at least one worker\r\n- num_workers = max(1, num_workers)\r\n-\r\n- # Using ProcessPoolExecutor is generally good if AssetProcessor tasks are CPU-bound.\r\n- # If tasks are mostly I/O bound, ThreadPoolExecutor might be sufficient.\r\n- # Important: Ensure Configuration and AssetProcessor are \"pickleable\".\r\n- try:\r\n- with ProcessPoolExecutor(max_workers=num_workers) as executor:\r\n- # Create futures\r\n- futures = {}\r\n- log.debug(f\"Submitting {len(valid_inputs)} tasks...\")\r\n- # Removed the 1-second delay for potentially faster submission in non-CLI use\r\n- for i, input_path in enumerate(valid_inputs):\r\n- log.debug(f\"Submitting task {i+1}/{len(valid_inputs)} for: {Path(input_path).name}\")\r\n- future = executor.submit(\r\n- process_single_asset_wrapper,\r\n- input_path,\r\n- preset_name,\r\n- output_dir_for_processor,\r\n- overwrite\r\n- )\r\n- futures[future] = input_path # Store future -> input_path mapping\r\n-\r\n- # Process completed futures\r\n- for i, future in enumerate(as_completed(futures), 1):\r\n- input_path = futures[future]\r\n- asset_name = Path(input_path).name\r\n- log.info(f\"--- [{i}/{len(valid_inputs)}] Worker finished attempt for: {asset_name} ---\")\r\n- try:\r\n- # Get result tuple: (input_path_str, status_string, error_message_or_None)\r\n- result_tuple = future.result()\r\n- results_list.append(result_tuple)\r\n- input_path_res, status, err_msg = result_tuple\r\n-\r\n- # Increment counters based on status\r\n- if status == \"processed\":\r\n- successful_processed_count += 1\r\n- elif status == \"skipped\":\r\n- skipped_count += 1\r\n- elif status == \"failed\":\r\n- failed_count += 1\r\n- else: # Should not happen, but log as warning/failure\r\n- log.warning(f\"Unknown status '{status}' received for {asset_name}. Counting as failed.\")\r\n- failed_count += 1\r\n-\r\n- except Exception as e:\r\n- # Catch errors if the future itself fails (e.g., worker process crashed hard)\r\n- log.exception(f\"Critical worker failure for {asset_name}: {e}\")\r\n- results_list.append((input_path, \"failed\", f\"Worker process crashed: {e}\"))\r\n- failed_count += 1 # Count crashes as failures\r\n-\r\n- except Exception as pool_exc:\r\n- log.exception(f\"An error occurred with the process pool: {pool_exc}\")\r\n- # Re-raise or handle as appropriate for the calling context (monitor.py)\r\n- # For now, log and return current counts\r\n- return {\r\n- \"processed\": successful_processed_count,\r\n- \"skipped\": skipped_count,\r\n- \"failed\": failed_count + (len(valid_inputs) - len(results_list)), # Count unprocessed as failed\r\n- \"results_list\": results_list,\r\n- \"pool_error\": str(pool_exc) # Add pool error info\r\n- }\r\n-\r\n- return {\r\n- \"processed\": successful_processed_count,\r\n- \"skipped\": skipped_count,\r\n- \"failed\": failed_count,\r\n- \"results_list\": results_list\r\n- }\r\n-\r\n-\r\n-# --- Main Execution (for CLI usage) ---\r\n-def main():\r\n- \"\"\"Parses arguments, sets up logging, runs processing, and reports summary.\"\"\"\r\n- parser = setup_arg_parser()\r\n- args = parser.parse_args()\r\n-\r\n- # Setup logging based on verbosity argument *before* logging status messages\r\n- setup_logging(args.verbose)\r\n-\r\n- start_time = time.time()\r\n- log.info(\"Asset Processor Script Started (CLI Mode)\")\r\n-\r\n- # --- Validate Input Paths ---\r\n- valid_inputs = []\r\n- for p_str in args.input_paths:\r\n- p = Path(p_str)\r\n- if p.exists():\r\n- if p.is_dir() or (p.is_file() and p.suffix.lower() == '.zip'):\r\n- valid_inputs.append(p_str) # Store the original string path\r\n+ # 2. Determine Blend File Paths if Blender Exe is available\r\n+ if blender_exe:\r\n+ # Nodegroup Blend Path\r\n+ nodegroup_blend_arg = args.nodegroup_blend\r\n+ if nodegroup_blend_arg:\r\n+ p = Path(nodegroup_blend_arg)\r\n+ if p.is_file() and p.suffix.lower() == '.blend':\r\n+ nodegroup_blend_path = str(p.resolve())\r\n+ log.info(f\"Using nodegroup blend file from argument: {nodegroup_blend_path}\")\r\n else:\r\n- log.warning(f\"Input is not a directory or .zip, skipping: {p_str}\")\r\n+ log.warning(f\"Invalid nodegroup blend file path from argument: '{nodegroup_blend_arg}'. Ignoring.\")\r\n else:\r\n- log.warning(f\"Input path not found, skipping: {p_str}\")\r\n+ default_ng_path_str = getattr(core_config_module, 'DEFAULT_NODEGROUP_BLEND_PATH', None)\r\n+ if default_ng_path_str:\r\n+ p = Path(default_ng_path_str)\r\n+ if p.is_file() and p.suffix.lower() == '.blend':\r\n+ nodegroup_blend_path = str(p.resolve())\r\n+ log.info(f\"Using default nodegroup blend file from config: {nodegroup_blend_path}\")\r\n+ else:\r\n+ log.warning(f\"Invalid default nodegroup blend file path in config: '{default_ng_path_str}'. Ignoring.\")\r\n \r\n- if not valid_inputs:\r\n- log.error(\"No valid input paths found. Exiting.\")\r\n- sys.exit(1) # Exit with error code\r\n-\r\n- # --- Determine Output Directory ---\r\n- output_dir_str = args.output_dir # Get value from args (might be None)\r\n- if not output_dir_str:\r\n- log.debug(\"Output directory not specified via -o, reading default from config.py.\")\r\n- try:\r\n- output_dir_str = getattr(core_config_module, 'OUTPUT_BASE_DIR', None)\r\n- if not output_dir_str:\r\n- log.error(\"Output directory not specified with -o and OUTPUT_BASE_DIR not found or empty in config.py. Exiting.\")\r\n- sys.exit(1)\r\n- log.info(f\"Using default output directory from config.py: {output_dir_str}\")\r\n- except Exception as e:\r\n- log.error(f\"Could not read OUTPUT_BASE_BASE_DIR from config.py: {e}\")\r\n- sys.exit(1)\r\n-\r\n- # --- Resolve Output Path (Handles Relative Paths Explicitly) ---\r\n- output_path_obj: Path\r\n- if os.path.isabs(output_dir_str):\r\n- output_path_obj = Path(output_dir_str)\r\n- log.info(f\"Using absolute output directory: {output_path_obj}\")\r\n- else:\r\n- # Path() interprets relative paths against CWD by default\r\n- output_path_obj = Path(output_dir_str)\r\n- log.info(f\"Using relative output directory '{output_dir_str}'. Resolved against CWD to: {output_path_obj.resolve()}\")\r\n-\r\n- # --- Validate and Setup Output Directory ---\r\n- try:\r\n- # Resolve to ensure we have an absolute path for consistency and creation\r\n- resolved_output_dir = output_path_obj.resolve()\r\n- log.info(f\"Ensuring output directory exists: {resolved_output_dir}\")\r\n- resolved_output_dir.mkdir(parents=True, exist_ok=True)\r\n- # Use the resolved absolute path string for the processor\r\n- output_dir_for_processor = str(resolved_output_dir)\r\n- except Exception as e:\r\n- log.error(f\"Cannot create or access output directory '{resolved_output_dir}': {e}\", exc_info=True)\r\n- sys.exit(1)\r\n-\r\n- # --- Check Preset Existence (Basic Check) ---\r\n- preset_dir = Path(__file__).parent / \"presets\"\r\n- preset_file = preset_dir / f\"{args.preset}.json\"\r\n- if not preset_file.is_file():\r\n- log.error(f\"Preset file not found: {preset_file}\")\r\n- log.error(f\"Ensure a file named '{args.preset}.json' exists in the directory: {preset_dir.resolve()}\")\r\n- sys.exit(1)\r\n-\r\n- # --- Execute Processing via the new function ---\r\n- processing_results = run_processing(\r\n- valid_inputs=valid_inputs,\r\n- preset_name=args.preset,\r\n- output_dir_for_processor=output_dir_for_processor,\r\n- overwrite=args.overwrite,\r\n- num_workers=args.workers\r\n- )\r\n-\r\n- # --- Report Summary ---\r\n- duration = time.time() - start_time\r\n- successful_processed_count = processing_results[\"processed\"]\r\n- skipped_count = processing_results[\"skipped\"]\r\n- failed_count = processing_results[\"failed\"]\r\n- results_list = processing_results[\"results_list\"]\r\n-\r\n- log.info(\"=\" * 40)\r\n- log.info(\"Processing Summary\")\r\n- log.info(f\" Duration: {duration:.2f} seconds\")\r\n- log.info(f\" Assets Attempted: {len(valid_inputs)}\")\r\n- log.info(f\" Successfully Processed: {successful_processed_count}\")\r\n- log.info(f\" Skipped (Already Existed): {skipped_count}\")\r\n- log.info(f\" Failed: {failed_count}\")\r\n-\r\n- if processing_results.get(\"pool_error\"):\r\n- log.error(f\" Process Pool Error: {processing_results['pool_error']}\")\r\n- # Ensure failed count reflects pool error if it happened\r\n- if failed_count == 0 and successful_processed_count == 0 and skipped_count == 0:\r\n- failed_count = len(valid_inputs) # Assume all failed if pool died early\r\n-\r\n- exit_code = 0\r\n- if failed_count > 0:\r\n- log.warning(\"Failures occurred:\")\r\n- # Iterate through results to show specific errors for failed items\r\n- for input_path, status, err_msg in results_list:\r\n- if status == \"failed\":\r\n- log.warning(f\" - {Path(input_path).name}: {err_msg}\")\r\n- exit_code = 1 # Exit with error code if failures occurred\r\n- else:\r\n- # Consider skipped assets as a form of success for the overall run exit code\r\n- if successful_processed_count > 0 or skipped_count > 0:\r\n- log.info(\"All assets processed or skipped successfully.\")\r\n- exit_code = 0 # Exit code 0 indicates success (including skips)\r\n+ # Materials Blend Path\r\n+ materials_blend_arg = args.materials_blend\r\n+ if materials_blend_arg:\r\n+ p = Path(materials_blend_arg)\r\n+ if p.is_file() and p.suffix.lower() == '.blend':\r\n+ materials_blend_path = str(p.resolve())\r\n+ log.info(f\"Using materials blend file from argument: {materials_blend_path}\")\r\n+ else:\r\n+ log.warning(f\"Invalid materials blend file path from argument: '{materials_blend_arg}'. Ignoring.\")\r\n else:\r\n- # This case might happen if all inputs were invalid initially\r\n- log.warning(\"No assets were processed, skipped, or failed (check input validation logs).\")\r\n- exit_code = 0 # Still exit 0 as the script itself didn't crash\r\n+ default_mat_path_str = getattr(core_config_module, 'DEFAULT_MATERIALS_BLEND_PATH', None)\r\n+ if default_mat_path_str:\r\n+ p = Path(default_mat_path_str)\r\n+ if p.is_file() and p.suffix.lower() == '.blend':\r\n+ materials_blend_path = str(p.resolve())\r\n+ log.info(f\"Using default materials blend file from config: {materials_blend_path}\")\r\n+ else:\r\n+ log.warning(f\"Invalid default materials blend file path in config: '{default_mat_path_str}'. Ignoring.\")\r\n \r\n- sys.exit(exit_code)\r\n+ # 3. Execute Scripts if Paths are Valid\r\n+ if blender_exe:\r\n+ script_dir = Path(__file__).parent / \"blenderscripts\"\r\n+ nodegroup_script_path = script_dir / \"create_nodegroups.py\"\r\n+ materials_script_path = script_dir / \"create_materials.py\"\r\n+ asset_output_root = output_dir_for_processor # Use the resolved output dir\r\n \r\n+ if nodegroup_blend_path:\r\n+ if nodegroup_script_path.is_file():\r\n+ log.info(\"-\" * 40)\r\n+ log.info(\"Starting Blender Node Group Script Execution...\")\r\n+ success_ng = run_blender_script(\r\n+ blender_exe_path=blender_exe,\r\n+ blend_file_path=nodegroup_blend_path,\r\n+ python_script_path=str(nodegroup_script_path),\r\n+ asset_root_dir=asset_output_root\r\n+ )\r\n+ if not success_ng:\r\n+ log.error(\"Blender node group script execution failed.\")\r\n+ # Optionally change exit code if Blender script fails?\r\n+ # exit_code = 1\r\n+ log.info(\"Finished Blender Node Group Script Execution.\")\r\n+ log.info(\"-\" * 40)\r\n+ else:\r\n+ log.error(f\"Node group script not found: {nodegroup_script_path}\")\r\n \r\n-if __name__ == \"__main__\":\r\n- # This ensures the main() function runs only when the script is executed directly\r\n- # Important for multiprocessing to work correctly on some platforms (like Windows)\r\n- main()\n-# main.py\r\n-\r\n-import argparse\r\n-import sys\r\n-import time\r\n-import os\r\n-import logging\r\n-from pathlib import Path\r\n-from concurrent.futures import ProcessPoolExecutor, as_completed\r\n-import platform # To potentially adjust worker count defaults\r\n-from typing import List, Dict, Tuple, Optional # Added for type hinting\r\n-\r\n-# --- Assuming classes are in sibling files ---\r\n-try:\r\n- from configuration import Configuration, ConfigurationError\r\n- from asset_processor import AssetProcessor, AssetProcessingError\r\n- import config as core_config_module # <<< IMPORT config.py HERE\r\n-except ImportError as e:\r\n- # Provide a more helpful error message if imports fail\r\n- script_dir = Path(__file__).parent.resolve()\r\n- print(f\"ERROR: Failed to import necessary classes: {e}\")\r\n- print(f\"Ensure 'configuration.py' and 'asset_processor.py' exist in the directory:\")\r\n- print(f\" {script_dir}\")\r\n- print(\"Or that the directory is included in your PYTHONPATH.\")\r\n- sys.exit(1)\r\n-\r\n-# --- Setup Logging ---\r\n-# Keep setup_logging as is, it's called by main() or potentially monitor.py\r\n-def setup_logging(verbose: bool):\r\n- \"\"\"Configures logging for the application.\"\"\"\r\n- log_level = logging.DEBUG if verbose else logging.INFO\r\n- log_format = '%(asctime)s [%(levelname)-8s] %(name)s: %(message)s'\r\n- date_format = '%Y-%m-%d %H:%M:%S'\r\n-\r\n- # Configure root logger\r\n- # Remove existing handlers to avoid duplication if re-run in same session\r\n- for handler in logging.root.handlers[:]:\r\n- logging.root.removeHandler(handler)\r\n-\r\n- logging.basicConfig(\r\n- level=log_level,\r\n- format=log_format,\r\n- datefmt=date_format,\r\n- handlers=[\r\n- logging.StreamHandler(sys.stdout) # Log to console\r\n- # Optional: Add FileHandler for persistent logs\r\n- # logging.FileHandler(\"asset_processor.log\", mode='a', encoding='utf-8')\r\n- ]\r\n- )\r\n- # Get logger specifically for this main script\r\n- log = logging.getLogger(__name__) # or use 'main'\r\n- log.info(f\"Logging level set to: {logging.getLevelName(log_level)}\")\r\n- # Suppress overly verbose messages from libraries if needed (e.g., cv2)\r\n- # logging.getLogger('cv2').setLevel(logging.WARNING)\r\n-\r\n-# Use module-level logger after configuration\r\n-log = logging.getLogger(__name__)\r\n-\r\n-\r\n-# --- Argument Parser Setup ---\r\n-# Keep setup_arg_parser as is, it's only used when running main.py directly\r\n-def setup_arg_parser():\r\n- \"\"\"Sets up and returns the command-line argument parser.\"\"\"\r\n- # Determine a sensible default worker count\r\n- default_workers = 1\r\n- try:\r\n- # Use half the cores, but at least 1, max maybe 8-16? Depends on task nature.\r\n- # Let's try max(1, os.cpu_count() // 2)\r\n- cores = os.cpu_count()\r\n- if cores:\r\n- default_workers = max(1, cores // 2)\r\n- # Cap default workers? Maybe not necessary, let user decide via flag.\r\n- # default_workers = min(default_workers, 8) # Example cap\r\n- except NotImplementedError:\r\n- log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n-\r\n- parser = argparse.ArgumentParser(\r\n- description=\"Process asset files (ZIPs or folders) into a standardized library format using presets.\",\r\n- formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message\r\n- )\r\n- parser.add_argument(\r\n- \"input_paths\",\r\n- metavar=\"INPUT_PATH\",\r\n- type=str,\r\n- nargs='+', # Requires one or more input paths\r\n- help=\"Path(s) to the input ZIP file(s) or folder(s) containing assets.\"\r\n- )\r\n- parser.add_argument(\r\n- \"-p\", \"--preset\",\r\n- type=str,\r\n- required=True,\r\n- help=\"Name of the configuration preset (e.g., 'poliigon') located in the 'presets' directory (without .json extension).\"\r\n- )\r\n- parser.add_argument(\r\n- \"-o\", \"--output-dir\",\r\n- type=str,\r\n- required=False, # No longer required\r\n- default=None, # Default is None, will check core_config later\r\n- help=\"Override the default base output directory defined in config.py.\" # Updated help\r\n- )\r\n- parser.add_argument(\r\n- \"-w\", \"--workers\",\r\n- type=int,\r\n- default=default_workers,\r\n- help=\"Maximum number of assets to process concurrently in parallel processes.\"\r\n- )\r\n- parser.add_argument(\r\n- \"-v\", \"--verbose\",\r\n- action=\"store_true\", # Makes it a flag, value is True if present\r\n- help=\"Enable detailed DEBUG level logging for troubleshooting.\"\r\n- )\r\n- parser.add_argument(\r\n- \"--overwrite\",\r\n- action=\"store_true\",\r\n- help=\"Force reprocessing and overwrite existing output asset folders if they exist.\"\r\n- )\r\n- # Potential future flags:\r\n- # parser.add_argument(\"--log-file\", type=str, default=None, help=\"Path to save log output to a file.\")\r\n- return parser\r\n-\r\n-\r\n-# --- Worker Function ---\r\n-# Keep process_single_asset_wrapper as is, it's called by the processing pool\r\n-def process_single_asset_wrapper(input_path_str: str, preset_name: str, output_dir_str: str, overwrite: bool) -> Tuple[str, str, Optional[str]]:\r\n- \"\"\"\r\n- Wrapper function for processing a single asset in a separate process.\r\n- Handles instantiation of Configuration and AssetProcessor, passes the overwrite flag, and catches errors.\r\n- Returns: (input_path_str, status_string [\"processed\", \"skipped\", \"failed\"], error_message_or_None)\r\n- \"\"\"\r\n- # Note: Logging setup might need re-initialization in child processes on some platforms\r\n- # if file handlers or complex configurations are used. Stdout usually works.\r\n- # Removed forced DEBUG logging for worker\r\n- worker_log = logging.getLogger(f\"Worker_{os.getpid()}\") # Log with worker PID\r\n- # Initial log message moved inside try block for better status reporting\r\n-\r\n- try:\r\n- worker_log.info(f\"Starting processing attempt for: {Path(input_path_str).name}\")\r\n- # Each worker needs its own Configuration instance based on the preset name\r\n- config = Configuration(preset_name)\r\n- # Ensure output_dir_str is treated as the absolute base path\r\n- output_base_path = Path(output_dir_str)\r\n- input_path = Path(input_path_str)\r\n-\r\n- # Pass the overwrite flag to the AssetProcessor\r\n- processor = AssetProcessor(input_path, config, output_base_path, overwrite=overwrite)\r\n- status = processor.process() # <<< Execute the main processing pipeline and get status\r\n-\r\n- # Log based on status\r\n- if status == \"skipped\":\r\n- worker_log.info(f\"Worker skipped: {input_path.name}\")\r\n- elif status == \"processed\":\r\n- worker_log.info(f\"Worker finished processing: {input_path.name}\")\r\n- # Note: Failures within process() should raise exceptions caught below\r\n-\r\n- return (input_path_str, status, None) # Return the status string\r\n-\r\n- except (ConfigurationError, AssetProcessingError) as e:\r\n- worker_log.error(f\"Failed: {Path(input_path_str).name} - {type(e).__name__}: {e}\")\r\n- return (input_path_str, \"failed\", f\"{type(e).__name__}: {e}\") # Return \"failed\" status\r\n- except Exception as e:\r\n- # Catch any other unexpected errors originating from the worker process\r\n- # Use exc_info=True to log the full traceback from the worker\r\n- worker_log.exception(f\"Unexpected failure processing {Path(input_path_str).name}: {e}\")\r\n- return (input_path_str, \"failed\", f\"Unexpected Worker Error: {e}\") # Return \"failed\" status\r\n-\r\n-\r\n-# --- Core Processing Function ---\r\n-def run_processing(\r\n- valid_inputs: List[str],\r\n- preset_name: str,\r\n- output_dir_for_processor: str,\r\n- overwrite: bool,\r\n- num_workers: int\r\n-) -> Dict:\r\n- \"\"\"\r\n- Executes the core asset processing logic using a process pool.\r\n-\r\n- Args:\r\n- valid_inputs: List of validated input file/directory paths (strings).\r\n- preset_name: Name of the preset to use.\r\n- output_dir_for_processor: Absolute path string for the output base directory.\r\n- overwrite: Boolean flag to force reprocessing.\r\n- num_workers: Maximum number of worker processes.\r\n-\r\n- Returns:\r\n- A dictionary containing processing results:\r\n- {\r\n- \"processed\": int,\r\n- \"skipped\": int,\r\n- \"failed\": int,\r\n- \"results_list\": List[Tuple[str, str, Optional[str]]] # (input_path, status, error_msg)\r\n- }\r\n- \"\"\"\r\n- log.info(f\"Processing {len(valid_inputs)} asset(s) using preset '{preset_name}' with up to {num_workers} worker(s)...\")\r\n- results_list = []\r\n- successful_processed_count = 0\r\n- skipped_count = 0\r\n- failed_count = 0\r\n-\r\n- # Ensure at least one worker\r\n- num_workers = max(1, num_workers)\r\n-\r\n- # Using ProcessPoolExecutor is generally good if AssetProcessor tasks are CPU-bound.\r\n- # If tasks are mostly I/O bound, ThreadPoolExecutor might be sufficient.\r\n- # Important: Ensure Configuration and AssetProcessor are \"pickleable\".\r\n- try:\r\n- with ProcessPoolExecutor(max_workers=num_workers) as executor:\r\n- # Create futures\r\n- futures = {}\r\n- log.debug(f\"Submitting {len(valid_inputs)} tasks...\")\r\n- # Removed the 1-second delay for potentially faster submission in non-CLI use\r\n- for i, input_path in enumerate(valid_inputs):\r\n- log.debug(f\"Submitting task {i+1}/{len(valid_inputs)} for: {Path(input_path).name}\")\r\n- future = executor.submit(\r\n- process_single_asset_wrapper,\r\n- input_path,\r\n- preset_name,\r\n- output_dir_for_processor,\r\n- overwrite\r\n+ if materials_blend_path:\r\n+ if materials_script_path.is_file():\r\n+ log.info(\"-\" * 40)\r\n+ log.info(\"Starting Blender Material Script Execution...\")\r\n+ success_mat = run_blender_script(\r\n+ blender_exe_path=blender_exe,\r\n+ blend_file_path=materials_blend_path,\r\n+ python_script_path=str(materials_script_path),\r\n+ asset_root_dir=asset_output_root\r\n )\r\n- futures[future] = input_path # Store future -> input_path mapping\r\n-\r\n- # Process completed futures\r\n- for i, future in enumerate(as_completed(futures), 1):\r\n- input_path = futures[future]\r\n- asset_name = Path(input_path).name\r\n- log.info(f\"--- [{i}/{len(valid_inputs)}] Worker finished attempt for: {asset_name} ---\")\r\n- try:\r\n- # Get result tuple: (input_path_str, status_string, error_message_or_None)\r\n- result_tuple = future.result()\r\n- results_list.append(result_tuple)\r\n- input_path_res, status, err_msg = result_tuple\r\n-\r\n- # Increment counters based on status\r\n- if status == \"processed\":\r\n- successful_processed_count += 1\r\n- elif status == \"skipped\":\r\n- skipped_count += 1\r\n- elif status == \"failed\":\r\n- failed_count += 1\r\n- else: # Should not happen, but log as warning/failure\r\n- log.warning(f\"Unknown status '{status}' received for {asset_name}. Counting as failed.\")\r\n- failed_count += 1\r\n-\r\n- except Exception as e:\r\n- # Catch errors if the future itself fails (e.g., worker process crashed hard)\r\n- log.exception(f\"Critical worker failure for {asset_name}: {e}\")\r\n- results_list.append((input_path, \"failed\", f\"Worker process crashed: {e}\"))\r\n- failed_count += 1 # Count crashes as failures\r\n-\r\n- except Exception as pool_exc:\r\n- log.exception(f\"An error occurred with the process pool: {pool_exc}\")\r\n- # Re-raise or handle as appropriate for the calling context (monitor.py)\r\n- # For now, log and return current counts\r\n- return {\r\n- \"processed\": successful_processed_count,\r\n- \"skipped\": skipped_count,\r\n- \"failed\": failed_count + (len(valid_inputs) - len(results_list)), # Count unprocessed as failed\r\n- \"results_list\": results_list,\r\n- \"pool_error\": str(pool_exc) # Add pool error info\r\n- }\r\n-\r\n- return {\r\n- \"processed\": successful_processed_count,\r\n- \"skipped\": skipped_count,\r\n- \"failed\": failed_count,\r\n- \"results_list\": results_list\r\n- }\r\n-\r\n-\r\n-# --- Main Execution (for CLI usage) ---\r\n-def main():\r\n- \"\"\"Parses arguments, sets up logging, runs processing, and reports summary.\"\"\"\r\n- parser = setup_arg_parser()\r\n- args = parser.parse_args()\r\n-\r\n- # Setup logging based on verbosity argument *before* logging status messages\r\n- setup_logging(args.verbose)\r\n-\r\n- start_time = time.time()\r\n- log.info(\"Asset Processor Script Started (CLI Mode)\")\r\n-\r\n- # --- Validate Input Paths ---\r\n- valid_inputs = []\r\n- for p_str in args.input_paths:\r\n- p = Path(p_str)\r\n- if p.exists():\r\n- if p.is_dir() or (p.is_file() and p.suffix.lower() == '.zip'):\r\n- valid_inputs.append(p_str) # Store the original string path\r\n+ if not success_mat:\r\n+ log.error(\"Blender material script execution failed.\")\r\n+ # Optionally change exit code if Blender script fails?\r\n+ # exit_code = 1\r\n+ log.info(\"Finished Blender Material Script Execution.\")\r\n+ log.info(\"-\" * 40)\r\n else:\r\n- log.warning(f\"Input is not a directory or .zip, skipping: {p_str}\")\r\n- else:\r\n- log.warning(f\"Input path not found, skipping: {p_str}\")\r\n+ log.error(f\"Material script not found: {materials_script_path}\")\r\n \r\n- if not valid_inputs:\r\n- log.error(\"No valid input paths found. Exiting.\")\r\n- sys.exit(1) # Exit with error code\r\n-\r\n- # --- Determine Output Directory ---\r\n- output_dir_str = args.output_dir # Get value from args (might be None)\r\n- if not output_dir_str:\r\n- log.debug(\"Output directory not specified via -o, reading default from config.py.\")\r\n- try:\r\n- output_dir_str = getattr(core_config_module, 'OUTPUT_BASE_DIR', None)\r\n- if not output_dir_str:\r\n- log.error(\"Output directory not specified with -o and OUTPUT_BASE_DIR not found or empty in config.py. Exiting.\")\r\n- sys.exit(1)\r\n- log.info(f\"Using default output directory from config.py: {output_dir_str}\")\r\n- except Exception as e:\r\n- log.error(f\"Could not read OUTPUT_BASE_BASE_DIR from config.py: {e}\")\r\n- sys.exit(1)\r\n-\r\n- # --- Resolve Output Path (Handles Relative Paths Explicitly) ---\r\n- output_path_obj: Path\r\n- if os.path.isabs(output_dir_str):\r\n- output_path_obj = Path(output_dir_str)\r\n- log.info(f\"Using absolute output directory: {output_path_obj}\")\r\n- else:\r\n- # Path() interprets relative paths against CWD by default\r\n- output_path_obj = Path(output_dir_str)\r\n- log.info(f\"Using relative output directory '{output_dir_str}'. Resolved against CWD to: {output_path_obj.resolve()}\")\r\n-\r\n- # --- Validate and Setup Output Directory ---\r\n- try:\r\n- # Resolve to ensure we have an absolute path for consistency and creation\r\n- resolved_output_dir = output_path_obj.resolve()\r\n- log.info(f\"Ensuring output directory exists: {resolved_output_dir}\")\r\n- resolved_output_dir.mkdir(parents=True, exist_ok=True)\r\n- # Use the resolved absolute path string for the processor\r\n- output_dir_for_processor = str(resolved_output_dir)\r\n- except Exception as e:\r\n- log.error(f\"Cannot create or access output directory '{resolved_output_dir}': {e}\", exc_info=True)\r\n- sys.exit(1)\r\n-\r\n- # --- Check Preset Existence (Basic Check) ---\r\n- preset_dir = Path(__file__).parent / \"presets\"\r\n- preset_file = preset_dir / f\"{args.preset}.json\"\r\n- if not preset_file.is_file():\r\n- log.error(f\"Preset file not found: {preset_file}\")\r\n- log.error(f\"Ensure a file named '{args.preset}.json' exists in the directory: {preset_dir.resolve()}\")\r\n- sys.exit(1)\r\n-\r\n- # --- Execute Processing via the new function ---\r\n- processing_results = run_processing(\r\n- valid_inputs=valid_inputs,\r\n- preset_name=args.preset,\r\n- output_dir_for_processor=output_dir_for_processor,\r\n- overwrite=args.overwrite,\r\n- num_workers=args.workers\r\n- )\r\n-\r\n- # --- Report Summary ---\r\n- duration = time.time() - start_time\r\n- successful_processed_count = processing_results[\"processed\"]\r\n- skipped_count = processing_results[\"skipped\"]\r\n- failed_count = processing_results[\"failed\"]\r\n- results_list = processing_results[\"results_list\"]\r\n-\r\n- log.info(\"=\" * 40)\r\n- log.info(\"Processing Summary\")\r\n- log.info(f\" Duration: {duration:.2f} seconds\")\r\n- log.info(f\" Assets Attempted: {len(valid_inputs)}\")\r\n- log.info(f\" Successfully Processed: {successful_processed_count}\")\r\n- log.info(f\" Skipped (Already Existed): {skipped_count}\")\r\n- log.info(f\" Failed: {failed_count}\")\r\n-\r\n- if processing_results.get(\"pool_error\"):\r\n- log.error(f\" Process Pool Error: {processing_results['pool_error']}\")\r\n- # Ensure failed count reflects pool error if it happened\r\n- if failed_count == 0 and successful_processed_count == 0 and skipped_count == 0:\r\n- failed_count = len(valid_inputs) # Assume all failed if pool died early\r\n-\r\n- exit_code = 0\r\n- if failed_count > 0:\r\n- log.warning(\"Failures occurred:\")\r\n- # Iterate through results to show specific errors for failed items\r\n- for input_path, status, err_msg in results_list:\r\n- if status == \"failed\":\r\n- log.warning(f\" - {Path(input_path).name}: {err_msg}\")\r\n- exit_code = 1 # Exit with error code if failures occurred\r\n- else:\r\n- # Consider skipped assets as a form of success for the overall run exit code\r\n- if successful_processed_count > 0 or skipped_count > 0:\r\n- log.info(\"All assets processed or skipped successfully.\")\r\n- exit_code = 0 # Exit code 0 indicates success (including skips)\r\n- else:\r\n- # This case might happen if all inputs were invalid initially\r\n- log.warning(\"No assets were processed, skipped, or failed (check input validation logs).\")\r\n- exit_code = 0 # Still exit 0 as the script itself didn't crash\r\n-\r\n+ # --- Final Exit ---\r\n+ log.info(\"Asset Processor Script Finished.\")\r\n sys.exit(exit_code)\r\n \r\n \r\n if __name__ == \"__main__\":\r\n"
|
|
},
|
|
{
|
|
"date": 1745317210429,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n@@ -135,55 +135,81 @@\n \r\n # --- Worker Function ---\r\n def process_single_asset_wrapper(input_path_str: str, preset_name: str, output_dir_str: str, overwrite: bool, verbose: bool) -> Tuple[str, str, Optional[str]]:\r\n \"\"\"\r\n- Wrapper function for processing a single asset in a separate process.\r\n- Handles instantiation of Configuration and AssetProcessor, passes the overwrite flag, and catches errors.\r\n+ Wrapper function for processing a single input path (which might contain multiple assets)\r\n+ in a separate process. Handles instantiation of Configuration and AssetProcessor,\r\n+ passes the overwrite flag, catches errors, and interprets the multi-asset status dictionary.\r\n+\r\n Ensures logging is configured for the worker process.\r\n- Returns: (input_path_str, status_string [\"processed\", \"skipped\", \"failed\"], error_message_or_None)\r\n+\r\n+ Returns:\r\n+ Tuple[str, str, Optional[str]]:\r\n+ - input_path_str: The original input path processed.\r\n+ - overall_status_string: A single status string summarizing the outcome\r\n+ (\"processed\", \"skipped\", \"failed\", \"partial_success\").\r\n+ - error_message_or_None: An error message if failures occurred, potentially\r\n+ listing failed assets.\r\n \"\"\"\r\n # Explicitly configure logging for this worker process\r\n worker_log = logging.getLogger(f\"Worker_{os.getpid()}\") # Log with worker PID\r\n- # Check if root logger already has handlers (might happen in some environments)\r\n if not logging.root.handlers:\r\n- # Basic config if no handlers are set up (should be handled by main, but safety)\r\n logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)-8s] %(name)s: %(message)s')\r\n- # Set the level for this worker's logger based on the verbose flag\r\n worker_log.setLevel(logging.DEBUG if verbose else logging.INFO)\r\n- # Also ensure the root logger level is set if verbose\r\n if verbose:\r\n logging.root.setLevel(logging.DEBUG)\r\n \r\n+ input_path_obj = Path(input_path_str)\r\n+ input_name = input_path_obj.name\r\n \r\n try:\r\n- worker_log.info(f\"Starting processing attempt for: {Path(input_path_str).name}\")\r\n- # Each worker needs its own Configuration instance based on the preset name\r\n+ worker_log.info(f\"Starting processing attempt for input: {input_name}\")\r\n config = Configuration(preset_name)\r\n- # Ensure output_dir_str is treated as the absolute base path\r\n output_base_path = Path(output_dir_str)\r\n- input_path = Path(input_path_str)\r\n \r\n- # Pass the overwrite flag to the AssetProcessor\r\n- processor = AssetProcessor(input_path, config, output_base_path, overwrite=overwrite)\r\n- status = processor.process() # <<< Execute the main processing pipeline and get status\r\n+ processor = AssetProcessor(input_path_obj, config, output_base_path, overwrite=overwrite)\r\n+ # processor.process() now returns a Dict[str, List[str]]\r\n+ status_dict = processor.process()\r\n \r\n- # Log based on status\r\n- if status == \"skipped\":\r\n- worker_log.info(f\"Worker skipped: {input_path.name}\")\r\n- elif status == \"processed\":\r\n- worker_log.info(f\"Worker finished processing: {input_path.name}\")\r\n- # Note: Failures within process() should raise exceptions caught below\r\n+ # --- Interpret the status dictionary ---\r\n+ processed_assets = status_dict.get(\"processed\", [])\r\n+ skipped_assets = status_dict.get(\"skipped\", [])\r\n+ failed_assets = status_dict.get(\"failed\", [])\r\n \r\n- return (input_path_str, status, None) # Return the status string\r\n+ overall_status_string = \"failed\" # Default\r\n+ error_message = None\r\n \r\n+ if failed_assets:\r\n+ overall_status_string = \"failed\"\r\n+ error_message = f\"Failed assets within {input_name}: {', '.join(failed_assets)}\"\r\n+ worker_log.error(error_message) # Log the failure details\r\n+ elif processed_assets:\r\n+ overall_status_string = \"processed\"\r\n+ # Check for partial success (mix of processed/skipped and failed should be caught above)\r\n+ if skipped_assets:\r\n+ worker_log.info(f\"Input '{input_name}' processed with some assets skipped. Processed: {processed_assets}, Skipped: {skipped_assets}\")\r\n+ else:\r\n+ worker_log.info(f\"Input '{input_name}' processed successfully. Assets: {processed_assets}\")\r\n+ elif skipped_assets:\r\n+ overall_status_string = \"skipped\"\r\n+ worker_log.info(f\"Input '{input_name}' skipped (all contained assets already exist). Assets: {skipped_assets}\")\r\n+ else:\r\n+ # Should not happen if input contained files, but handle as failure.\r\n+ worker_log.warning(f\"Input '{input_name}' resulted in no processed, skipped, or failed assets. Reporting as failed.\")\r\n+ overall_status_string = \"failed\"\r\n+ error_message = f\"No assets processed, skipped, or failed within {input_name}.\"\r\n+\r\n+\r\n+ return (input_path_str, overall_status_string, error_message)\r\n+\r\n except (ConfigurationError, AssetProcessingError) as e:\r\n- worker_log.error(f\"Failed: {Path(input_path_str).name} - {type(e).__name__}: {e}\")\r\n- return (input_path_str, \"failed\", f\"{type(e).__name__}: {e}\") # Return \"failed\" status\r\n+ # Catch errors during processor setup or the process() call itself if it raises before returning dict\r\n+ worker_log.error(f\"Processing failed for input '{input_name}': {type(e).__name__}: {e}\")\r\n+ return (input_path_str, \"failed\", f\"{type(e).__name__}: {e}\")\r\n except Exception as e:\r\n- # Catch any other unexpected errors originating from the worker process\r\n- # Use exc_info=True to log the full traceback from the worker\r\n- worker_log.exception(f\"Unexpected failure processing {Path(input_path_str).name}: {e}\")\r\n- return (input_path_str, \"failed\", f\"Unexpected Worker Error: {e}\") # Return \"failed\" status\r\n+ # Catch any other unexpected errors\r\n+ worker_log.exception(f\"Unexpected worker failure processing input '{input_name}': {e}\")\r\n+ return (input_path_str, \"failed\", f\"Unexpected Worker Error: {e}\")\r\n \r\n \r\n # --- Core Processing Function ---\r\n def run_processing(\r\n"
|
|
},
|
|
{
|
|
"date": 1745506943322,
|
|
"content": "Index: \n===================================================================\n--- \n+++ \n@@ -401,12 +401,13 @@\n valid_inputs = []\r\n for p_str in args.input_paths:\r\n p = Path(p_str)\r\n if p.exists():\r\n- if p.is_dir() or (p.is_file() and p.suffix.lower() == '.zip'):\r\n+ suffix = p.suffix.lower()\r\n+ if p.is_dir() or (p.is_file() and suffix in ['.zip', '.rar', '.7z']):\r\n valid_inputs.append(p_str) # Store the original string path\r\n else:\r\n- log.warning(f\"Input is not a directory or .zip, skipping: {p_str}\")\r\n+ log.warning(f\"Input is not a directory or a supported archive type (.zip, .rar, .7z), skipping: {p_str}\")\r\n else:\r\n log.warning(f\"Input path not found, skipping: {p_str}\")\r\n \r\n if not valid_inputs:\r\n"
|
|
}
|
|
],
|
|
"date": 1745226384674,
|
|
"name": "Commit-0",
|
|
"content": "# main.py\r\n\r\nimport argparse\r\nimport sys\r\nimport time\r\nimport os\r\nimport logging\r\nfrom pathlib import Path\r\nfrom concurrent.futures import ProcessPoolExecutor, as_completed\r\nimport platform # To potentially adjust worker count defaults\r\nfrom typing import List, Dict, Tuple, Optional # Added for type hinting\r\n\r\n# --- Assuming classes are in sibling files ---\r\ntry:\r\n from configuration import Configuration, ConfigurationError\r\n from asset_processor import AssetProcessor, AssetProcessingError\r\n import config as core_config_module # <<< IMPORT config.py HERE\r\nexcept ImportError as e:\r\n # Provide a more helpful error message if imports fail\r\n script_dir = Path(__file__).parent.resolve()\r\n print(f\"ERROR: Failed to import necessary classes: {e}\")\r\n print(f\"Ensure 'configuration.py' and 'asset_processor.py' exist in the directory:\")\r\n print(f\" {script_dir}\")\r\n print(\"Or that the directory is included in your PYTHONPATH.\")\r\n sys.exit(1)\r\n\r\n# --- Setup Logging ---\r\n# Keep setup_logging as is, it's called by main() or potentially monitor.py\r\ndef setup_logging(verbose: bool):\r\n \"\"\"Configures logging for the application.\"\"\"\r\n log_level = logging.DEBUG if verbose else logging.INFO\r\n log_format = '%(asctime)s [%(levelname)-8s] %(name)s: %(message)s'\r\n date_format = '%Y-%m-%d %H:%M:%S'\r\n\r\n # Configure root logger\r\n # Remove existing handlers to avoid duplication if re-run in same session\r\n for handler in logging.root.handlers[:]:\r\n logging.root.removeHandler(handler)\r\n\r\n logging.basicConfig(\r\n level=log_level,\r\n format=log_format,\r\n datefmt=date_format,\r\n handlers=[\r\n logging.StreamHandler(sys.stdout) # Log to console\r\n # Optional: Add FileHandler for persistent logs\r\n # logging.FileHandler(\"asset_processor.log\", mode='a', encoding='utf-8')\r\n ]\r\n )\r\n # Get logger specifically for this main script\r\n log = logging.getLogger(__name__) # or use 'main'\r\n log.info(f\"Logging level set to: {logging.getLevelName(log_level)}\")\r\n # Suppress overly verbose messages from libraries if needed (e.g., cv2)\r\n # logging.getLogger('cv2').setLevel(logging.WARNING)\r\n\r\n# Use module-level logger after configuration\r\nlog = logging.getLogger(__name__)\r\n\r\n\r\n# --- Argument Parser Setup ---\r\n# Keep setup_arg_parser as is, it's only used when running main.py directly\r\ndef setup_arg_parser():\r\n \"\"\"Sets up and returns the command-line argument parser.\"\"\"\r\n # Determine a sensible default worker count\r\n default_workers = 1\r\n try:\r\n # Use half the cores, but at least 1, max maybe 8-16? Depends on task nature.\r\n # Let's try max(1, os.cpu_count() // 2)\r\n cores = os.cpu_count()\r\n if cores:\r\n default_workers = max(1, cores // 2)\r\n # Cap default workers? Maybe not necessary, let user decide via flag.\r\n # default_workers = min(default_workers, 8) # Example cap\r\n except NotImplementedError:\r\n log.warning(\"Could not detect CPU count, defaulting workers to 1.\")\r\n\r\n parser = argparse.ArgumentParser(\r\n description=\"Process asset files (ZIPs or folders) into a standardized library format using presets.\",\r\n formatter_class=argparse.ArgumentDefaultsHelpFormatter # Shows default values in help message\r\n )\r\n parser.add_argument(\r\n \"input_paths\",\r\n metavar=\"INPUT_PATH\",\r\n type=str,\r\n nargs='+', # Requires one or more input paths\r\n help=\"Path(s) to the input ZIP file(s) or folder(s) containing assets.\"\r\n )\r\n parser.add_argument(\r\n \"-p\", \"--preset\",\r\n type=str,\r\n required=True,\r\n help=\"Name of the configuration preset (e.g., 'poliigon') located in the 'presets' directory (without .json extension).\"\r\n )\r\n parser.add_argument(\r\n \"-o\", \"--output-dir\",\r\n type=str,\r\n required=False, # No longer required\r\n default=None, # Default is None, will check core_config later\r\n help=\"Override the default base output directory defined in config.py.\" # Updated help\r\n )\r\n parser.add_argument(\r\n \"-w\", \"--workers\",\r\n type=int,\r\n default=default_workers,\r\n help=\"Maximum number of assets to process concurrently in parallel processes.\"\r\n )\r\n parser.add_argument(\r\n \"-v\", \"--verbose\",\r\n action=\"store_true\", # Makes it a flag, value is True if present\r\n help=\"Enable detailed DEBUG level logging for troubleshooting.\"\r\n )\r\n parser.add_argument(\r\n \"--overwrite\",\r\n action=\"store_true\",\r\n help=\"Force reprocessing and overwrite existing output asset folders if they exist.\"\r\n )\r\n # Potential future flags:\r\n # parser.add_argument(\"--log-file\", type=str, default=None, help=\"Path to save log output to a file.\")\r\n return parser\r\n\r\n\r\n# --- Worker Function ---\r\n# Keep process_single_asset_wrapper as is, it's called by the processing pool\r\ndef process_single_asset_wrapper(input_path_str: str, preset_name: str, output_dir_str: str, overwrite: bool) -> Tuple[str, str, Optional[str]]:\r\n \"\"\"\r\n Wrapper function for processing a single asset in a separate process.\r\n Handles instantiation of Configuration and AssetProcessor, passes the overwrite flag, and catches errors.\r\n Returns: (input_path_str, status_string [\"processed\", \"skipped\", \"failed\"], error_message_or_None)\r\n \"\"\"\r\n # Note: Logging setup might need re-initialization in child processes on some platforms\r\n # if file handlers or complex configurations are used. Stdout usually works.\r\n # Removed forced DEBUG logging for worker\r\n worker_log = logging.getLogger(f\"Worker_{os.getpid()}\") # Log with worker PID\r\n # Initial log message moved inside try block for better status reporting\r\n\r\n try:\r\n worker_log.info(f\"Starting processing attempt for: {Path(input_path_str).name}\")\r\n # Each worker needs its own Configuration instance based on the preset name\r\n config = Configuration(preset_name)\r\n # Ensure output_dir_str is treated as the absolute base path\r\n output_base_path = Path(output_dir_str)\r\n input_path = Path(input_path_str)\r\n\r\n # Pass the overwrite flag to the AssetProcessor\r\n processor = AssetProcessor(input_path, config, output_base_path, overwrite=overwrite)\r\n status = processor.process() # <<< Execute the main processing pipeline and get status\r\n\r\n # Log based on status\r\n if status == \"skipped\":\r\n worker_log.info(f\"Worker skipped: {input_path.name}\")\r\n elif status == \"processed\":\r\n worker_log.info(f\"Worker finished processing: {input_path.name}\")\r\n # Note: Failures within process() should raise exceptions caught below\r\n\r\n return (input_path_str, status, None) # Return the status string\r\n\r\n except (ConfigurationError, AssetProcessingError) as e:\r\n worker_log.error(f\"Failed: {Path(input_path_str).name} - {type(e).__name__}: {e}\")\r\n return (input_path_str, \"failed\", f\"{type(e).__name__}: {e}\") # Return \"failed\" status\r\n except Exception as e:\r\n # Catch any other unexpected errors originating from the worker process\r\n # Use exc_info=True to log the full traceback from the worker\r\n worker_log.exception(f\"Unexpected failure processing {Path(input_path_str).name}: {e}\")\r\n return (input_path_str, \"failed\", f\"Unexpected Worker Error: {e}\") # Return \"failed\" status\r\n\r\n\r\n# --- Core Processing Function ---\r\ndef run_processing(\r\n valid_inputs: List[str],\r\n preset_name: str,\r\n output_dir_for_processor: str,\r\n overwrite: bool,\r\n num_workers: int\r\n) -> Dict:\r\n \"\"\"\r\n Executes the core asset processing logic using a process pool.\r\n\r\n Args:\r\n valid_inputs: List of validated input file/directory paths (strings).\r\n preset_name: Name of the preset to use.\r\n output_dir_for_processor: Absolute path string for the output base directory.\r\n overwrite: Boolean flag to force reprocessing.\r\n num_workers: Maximum number of worker processes.\r\n\r\n Returns:\r\n A dictionary containing processing results:\r\n {\r\n \"processed\": int,\r\n \"skipped\": int,\r\n \"failed\": int,\r\n \"results_list\": List[Tuple[str, str, Optional[str]]] # (input_path, status, error_msg)\r\n }\r\n \"\"\"\r\n log.info(f\"Processing {len(valid_inputs)} asset(s) using preset '{preset_name}' with up to {num_workers} worker(s)...\")\r\n results_list = []\r\n successful_processed_count = 0\r\n skipped_count = 0\r\n failed_count = 0\r\n\r\n # Ensure at least one worker\r\n num_workers = max(1, num_workers)\r\n\r\n # Using ProcessPoolExecutor is generally good if AssetProcessor tasks are CPU-bound.\r\n # If tasks are mostly I/O bound, ThreadPoolExecutor might be sufficient.\r\n # Important: Ensure Configuration and AssetProcessor are \"pickleable\".\r\n try:\r\n with ProcessPoolExecutor(max_workers=num_workers) as executor:\r\n # Create futures\r\n futures = {}\r\n log.debug(f\"Submitting {len(valid_inputs)} tasks...\")\r\n # Removed the 1-second delay for potentially faster submission in non-CLI use\r\n for i, input_path in enumerate(valid_inputs):\r\n log.debug(f\"Submitting task {i+1}/{len(valid_inputs)} for: {Path(input_path).name}\")\r\n future = executor.submit(\r\n process_single_asset_wrapper,\r\n:start_line:217\r\n-------\r\n input_path,\r\n preset_name,\r\n output_dir_for_processor,\r\n overwrite,\r\n args.verbose # Pass the verbose flag\r\n )\r\n futures[future] = input_path # Store future -> input_path mapping\r\n\r\n # Process completed futures\r\n for i, future in enumerate(as_completed(futures), 1):\r\n input_path = futures[future]\r\n asset_name = Path(input_path).name\r\n log.info(f\"--- [{i}/{len(valid_inputs)}] Worker finished attempt for: {asset_name} ---\")\r\n try:\r\n # Get result tuple: (input_path_str, status_string, error_message_or_None)\r\n result_tuple = future.result()\r\n results_list.append(result_tuple)\r\n input_path_res, status, err_msg = result_tuple\r\n\r\n # Increment counters based on status\r\n if status == \"processed\":\r\n successful_processed_count += 1\r\n elif status == \"skipped\":\r\n skipped_count += 1\r\n elif status == \"failed\":\r\n failed_count += 1\r\n else: # Should not happen, but log as warning/failure\r\n log.warning(f\"Unknown status '{status}' received for {asset_name}. Counting as failed.\")\r\n failed_count += 1\r\n\r\n except Exception as e:\r\n # Catch errors if the future itself fails (e.g., worker process crashed hard)\r\n log.exception(f\"Critical worker failure for {asset_name}: {e}\")\r\n results_list.append((input_path, \"failed\", f\"Worker process crashed: {e}\"))\r\n failed_count += 1 # Count crashes as failures\r\n\r\n except Exception as pool_exc:\r\n log.exception(f\"An error occurred with the process pool: {pool_exc}\")\r\n # Re-raise or handle as appropriate for the calling context (monitor.py)\r\n # For now, log and return current counts\r\n return {\r\n \"processed\": successful_processed_count,\r\n \"skipped\": skipped_count,\r\n \"failed\": failed_count + (len(valid_inputs) - len(results_list)), # Count unprocessed as failed\r\n \"results_list\": results_list,\r\n \"pool_error\": str(pool_exc) # Add pool error info\r\n }\r\n\r\n return {\r\n \"processed\": successful_processed_count,\r\n \"skipped\": skipped_count,\r\n \"failed\": failed_count,\r\n \"results_list\": results_list\r\n }\r\n\r\n\r\n# --- Main Execution (for CLI usage) ---\r\ndef main():\r\n \"\"\"Parses arguments, sets up logging, runs processing, and reports summary.\"\"\"\r\n parser = setup_arg_parser()\r\n args = parser.parse_args()\r\n\r\n # Setup logging based on verbosity argument *before* logging status messages\r\n setup_logging(args.verbose)\r\n\r\n start_time = time.time()\r\n log.info(\"Asset Processor Script Started (CLI Mode)\")\r\n\r\n # --- Validate Input Paths ---\r\n valid_inputs = []\r\n for p_str in args.input_paths:\r\n p = Path(p_str)\r\n if p.exists():\r\n if p.is_dir() or (p.is_file() and p.suffix.lower() == '.zip'):\r\n valid_inputs.append(p_str) # Store the original string path\r\n else:\r\n log.warning(f\"Input is not a directory or .zip, skipping: {p_str}\")\r\n else:\r\n log.warning(f\"Input path not found, skipping: {p_str}\")\r\n\r\n if not valid_inputs:\r\n log.error(\"No valid input paths found. Exiting.\")\r\n sys.exit(1) # Exit with error code\r\n\r\n # --- Determine Output Directory ---\r\n output_dir_str = args.output_dir # Get value from args (might be None)\r\n if not output_dir_str:\r\n log.debug(\"Output directory not specified via -o, reading default from config.py.\")\r\n try:\r\n output_dir_str = getattr(core_config_module, 'OUTPUT_BASE_DIR', None)\r\n if not output_dir_str:\r\n log.error(\"Output directory not specified with -o and OUTPUT_BASE_DIR not found or empty in config.py. Exiting.\")\r\n sys.exit(1)\r\n log.info(f\"Using default output directory from config.py: {output_dir_str}\")\r\n except Exception as e:\r\n log.error(f\"Could not read OUTPUT_BASE_DIR from config.py: {e}\")\r\n sys.exit(1)\r\n\r\n # --- Resolve Output Path (Handles Relative Paths Explicitly) ---\r\n output_path_obj: Path\r\n if os.path.isabs(output_dir_str):\r\n output_path_obj = Path(output_dir_str)\r\n log.info(f\"Using absolute output directory: {output_path_obj}\")\r\n else:\r\n # Path() interprets relative paths against CWD by default\r\n output_path_obj = Path(output_dir_str)\r\n log.info(f\"Using relative output directory '{output_dir_str}'. Resolved against CWD to: {output_path_obj.resolve()}\")\r\n\r\n # --- Validate and Setup Output Directory ---\r\n try:\r\n # Resolve to ensure we have an absolute path for consistency and creation\r\n resolved_output_dir = output_path_obj.resolve()\r\n log.info(f\"Ensuring output directory exists: {resolved_output_dir}\")\r\n resolved_output_dir.mkdir(parents=True, exist_ok=True)\r\n # Use the resolved absolute path string for the processor\r\n output_dir_for_processor = str(resolved_output_dir)\r\n except Exception as e:\r\n log.error(f\"Cannot create or access output directory '{resolved_output_dir}': {e}\", exc_info=True)\r\n sys.exit(1)\r\n\r\n # --- Check Preset Existence (Basic Check) ---\r\n preset_dir = Path(__file__).parent / \"presets\"\r\n preset_file = preset_dir / f\"{args.preset}.json\"\r\n if not preset_file.is_file():\r\n log.error(f\"Preset file not found: {preset_file}\")\r\n log.error(f\"Ensure a file named '{args.preset}.json' exists in the directory: {preset_dir.resolve()}\")\r\n sys.exit(1)\r\n\r\n # --- Execute Processing via the new function ---\r\n processing_results = run_processing(\r\n valid_inputs=valid_inputs,\r\n preset_name=args.preset,\r\n output_dir_for_processor=output_dir_for_processor,\r\n overwrite=args.overwrite,\r\n num_workers=args.workers\r\n )\r\n\r\n # --- Report Summary ---\r\n duration = time.time() - start_time\r\n successful_processed_count = processing_results[\"processed\"]\r\n skipped_count = processing_results[\"skipped\"]\r\n failed_count = processing_results[\"failed\"]\r\n results_list = processing_results[\"results_list\"]\r\n\r\n log.info(\"=\" * 40)\r\n log.info(\"Processing Summary\")\r\n log.info(f\" Duration: {duration:.2f} seconds\")\r\n log.info(f\" Assets Attempted: {len(valid_inputs)}\")\r\n log.info(f\" Successfully Processed: {successful_processed_count}\")\r\n log.info(f\" Skipped (Already Existed): {skipped_count}\")\r\n log.info(f\" Failed: {failed_count}\")\r\n\r\n if processing_results.get(\"pool_error\"):\r\n log.error(f\" Process Pool Error: {processing_results['pool_error']}\")\r\n # Ensure failed count reflects pool error if it happened\r\n if failed_count == 0 and successful_processed_count == 0 and skipped_count == 0:\r\n failed_count = len(valid_inputs) # Assume all failed if pool died early\r\n\r\n exit_code = 0\r\n if failed_count > 0:\r\n log.warning(\"Failures occurred:\")\r\n # Iterate through results to show specific errors for failed items\r\n for input_path, status, err_msg in results_list:\r\n if status == \"failed\":\r\n log.warning(f\" - {Path(input_path).name}: {err_msg}\")\r\n exit_code = 1 # Exit with error code if failures occurred\r\n else:\r\n # Consider skipped assets as a form of success for the overall run exit code\r\n if successful_processed_count > 0 or skipped_count > 0:\r\n log.info(\"All assets processed or skipped successfully.\")\r\n exit_code = 0 # Exit code 0 indicates success (including skips)\r\n else:\r\n # This case might happen if all inputs were invalid initially\r\n log.warning(\"No assets were processed, skipped, or failed (check input validation logs).\")\r\n exit_code = 0 # Still exit 0 as the script itself didn't crash\r\n\r\n sys.exit(exit_code)\r\n\r\n\r\nif __name__ == \"__main__\":\r\n # This ensures the main() function runs only when the script is executed directly\r\n # Important for multiprocessing to work correctly on some platforms (like Windows)\r\n main()"
|
|
}
|
|
]
|
|
} |