import boto3 # for S3
from botocore.exceptions import NoCredentialsError, PartialCredentialsError, ClientError  # for exceptions
import logging # for logging
import json # for json.loads
import os # for os.path
import psycopg2 # for PostgreSQL

# Import custom modules
from file_utils import retrieve_file_contents, check_related_files, extract_and_validate_file_info # for file operations
from error_handler import notify_error
from db_utils import get_db_connection, check_inventory_in_db, check_objkey_in_file_db, add_file_record_and_relationship, retrieve_digital_file_names # for database operations

import config

# Function to check the existence of related files and validate in PostgreSQL 
def parse_s3_files( s3, s3_files, ach_variables, excluded_folders=[], s3_listing_cache=None):
    """
    Parses the S3 files and performs various operations on them.
    Args:
        s3 (S3): The S3 object for accessing S3 services.
        s3_files (list): The list of S3 files to be processed.
        s3_listing_cache (dict, optional): Mapping of S3 keys to listing objects.
    Returns:
        tuple: (uploaded_files_count, warning_files_count, error_files_count)
    Raises:
        FileNotFoundError: If a required file is not found in S3.
        ValueError: If a file has zero size or if the file type is unsupported.
        Exception: If any other unexpected exception occurs.
    """
    # Load the configuration from the .env file
    _ , db_config, _, bucket_name, _ = config.load_config()
    # logg ach_variables
    logging.info(f"ach_variables: {ach_variables}")

    # ---------------------------------------------------------------------
    # PHASE 3: PARSE & INSERT INTO DATABASE (DETAILS)
    #
    # 3.1) Filter out excluded prefixes and keep only files we care about.
    # 3.2) Validate each media file alongside its related sidecars (.json, .md5, .pdf).
    # 3.3) Cross-check the inventory code in the database and insert new records.
    # ---------------------------------------------------------------------
    logging.info("PHASE 3: Parse & insert - starting detailed file processing")

    try:
        # Ensure db_config is not None
        if db_config is None:
            raise ValueError("Database configuration is not loaded")
            # return # Exit the function if db_config is None

        conn = psycopg2.connect(**db_config)
        cur = conn.cursor()

        # Filter files with the desired prefix
        # excluded_prefix = ['TEST-FOLDER-DEV/', 'DOCUMENTAZIONE_FOTOGRAFICA/', 'BTC/', 'VHS/', 'UMT/', 'OV2/', 'OA4/']
        excluded_prefix = excluded_folders
        # Exclude files that start with any prefix in the excluded_prefix array
        filtered_files = [file for file in s3_files if not any(file.startswith(prefix) for prefix in excluded_prefix)]

        # Filter files with the desired prefix
        # DEBUG : filtered_files = [file for file in s3_files if file.startswith('TestFolderDev/')]
        # Length of filtered files
        logging.info(f"Array Length of filtered files: {len(filtered_files)}")
        # Counters
        error_files_count = 0
        warning_files_count = 0
        uploaded_files_count = 0  
        total_files = len(filtered_files)
        #for file in s3_files:
        for idx, file in enumerate(filtered_files, start=1):
            # Display progress to console only (not written to log files)
            print(f"--------------\n---  file {idx} of {total_files}  ---\n--------------", flush=True)

            # Ensure we start each file with a clean transaction state.
            try:
                conn.rollback()
            except Exception as e:
                logging.error(f"Rollback failed before processing {file}: {e}")

            try:
                cur.close()
            except Exception:
                pass
            cur = conn.cursor()

            try:
                # Use a savepoint per file to allow rollback on individual failures
                # without aborting the full batch.
                cur.execute("SAVEPOINT file_save")
            except Exception as e:
                # If the transaction is aborted, log and retry once.
                import traceback
                logging.error(f"Transaction aborted before processing {file}, retrying after reset: {e}")
                logging.error(traceback.format_exc())

                try:
                    conn.rollback()
                except Exception as rollback_err:
                    logging.error(f"Rollback failed while recovering from aborted transaction: {rollback_err}")

                try:
                    cur.close()
                except Exception:
                    pass
                cur = conn.cursor()

                # Retry savepoint once after recovery
                cur.execute("SAVEPOINT file_save")

            try:
                if file.endswith(('.mp4', '.mp3')):  # Check for both .mp4 and .mp3
                    logging.info("Processing file: %s in the bucket: %s", file, bucket_name)
                    # check if file exists in db
                    result = check_objkey_in_file_db(cur, file)
                    # Check the result and proceed accordingly
                    if result:
                        # logging.warning(f"File {file} already exists in the database.")
                        warning_files_count += 1
                        if os.getenv('ACH_SAFE_RUN', 'true').lower() == 'true':
                            logging.error("ACH_SAFE_RUN=true: aborting Phase 3 due to warnings (file already exists in DB): %s", file)
                            raise ValueError("ACH_SAFE_RUN=true: aborting due to warnings in Phase 3")
                        # Rollback to savepoint to undo any partial changes for this file
                        cur.execute("ROLLBACK TO SAVEPOINT file_save")
                        continue
 
                    ach_variables['file_fullpath'] = file #  is the Object key
                    ach_variables['inventory_code'] = os.path.splitext(os.path.basename(file))[0][:12]
                    logging.info(f"ach_variables['inventory_code'] {ach_variables['inventory_code']}: {file}")           
                    # Extract the file extension
                    ach_variables['objectKeys']['media'] = file
                    ach_variables['objectKeys']['pdf'] = f"{os.path.splitext(file)[0]}.pdf"
                    ach_variables['objectKeys']['pdf'] = ach_variables['objectKeys']['pdf'].replace('_H264', '')
                    from config import EXTENSION_MIME_MAP
                    if file.endswith('.mp4'):
                        ach_variables['objectKeys']['conservative_copy'] = f"{os.path.splitext(file)[0]}.mov"  #  remove _H264 is done later
                    elif file.endswith('.mp3'):
                        ach_variables['objectKeys']['conservative_copy'] = f"{os.path.splitext(file)[0]}.wav"
                    else:
                        logging.error(f"Unsupported file type: {file}")
                        error_files_count +=1
                        # Rollback to savepoint for this file
                        cur.execute("ROLLBACK TO SAVEPOINT file_save")
                        continue
                    
                    # Extract the file extension
                    file_extension = os.path.splitext(file)[1]  
                    ach_variables['extension'] = file_extension  # Store the file extension in ach_variables
                    logging.info(f"the file File extension: {file_extension}")
                    
                    # Extract the file name with directory part
                    file_name_with_path = os.path.splitext(file)[0]  # Remove the extension but keep path
                    logging.info(f"File name with path: {file_name_with_path}")

                # Extract the base name from the file name
                base_name = os.path.basename(file_name_with_path)  # Extract the base name with path removed
                logging.info(f"Base name: {base_name}")

                # Apply _H264 removal only for .mp4 files
                if file.endswith('.mp4'):
                    logging.info(f"File is an mp4 file: {file}. remove _H264")
                    base_name = base_name.replace('_H264', '')
                    file_name_with_path = file_name_with_path.replace('_H264', '')
                    logging.info(f"Modified base name for mp4: {base_name}")
                    logging.info(f"Modified file name with path for mp4: {file_name_with_path}")

                try:
                    # Retrieve and log the file size
                    # Optimized: Check cache first
                    if s3_listing_cache and file in s3_listing_cache:
                        file_size = s3_listing_cache[file].get('Size')
                        logging.info(f"Retrieved file size from cache for: {file}")
                    else:
                        file_size = get_file_size(s3, bucket_name, file)
                    
                    # maybe can trow an error inside te get_file_size function and catch it here
                    if file_size is not None:
                        ach_variables['media_disk_size'] = file_size
                        logging.info(f"The media file disk size is: {ach_variables['media_disk_size']}")
                    else:
                        logging.warning("Could not retrieve file size for %s.", file)
                        warning_files_count += 1
                        if os.getenv('ACH_SAFE_RUN', 'true').lower() == 'true':
                            logging.error("ACH_SAFE_RUN=true: aborting Phase 3 due to warnings (missing file size): %s", file)
                            raise ValueError("ACH_SAFE_RUN=true: aborting due to warnings in Phase 3")
                        continue  # Skip to the next file in the loop

                    logging.info("Start Validating files for %s...", base_name)
                    # Check if related file exist and retreive .pdf file size
                    try:
                        # Check if the required files exist in S3
                        ach_variables['pdf_disk_size'] = check_related_files(s3, file_name_with_path, file, bucket_name, s3_listing_cache=s3_listing_cache)
                        logging.info(f"PDF disk size: {ach_variables['pdf_disk_size']}")
                    except FileNotFoundError as e:
                        # Handle case where the file is not found
                        logging.error(f"File not found error: {e}")
                        error_files_count +=1
                        continue  # Move on to the next file in the loop 
                    except ValueError as e:
                        # Handle value errors
                        logging.error(f"Value error: {e} probabli filesize zero")
                        error_files_count +=1
                        continue  # Move on to the next file in the loop
                    except PermissionError as e:
                        # Handle permission errors
                        logging.error(f"Permission error: {e}")
                        error_files_count +=1    
                        continue  # Move on to the next file in the loop      
                    except Exception as e:
                        # Handle any other exceptions
                        logging.error(f"Validation step failed for {file}: {e}")
                        cur.execute("ROLLBACK TO SAVEPOINT file_save")
                        error_files_count += 1
                        continue
                    
                    # Retrieve the file contents for related files: .md5, .json
                    try:
                        # Check if the file exists in S3 and retrieve file contents
                        logging.info(f"Retrieving file contents for {file_name_with_path}...")
                        file_contents = retrieve_file_contents(s3, f"{file_name_with_path}")
                    except Exception as e:
                        # Log the error 
                        logging.error(f"Error retrieving file contents for {file_name_with_path}: {e}")
                        cur.execute("ROLLBACK TO SAVEPOINT file_save")
                        file_contents = None  # Set file_contents to None or handle it as needed
                        error_files_count +=1
                        continue  # Move on to the next file in the loop

                    # if contents dont exists
                    if file_contents is None:
                        logging.error(f"Error retrieving file contents for {file}.")
                        cur.execute("ROLLBACK TO SAVEPOINT file_save")
                        error_files_count +=1
                        continue  # Move on to the next file in the loop

                    # Ensure file_contents is a dictionary
                    if isinstance(file_contents, str):
                        file_contents = json.loads(file_contents)
                    
                    # Extract and validate file information
                    ach_variables['custom_data_in'], ach_variables['disk_size'], ach_variables['conservative_copy_extension'] = extract_and_validate_file_info(file_contents, file, ach_variables)
                    logging.info(f"Custom data extracted: {ach_variables['custom_data_in']}")
                    logging.info(f"Disk size extracted: {ach_variables['disk_size']}")
                    logging.info(f"Conservative copy extension extracted: {ach_variables['conservative_copy_extension']}")  
                    logging.info(f"File {file} file validation completed")
                except Exception as e:
                    logging.error(f"Error processing file {file}: {e}")
                    error_files_count +=1
                    continue  # Move on to the next file in the loop

                # no need truncate base name on this point    
                # base_name = base_name[:12]  # Truncate the base name to 12 characters
                # ////////////////////////////////////////////////////////////////////////////////

                # Check if the base name exists in the database
                logging.info(f"Checking database for {base_name}...")
                
                try:
                    # Call the function to check the inventory code in the database and get the result
                    result, truncated_base_name = check_inventory_in_db(s3, cur, base_name)
                    logging.info(f"base name {base_name}, truncated_base_name: {truncated_base_name}") 
                    # Check the result and proceed accordingly
                    if result:
                        logging.info(f"Inventory code {base_name} found in the database.")
                        # Call the function to retrieve digital file names
                        if retrieve_digital_file_names(s3, cur, base_name, ach_variables['objectKeys']['media']) == True:
                            # Call the function to add a file record and its relationship to the support record
                            # logging.info(f"ach_variables: {ach_variables}")
                            add_file_record_and_relationship(s3, cur, base_name, ach_variables)
                        else:
                            logging.warning(f"File record already exists for {base_name}.")
                            cur.execute("ROLLBACK TO SAVEPOINT file_save")
                            warning_files_count +=1
                            continue        
                    else:
                        logging.error(f"Inventory code {base_name} not found in the database.")
                        cur.execute("ROLLBACK TO SAVEPOINT file_save")
                        error_files_count +=1
                        continue    
                except Exception as e:
                    logging.error(f"DB operation failed for {base_name}: {e}")
                    cur.execute("ROLLBACK TO SAVEPOINT file_save")
                    error_files_count +=1
                    continue 
                
                # Commit the changes to the database    
                logging.info(f"commit to databse {base_name}...")

                # Commit to the database (conn, cur) only if everything is okay; otherwise, perform a rollback.
                conn.commit()
                uploaded_files_count +=1          
            except Exception as e:
                # Roll back the changes done for this file only and continue processing others
                import traceback
                logging.error(f"Error processing {file}: {e}. Rolling back this file's changes.")
                logging.error(traceback.format_exc())
                try:
                    cur.execute("ROLLBACK TO SAVEPOINT file_save")
                except Exception as rollback_err:
                    logging.error(f"Failed to rollback savepoint for {file}: {rollback_err}")
                error_files_count += 1
                continue
            finally:
                # Release the savepoint so it doesn't linger in the session
                try:
                    cur.execute("RELEASE SAVEPOINT file_save")
                except Exception:
                    # Ignore release errors; rollback already cleaned up state if needed
                    pass
        cur.close()
        conn.close()
    except ValueError as e:
        # Handle specific validation errors
        logging.error(f"Validation error: {e}")
        #handle_error(e)  # Pass the ValueError to handle_error
        raise e  # Raise the exception to the calling function
    except Exception as e:
        # Handle any other unexpected errors
        import traceback
        notify_error("FATAL ERROR in Phase 3 process", e)
        raise e  # Raise the exception to the calling function
    
    # return the file saved
    return uploaded_files_count, warning_files_count, error_files_count

# Function to create an S3 client
def create_s3_client(aws_config):
    logging.info(f'Creating S3 client with endpoint: {aws_config["endpoint_url"]}')
    try:
        s3 = boto3.client(
            's3',
            endpoint_url=aws_config['endpoint_url'],
            aws_access_key_id=aws_config['aws_access_key_id'],
            aws_secret_access_key=aws_config['aws_secret_access_key'],
            region_name=aws_config['region_name'],
            config=boto3.session.Config(
                signature_version='s3v4',
                s3={'addressing_style': 'path'}
            )
        )
        logging.info('S3 client created successfully')
        return s3
    except (NoCredentialsError, PartialCredentialsError) as e:
        logging.error(f'Error creating S3 client: {e}')
        raise e

# Function to list the contents of an S3 bucket
def list_s3_bucket(s3_client, bucket_name):
    """
    Lists S3 bucket contents with optional local JSON caching.
    Uses ACH_CACHE_S3_LIST from .env to decide if it should cache/read from 's3_cache.json'.
    """
    cache_enabled = os.getenv('ACH_CACHE_S3_LIST', 'false').lower() == 'true'
    cache_file = 's3_cache.json'

    if cache_enabled and os.path.exists(cache_file):
        try:
            with open(cache_file, 'r', encoding='utf-8') as f:
                cached_data = json.load(f)
                logging.info(f"Loaded {len(cached_data)} items from local cache: {cache_file}")
                return cached_data
        except Exception as e:
            logging.warning(f"Failed to read S3 cache file: {e}. Falling back to S3 listing.")

    try:
        logging.info(f"Listing all objects in bucket: {bucket_name}...")
        paginator = s3_client.get_paginator('list_objects_v2')
        bucket_contents = []

        # Convert datetime objects to string for JSON serialization
        def _serialize_datetime(obj):
            if isinstance(obj, datetime):
                return obj.isoformat()
            return obj

        from datetime import datetime

        for page in paginator.paginate(Bucket=bucket_name):
            if 'Contents' in page:
                for obj in page['Contents']:
                    # Normalize dates for JSON compatibility
                    if 'LastModified' in obj:
                        obj['LastModified'] = obj['LastModified'].isoformat()
                    bucket_contents.extend([obj])

        logging.info(f"Retrieved {len(bucket_contents)} items from the bucket.")

        # Save to cache if enabled
        if cache_enabled:
            try:
                with open(cache_file, 'w', encoding='utf-8') as f:
                    json.dump(bucket_contents, f)
                logging.info(f"S3 bucket listing saved to local cache: {cache_file}")
            except Exception as e:
                logging.warning(f"Failed to write S3 cache file: {e}")

        return bucket_contents
    except ClientError as e:
        logging.error(f'Error listing bucket contents: {e}')
        raise e
       
# Function to get file size from S3
def get_file_size(s3_client, bucket_name, file_key):
    try:
        response = s3_client.head_object(Bucket=bucket_name, Key=file_key)
        return response['ContentLength']
    except ClientError as e:
        logging.error(f"Failed to retrieve file size for {file_key}: {e}")
        return None  # or an appropriate fallback value
    except Exception as e:
        logging.error(f"An unexpected error occurred: {e}")
        return None

# Function to check if a file exists in S3
def check_file_exists_in_s3(s3, file_name,bucket_name):
    """
    Checks if a file exists in an S3 bucket.

    Parameters:
    - s3 (boto3.client): The S3 client object.
    - file_name (str): The name of the file to check.

    Returns:
    - bool: True if the file exists, False otherwise.

    Raises:
    - ClientError: If there is an error checking the file.

    """
    try:
        s3.head_object(Bucket=bucket_name, Key=file_name)
        return True
    except ClientError as e:
        if e.response['Error']['Code'] == '404':
            return False
        else:          
            logging.error(f'Error checking file {file_name}: {e}')
            raise e

# Function to retrieve file contents from S3
def upload_file_to_s3(s3_client, file_path, bucket_name, object_name=None):
    if object_name is None:
        object_name = file_path
    try:
        s3_client.upload_file(file_path, bucket_name, object_name)
        logging.info(f"File {file_path} uploaded to {bucket_name}/{object_name}")
    except ClientError as e:
        logging.error(f'Error uploading file {file_path} to bucket {bucket_name}: {e}')
        raise e

# Function to download a file from S3
def download_file_from_s3(s3_client, bucket_name, object_name, file_path):
    try:
        s3_client.download_file(bucket_name, object_name, file_path)
        logging.info(f"File {object_name} downloaded from {bucket_name} to {file_path}")
    except ClientError as e:
        logging.error(f'Error downloading file {object_name} from bucket {bucket_name}: {e}')
        raise e 