#!/bin/bash
 
# This script identifies and lists near-duplicate composite files that can get created in 
# the composite directories during the processing of granules. Any duplicate images that
# have a nominal timestamp within 10 minutes of each other are removed. 
# 

vsat=$1   # 'npp' 'j01' 'j02' 'Tandem'

# The directory containing the files
DIRECTORY="/data/smcd6/abailey/VIIRS/Temporary_Data/${vsat}_maps"

# The log file to store the list of files to be deleted
LOG_FILE="${vsat}_files_to_delete.log"
 
# Clear the log file if it exists
> "$LOG_FILE"
 
# Create an associative array to store the file timestamps and prefixes
declare -A file_timestamps
declare -A file_prefixes
 
# Iterate through each file in the directory
for file in "$DIRECTORY"/*.nc; do

    # Extract the base filename without the path
    filename=$(basename "$file")

    # Check if the filename matches the expected format
    if [[ "$filename" =~ ^(sps|nps)_(NPP|NOAA20|NOAA21)_VIIRS_750M_Granule_3305x3305_([0-9]{7})_([0-9]{4})_([0-9]{2})\.nc$ ]]; then
        # Extract the prefix, satellite, date, and time parts
        prefix="${BASH_REMATCH[1]}"         # "sps" or "nps" 
        satellite="${BASH_REMATCH[2]}"      # "NPP", "NOAA20", or "NOAA21" 
        date_part="${BASH_REMATCH[3]}"      # YYYYDDD
        time_part="${BASH_REMATCH[4]}"      # HHMM
	#time_sec="${BASH_REMATCH[5]}"      # SS...not used

        # Extract the year, day of year, hour, and minute 
        year=${date_part:0:4}
        day_of_year=${date_part:4:3}
        hour=${time_part:0:2}
        minute=${time_part:2:2}

        # Convert the year and day of year to a Unix timestamp 
        timestamp=$(date -d "${year}-01-01 +$((10#$day_of_year - 1)) days" +%s)

        # Add the hours and minutes to the timestamp in seconds
        timestamp=$((timestamp + 10#$hour * 3600 + 10#$minute * 60))
 
        # Check for near-duplicate files
        for existing_file in "${!file_timestamps[@]}"; do
            existing_timestamp=${file_timestamps[$existing_file]}
            existing_prefix=${file_prefixes[$existing_file]}
 
            # Calculate the time difference
            difference=$(( (timestamp - existing_timestamp) / 60 ))
 
            # If the prefixes are the same AND the time difference is less than 11 minutes, it's a near-duplicate
            if [[ "$prefix" == "$existing_prefix" && "$difference" -lt 11 && "$difference" -gt -11 && "$file" != "$existing_file" ]]; then
                #echo "Near-duplicate found: $(basename "$file") is a near-duplicate of $(basename "$existing_file") (Prefix: $prefix)" >> "$LOG_FILE"
                echo $(basename "$file") >> "$LOG_FILE"
            fi
        done
 
        # Add the current file and its data to the associative arrays
        file_timestamps["$file"]=$timestamp
        file_prefixes["$file"]=$prefix
    fi
done

# Define the file containing the list of duplicates
dup_input_file="$LOG_FILE"

# Check if the file exists and is not empty
if [[ -s "$dup_input_file" ]]; then
    echo "Starting removal of duplicate files..."

    # Loop through each line of the file
    while IFS= read -r line; do
        # Construct the full path to the file to be removed
        full_path="${DIRECTORY}/$line"

        echo "Removing: $full_path"
        rm "$full_path"

    done < "$dup_input_file"

    echo "Finished removing files."
else
    echo "Input file '$dup_input_file' is empty or does not exist. No files to remove."
fi



