dlmodels.sh

#!/bin/bash

# Check if a file argument is provided
if [[ -z "$1" ]]; then
    echo "Usage: $0 <input_file>"
    exit 1
fi

# Set the input file from the first argument
INPUT_FILE="$1"
CHECKSUM_FILE=".checksums"

# Function to generate SHA-256 checksum for a given file
generate_checksum() {
    local filepath="$1"
    sha256sum "$filepath" | awk '{print $1}'
}

# Function to retrieve the remote checksum if available
get_remote_checksum() {
    local url="$1"
    local checksum_url="${url}.sha256"  # Assuming the checksum URL is <file_url>.sha256

    # Attempt to fetch the remote checksum
    remote_checksum=$(curl -s "$checksum_url")
    if [[ -z "$remote_checksum" ]]; then
        echo "no_remote_checksum"
    else
        echo "$remote_checksum"
    fi
}

# Function to download the file and verify its integrity
download_file() {
    local filepath="$1"
    local url="$2"

    echo -e "\nChecking file: $filepath"

    # Check if file exists locally and get its checksum
    if [[ -f "$filepath" ]]; then
        local_checksum=$(generate_checksum "$filepath")
        stored_checksum=$(grep "$filepath" "$CHECKSUM_FILE" | awk '{print $1}')

        # Check if local file's checksum matches stored checksum
        if [[ "$local_checksum" == "$stored_checksum" ]]; then
            echo -e "[SKIP] $filepath already exists and matches the stored checksum."
            return 0
        fi
    fi

    # Check remote checksum if available
    remote_checksum=$(get_remote_checksum "$url")
    if [[ "$remote_checksum" != "no_remote_checksum" ]]; then
        echo "Remote checksum found: $remote_checksum"
        # Compare with local file's checksum if it exists
        if [[ -f "$filepath" && "$remote_checksum" == "$local_checksum" ]]; then
            echo -e "[SKIP] $filepath already matches the remote checksum."
            return 0
        fi
    else
        echo "[INFO] No remote checksum available for $url. Proceeding with download."
    fi

    # Download the file with wget, showing progress
    echo -e "[DOWNLOAD] Downloading $filepath..."
    wget -c --progress=dot -O "$filepath" "$url" 2>&1 | \
    grep --line-buffered "%" | \
    sed -u -e "s,\.,,g" | \
    awk '{printf("\r[DOWNLOADING] %s - %s", "'$filepath'", $2)}'

    # Confirm completion
    echo -e "\n[COMPLETE] Downloaded $filepath"

    # Generate and store checksum after download
    new_checksum=$(generate_checksum "$filepath")
    echo "$new_checksum $filepath" >> "$CHECKSUM_FILE"
}

# Read and process each line in the specified input file
while IFS= read -r line; do
    # Split each line into filepath and URL
    filepath=$(echo "$line" | awk '{print $1}')
    url=$(echo "$line" | awk '{print $2}')

    # Call the download function for each file
    download_file "$filepath" "$url" &
done < "$INPUT_FILE"

# Wait for all background download processes to finish
wait
echo -e "\nAll downloads are completed."