I would probably keep the exclusions in a separate file for manageability (exclude.config), then have the script look for that file: if found - then exclude files or folders listed in it, if not - dont exclude anything.
# exclude.config
# Lines starting with # are comments
# Exclude specific files
- desktop.ini
# Exclude specific folders (use trailing slash for directories)
- $RECYCLE.BIN/
- Thumbs.db
# Exclude any folder named "Backup" anywhere in the path
- **/Backup/
# Exclude all .tmp files
- *.tmp
This format will be the same as rclone: - for exclusion, ** for recursive matching, / for directories. This aligns with rclone’s --exclude or --filter-from options
Then add the logic to the top of existing script:
# Path to exclusion file
EXCLUDE_CONFIG="$DB_DIR/exclude.config"
EXCLUDE_FILTER="$DB_DIR/exclude_filter.txt"
# Initialize exclude filter file
: > "$EXCLUDE_FILTER"
# Check if exclude.config exists and convert to find-compatible patterns
if [ -f "$EXCLUDE_CONFIG" ]; then
write_log "Applying exclusions from $EXCLUDE_CONFIG"
# Convert rclone filter patterns to find-compatible patterns
while IFS= read -r line; do
# Skip comments and empty lines
[[ "$line" =~ ^#.*$ || -z "$line" ]] && continue
# Remove leading '-' and trim whitespace
pattern=$(echo "$line" | sed 's/^- *//')
# Handle rclone patterns (simplified for find)
if [[ "$pattern" == */ ]]; then
# Directory (ends with /)
pattern=$(echo "$pattern" | sed 's/\/$//')
echo "-path '*/$pattern' -prune" >> "$EXCLUDE_FILTER"
else
# File or pattern
if [[ "$pattern" == **/* ]]; then
pattern=$(echo "$pattern" | sed 's/\*\*\/*//')
echo "-name '$pattern' -prune" >> "$EXCLUDE_FILTER"
else
echo "-name '$pattern' -prune" >> "$EXCLUDE_FILTER"
fi
fi
# Add to rclone filter file
echo "- $pattern" >> "$EXCLUDE_FILTER.rclone"
done < "$EXCLUDE_CONFIG"
fi
will also need to modify the find command:
# Build find exclude arguments
FIND_EXCLUDE_ARGS=""
if [ -s "$EXCLUDE_FILTER" ]; then
FIND_EXCLUDE_ARGS=$(cat "$EXCLUDE_FILTER" | tr '\n' ' ')
fi
# Update current file info with exclusions
find "$SOURCE_DIR" \( $FIND_EXCLUDE_ARGS \) -o -type f -printf "%p|%T@|%s\n" > "$CURRENT_FILES"
sync and folder tracking will also need to be updated with filters. The filtering script should look something like this (I DID NOT TEST THIS):
#!/bin/bash
# /mnt/Datapool/home/USERNAME/proton_file_tracker/pd_file_sync_script.sh
# v2 - adding folder tracking
# v3 - adding reset state mechanism
# v4 - adding exclusion filtering
SOURCE_DIR="/mnt/Datapool/SOURCE_DIR" #source dir to sync up, dont include the last forward slash: something/something/source
REMOTE_DIR="proton_drive:PROTON_DESTINATION_DIR" #destination on Proton Drive for synced files, dont include the last forward slash
LOG_FILE="/mnt/Datapool/Docs/PATH_TO_LOG/LOGFILENAME.log" #full path to log file
DB_DIR="/mnt/Datapool/home/USERNAME/proton_file_tracker" #dir where databases and temp files are stored
DB_FILES="$DB_DIR/file_tracker.db" #db holding files state for tracking new/changed/deleted files
CURRENT_FILES="$DB_DIR/current_files.txt" #temp files
CHANGED_FILES="$DB_DIR/changed_files.txt"
DELETED_FILES="$DB_DIR/deleted_files.txt"
DB_FOLDERS="$DB_DIR/folder_tracker.db" #db holding folders state to track deleted folders
CURRENT_FOLDERS="$DB_DIR/current_folders.txt" #temp files
DELETED_FOLDERS="$DB_DIR/deleted_folders.txt"
EXCLUDE_CONFIG="$DB_DIR/exclude.config"
EXCLUDE_FILTER="$DB_DIR/exclude_filter.txt"
EXCLUDE_FILTER_RCLONE="$DB_DIR/exclude_filter_rclone.txt"
RESET_STATE=0
FIRST_RUN=0
# Define write_log function early
write_log() {
if [ -f "$1" ]; then
while IFS= read -r line; do
if ! echo "$line" | grep -q "A file or folder with that name already exists (Code=2500, Status=422)"; then
if ! echo "$line" | grep -qE "Transferred:|Errors:|Checks:|Deleted:|Elapsed time:"; then
echo "$(date '+%Y-%m-%d %H:%M:%S') - $line" >> "$LOG_FILE"
fi
fi
done < "$1"
else
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
fi
}
# Initial log header
mkdir -p "$(dirname "$LOG_FILE")"
echo "" >> "$LOG_FILE"
printf "#%.0s" {1..80} >> "$LOG_FILE"; echo >> "$LOG_FILE"
write_log "Sync operation started"
printf "#%.0s" {1..80} >> "$LOG_FILE"; echo >> "$LOG_FILE"
# Check for --resetstate parameter (case-insensitive)
for arg in "$@"; do
if [[ "${arg,,}" == "--resetstate" ]]; then
RESET_STATE=1
write_log "Resetting state with --resetstate parameter"
write_log "Creating file and folder tracking initial state"
rm -f "$DB_FILES" "$DB_FOLDERS"
break
fi
done
mkdir -p "$DB_DIR"
# Initialize exclude filter files
: > "$EXCLUDE_FILTER"
: > "$EXCLUDE_FILTER_RCLONE"
# Process exclude.config if it exists
if [ -f "$EXCLUDE_CONFIG" ]; then
write_log "Applying exclusions from $EXCLUDE_CONFIG"
while IFS= read -r line; do
# Skip comments and empty lines
[[ "$line" =~ ^#.*$ || -z "$line" ]] && continue
# Remove leading '-' and trim whitespace
pattern=$(echo "$line" | sed 's/^- *//')
# Add to rclone filter file
echo "- $pattern" >> "$EXCLUDE_FILTER_RCLONE"
# Convert to find-compatible pattern
if [[ "$pattern" == */ ]]; then
# Directory (ends with /)
pattern=$(echo "$pattern" | sed 's/\/$//')
echo "-path '*/$pattern' -prune" >> "$EXCLUDE_FILTER"
else
# File or pattern
if [[ "$pattern" == **/* ]]; then
pattern=$(echo "$pattern" | sed 's/\*\*\/*//')
echo "-name '$pattern' -prune" >> "$EXCLUDE_FILTER"
else
echo "-name '$pattern' -prune" >> "$EXCLUDE_FILTER"
fi
fi
done < "$EXCLUDE_CONFIG"
fi
# Build find exclude arguments
FIND_EXCLUDE_ARGS=""
if [ -s "$EXCLUDE_FILTER" ]; then
FIND_EXCLUDE_ARGS=$(cat "$EXCLUDE_FILTER" | tr '\n' ' ')
fi
# Define log_footer function
log_footer() {
printf "#%.0s" {1..80} >> "$LOG_FILE"; echo >> "$LOG_FILE"
write_log "Sync completed successfully"
write_log "New/changed files: $NEW_CHANGED_COUNT"
write_log "Deleted files: $DELETED_COUNT"
write_log "Deleted folders: $DELETED_FOLDER_COUNT"
printf "#%.0s" {1..80} >> "$LOG_FILE"; echo >> "$LOG_FILE"
echo "" >> "$LOG_FILE"
}
# Initialize file database
if [ ! -f "$DB_FILES" ]; then
sqlite3 "$DB_FILES" <<SQL
CREATE TABLE files (path TEXT PRIMARY KEY, mtime INTEGER, size INTEGER);
CREATE TABLE metadata (key TEXT PRIMARY KEY, value INTEGER);
INSERT INTO metadata (key, value) VALUES ('last_run_time', 0);
SQL
fi
# Update current file info with exclusions
find "$SOURCE_DIR" \( $FIND_EXCLUDE_ARGS \) -o -type f -printf "%p|%T@|%s\n" > "$CURRENT_FILES"
CURRENT_TIMESTAMP=$(date +%s)
RELATIVE_SQL_CHANGED="SELECT substr(f.path, length('$SOURCE_DIR') + 1) AS path FROM files f LEFT JOIN files_last_run flr ON f.path = flr.path WHERE flr.path IS NULL OR f.mtime > COALESCE(flr.mtime, 0);"
RELATIVE_SQL_DELETED="SELECT substr(flr.path, length('$SOURCE_DIR') + 1) AS path FROM files_last_run flr WHERE flr.path NOT IN (SELECT path FROM files);"
sqlite3 "$DB_FILES" <<SQL > /dev/null 2>&1
PRAGMA synchronous=OFF;
PRAGMA journal_mode=WAL;
BEGIN TRANSACTION;
DROP TABLE IF EXISTS files_last_run;
ALTER TABLE files RENAME TO files_last_run;
CREATE TABLE files (path TEXT PRIMARY KEY, mtime INTEGER, size INTEGER);
CREATE INDEX IF NOT EXISTS idx_files_path ON files(path);
CREATE INDEX IF NOT EXISTS idx_files_last_run_path ON files_last_run(path);
.import $CURRENT_FILES files
.output $CHANGED_FILES
$RELATIVE_SQL_CHANGED
.output $DELETED_FILES
$RELATIVE_SQL_DELETED
INSERT OR REPLACE INTO metadata (key, value) VALUES ('last_run_time', $CURRENT_TIMESTAMP);
COMMIT;
SQL
# Sync changed files
if [ -s "$CHANGED_FILES" ] && [ "$RESET_STATE" -eq 0 ]; then
TEMP_LOG="$LOG_FILE.tmp"
RCLONE_FILTER=""
if [ -f "$EXCLUDE_FILTER_RCLONE" ] && [ -s "$EXCLUDE_FILTER_RCLONE" ]; then
RCLONE_FILTER="--filter-from $EXCLUDE_FILTER_RCLONE"
fi
if ! rclone sync "$SOURCE_DIR" "$REMOTE_DIR" --files-from "$CHANGED_FILES" --update --local-no-check-updated --protondrive-replace-existing-draft=true $RCLONE_FILTER --log-file="$TEMP_LOG" --log-format "date:'2006-01-02 15:04:05',level,msg" --log-level INFO --stats 0; then
echo "Error syncing files. Details in $TEMP_LOG" >> "$LOG_FILE"
exit 1
fi
write_log "$TEMP_LOG"
rm -f "$TEMP_LOG"
fi
# Delete files
if [ -s "$DELETED_FILES" ] && [ "$RESET_STATE" -eq 0 ]; then
TEMP_LOG="$LOG_FILE.tmp"
RCLONE_FILTER=""
if [ -f "$EXCLUDE_FILTER_RCLONE" ] && [ -s "$EXCLUDE_FILTER_RCLONE" ]; then
RCLONE_FILTER="--filter-from $EXCLUDE_FILTER_RCLONE"
fi
if ! rclone delete "$REMOTE_DIR" --include-from "$DELETED_FILES" --protondrive-replace-existing-draft=true $RCLONE_FILTER --log-file="$TEMP_LOG" --log-format "date:'2006-01-02 15:04:05',level,msg" --log-level INFO --stats 0; then
echo "Error deleting files. Details in $TEMP_LOG" >> "$LOG_FILE"
exit 1
fi
write_log "$TEMP_LOG"
rm -f "$TEMP_LOG"
fi
# Initialize folder database
if [ ! -f "$DB_FOLDERS" ]; then
sqlite3 "$DB_FOLDERS" <<SQL
CREATE TABLE folders (path TEXT PRIMARY KEY);
CREATE TABLE metadata (key TEXT PRIMARY KEY, value INTEGER);
INSERT INTO metadata (key, value) VALUES ('last_run_time', 0);
SQL
FIRST_RUN=1
write_log "Creating folder tracking initial state"
fi
# Generate current folders with exclusions
awk -F'|' '{sub(/[^/]*$/, "", $1); print $1}' "$CURRENT_FILES" | sort -u > "$CURRENT_FOLDERS"
# Filter out excluded folders
if [ -f "$EXCLUDE_CONFIG" ]; then
TEMP_FOLDERS="$DB_DIR/temp_folders.txt"
: > "$TEMP_FOLDERS"
while IFS= read -r folder; do
skip=0
while IFS= read -r line; do
[[ "$line" =~ ^#.*$ || -z "$line" ]] && continue
pattern=$(echo "$line" | sed 's/^- *//')
if [[ "$pattern" == */ ]]; then
pattern=$(echo "$pattern" | sed 's/\/$//')
if [[ "$folder" == *"$pattern"* ]]; then
skip=1
break
fi
fi
done < "$EXCLUDE_CONFIG"
if [ "$skip" -eq 0 ]; then
echo "$folder" >> "$TEMP_FOLDERS"
fi
done < "$CURRENT_FOLDERS"
mv "$TEMP_FOLDERS" "$CURRENT_FOLDERS"
fi
RELATIVE_SQL_DELETED_FOLDERS="SELECT substr(flr.path, length('$SOURCE_DIR') + 1) AS path FROM folders_last_run flr WHERE flr.path NOT IN (SELECT path FROM folders) ORDER BY path DESC;"
sqlite3 "$DB_FOLDERS" <<SQL > /dev/null 2>&1
PRAGMA synchronous=OFF;
PRAGMA journal_mode=WAL;
BEGIN TRANSACTION;
$( [ "$FIRST_RUN" -eq 0 ] && echo "DROP TABLE IF EXISTS folders_last_run; ALTER TABLE folders RENAME TO folders_last_run;" )
CREATE TABLE folders (path TEXT PRIMARY KEY);
.import --csv $CURRENT_FOLDERS folders
$( [ "$FIRST_RUN" -eq 0 ] && echo ".output $DELETED_FOLDERS" )
$( [ "$FIRST_RUN" -eq 0 ] && echo "$RELATIVE_SQL_DELETED_FOLDERS" )
INSERT OR REPLACE INTO metadata (key, value) VALUES ('last_run_time', $CURRENT_TIMESTAMP);
COMMIT;
SQL
# Delete folders
if [ "$FIRST_RUN" -eq 0 ] && [ -s "$DELETED_FOLDERS" ]; then
TEMP_LOG="$LOG_FILE.tmp"
RCLONE_FILTER=""
if [ -f "$EXCLUDE_FILTER_RCLONE" ] && [ -s "$EXCLUDE_FILTER_RCLONE" ]; then
RCLONE_FILTER="--filter-from $EXCLUDE_FILTER_RCLONE"
fi
while IFS= read -r folder; do
if ! rclone rmdir "$REMOTE_DIR${folder}" $RCLONE_FILTER --log-file="$TEMP_LOG" --log-format "date:'2006-01-02 15:04:05',level,msg" --log-level INFO --stats 0; then
write_log "Error deleting folder '$folder'. Details in $TEMP_LOG"
fi
write_log "$TEMP_LOG"
rm -f "$TEMP_LOG"
done < "$DELETED_FOLDERS"
fi
# Count operations
NEW_CHANGED_COUNT=$(wc -l < "$CHANGED_FILES")
DELETED_COUNT=$(wc -l < "$DELETED_FILES")
DELETED_FOLDER_COUNT=0
if [ "$FIRST_RUN" -eq 0 ]; then
DELETED_FOLDER_COUNT=$(wc -l < "$DELETED_FOLDERS" 2>/dev/null || echo 0)
fi
# Clean up temporary files
rm -f "$CHANGED_FILES" "$DELETED_FILES" "$CURRENT_FILES" "$DELETED_FOLDERS" "$CURRENT_FOLDERS" "$EXCLUDE_FILTER" "$EXCLUDE_FILTER_RCLONE" "$TEMP_FOLDERS"
# Write log footer
log_footer
# Final log file push
LOG_DIR=$(dirname "$LOG_FILE")
rclone sync "$LOG_FILE" "$REMOTE_DIR/${LOG_DIR#$SOURCE_DIR/}" --update --local-no-check-updated --protondrive-replace-existing-draft=true --stats 0 > /dev/null 2>&1
