This is something I liked from QNAP with QTiering.  In UNRAID it would be fundamentally different, but with the plethora of storage options out there, it would be nice to allow users to both use more than one cache pool within a share, and then allow the user to specify tiering options.  For instance, I would like the ability to tier1 pool of high-speed write nvme; tier2 pool of high-capacity nvme; tier3 array.  My use case would be all incoming data hits the tier1, rolls off slowly to the tier2 after 24 hours and then depending on how much it is accessed, either stays until it becomes barely touched or is moved down to the array.

```

#!/bin/bash
################################################################################
# tiered_mover.sh
# Simplified script with robust demotion logic (media_cache → cache → array).
################################################################################

#################################
# CONFIGURATION
#################################

# Paths
MEDIA_CACHE_PATH="/mnt/media_cache/media"
CACHE_PATH="/mnt/cache/media"
ARRAY_PATH="/mnt/user0/media"
APPDATA_PATH="/mnt/cache/appdata"

# Usage thresholds (in %)
MEDIA_CACHE_MAX=75
CACHE_MAX=70

# Skipped file timeout (in seconds; 86400 = 24 hours)
SKIP_TIMEOUT=86400

# Space reservation for /appdata
APPDATA_BUFFER_PERCENT=10

# Logging
LOG_DIR="/var/log/tiered_mover"
LOG_FILE="$LOG_DIR/tiered_mover.log"
MAX_LOG_AGE=30

# Rsync flags
RSYNC_FLAGS="-aH --remove-source-files"

# Dry-run toggle (1 = simulate moves, 0 = actually move files)
DRY_RUN=1

#################################
# TRACKING VARIABLES
#################################
FILES_MOVED=0
FILES_SKIPPED=0
FILES_ERRORS=0
SIZE_MOVED=0  # Total size moved in bytes

#################################
# HELPER FUNCTIONS
#################################

log() {
  local msg="$1"
  echo "[$(date +'%Y-%m-%d %H:%M:%S')] $msg" | tee -a "$LOG_FILE"
}

rotate_logs() {
  find "$LOG_DIR" -type f -name "*.log*" -mtime +$MAX_LOG_AGE -exec rm -f {} \;
}

calculate_usage() {
  sync && sleep 1  # Flush filesystem buffers and wait
  df --output=pcent "$1" 2>/dev/null | tail -n 1 | tr -dc '0-9'
}

adjust_cache_max() {
  local cache_total cache_free buffer
  cache_total=$(df --output=size "/mnt/cache" | tail -1 | tr -d ' ')
  cache_free=$(df --output=avail "/mnt/cache" | tail -1 | tr -d ' ')
  buffer=$((cache_free * APPDATA_BUFFER_PERCENT / 100))

  local used_and_reserved=$((cache_total - cache_free + buffer))
  CACHE_MAX=$((100 - (used_and_reserved * 100 / cache_total)))

  log "Adjusted CACHE_MAX to $CACHE_MAX% (reserved buffer=$((buffer / 1024)) MB)."
}

#################################
# SKIP FILE HANDLING
#################################

should_skip_file() {
  local file="$1"
  local last_skipped
  last_skipped=$(getfattr --only-values -n user.last_skipped "$file" 2>/dev/null)

  if [ -z "$last_skipped" ]; then
    return 1  # No attribute, do not skip
  fi

  local current_time=$(date +%s)
  local time_diff=$((current_time - last_skipped))

  if (( time_diff < SKIP_TIMEOUT )); then
    return 0  # Skip this file
  fi

  return 1  # Do not skip
}

mark_file_as_skipped() {
  local file="$1"
  local current_time=$(date +%s)
  setfattr -n user.last_skipped -v "$current_time" "$file" 2>/dev/null
}

#################################
# FILE MOVEMENT FUNCTIONS
#################################

process_file_action() {
  local action="$1" source="$2" destination="$3"
  mkdir -p "$(dirname "$destination")"

  local file_size
  file_size=$(stat --format="%s" "$source")

  local local_flags="$RSYNC_FLAGS"
  [ "$DRY_RUN" -eq 1 ] && local_flags+=" --dry-run"

  log "$action (rsync) $source -> $destination"
  rsync $local_flags "$source" "$destination"
  local status=$?

  if [ "$DRY_RUN" -eq 0 ]; then
    chown nobody:users "$destination"
    chmod 0666 "$destination"
  fi

  if [ $status -eq 0 ]; then
    log "$action OK: $source (size: $((file_size / 1024 / 1024)) MB)"
    FILES_MOVED=$((FILES_MOVED + 1))
    SIZE_MOVED=$((SIZE_MOVED + file_size))
  else
    log "ERROR $action $source (rsync status=$status)"
    FILES_ERRORS=$((FILES_ERRORS + 1))
    file_size=0
  fi

  file_size_return="$file_size"
}

#################################
# (A) Demote from Media Cache -> Cache
#################################

demote_from_media_cache() {
  local usage_mc=$(calculate_usage "$MEDIA_CACHE_PATH")
  local usage_cache=$(calculate_usage "$CACHE_PATH")

  if (( usage_mc <= MEDIA_CACHE_MAX )); then
    log "media_cache usage ${usage_mc}% <= ${MEDIA_CACHE_MAX}%. No demotion needed."
    return
  fi

  log "Demoting from media_cache -> cache (usage=${usage_mc}% > ${MEDIA_CACHE_MAX}%)."

  # Calculate the total space to free (in KB)
  local media_cache_total_space
  media_cache_total_space=$(df --output=size "$MEDIA_CACHE_PATH" | tail -1 | tr -d ' ')
  local space_to_free=$((media_cache_total_space * (usage_mc - MEDIA_CACHE_MAX) / 100))
  local space_freed=0
  local files_processed=0
  local files_skipped=0
  local simulated_usage=$((usage_mc * media_cache_total_space / 100))  # Simulate usage in test mode

  log "Total space to free to meet threshold: $((space_to_free / 1024)) MB"

  exec 3< <(find "$MEDIA_CACHE_PATH" -type f -mtime +1 \
    \( -not -name '*.part' -a -not -name '*.!qB' -a -not -name '*.tmp' \))

  while read -r file_path <&3; do
    [ -z "$file_path" ] && continue

    # Reset file_size to avoid carry-over values
    local file_size=0

    # Check if the file should be skipped
    if should_skip_file "$file_path"; then
      files_skipped=$((files_skipped + 1))
      log "Skipping recently marked file: $file_path"
      continue
    fi

    # Check if the file is in use
    if lsof "$file_path" 2>/dev/null | grep -Eq '(Plex|Tdarr|qbittorrent|transmission)'; then
      files_skipped=$((files_skipped + 1))
      log "Skipping in-use file: $file_path"
      mark_file_as_skipped "$file_path"
      continue
    fi

    # Get the file size in MB
    file_size=$(stat --format="%s" "$file_path" 2>/dev/null || echo 0)
    file_size=$((file_size / 1024 / 1024))  # Convert to MB
    log "File size retrieved: ${file_size} MB for $file_path"

    [ "$file_size" -eq 0 ] && continue

    # Simulate moving the file in test mode
    if [ "$DRY_RUN" -eq 1 ]; then
      space_freed=$((space_freed + file_size))
      simulated_usage=$((simulated_usage - file_size * 1024))  # Simulate reduced usage in KB
      log "DRY RUN: Simulating demotion of $file_path (size: ${file_size} MB)"
    else
      # Move the file in live mode
      local rel_path="${file_path#$MEDIA_CACHE_PATH/}"
      local dest_path="$CACHE_PATH/$rel_path"

      process_file_action "Demoting" "$file_path" "$dest_path"
      if [ $? -eq 0 ]; then
        space_freed=$((space_freed + file_size))
        files_processed=$((files_processed + 1))
        log "Added file size: ${file_size} MB to space_freed. Total space_freed: ${space_freed} MB"
      fi
    fi

    # Update simulated space left to free
    local space_remaining=$((space_to_free - space_freed))
    log "Space freed: ${space_freed} MB. Space left to free (simulated): $((space_remaining / 1024)) MB"

    # Stop if enough space has been freed
    if (( space_freed >= space_to_free )); then
      log "Enough space freed. media_cache usage reduced to acceptable level."
      break
    fi
  done || true  # Suppress broken pipe errors

  exec 3<&-  # Close the file descriptor

  # Final Summary
  log "Demotion Summary: Processed $files_processed files, skipped $files_skipped, freed ${space_freed} MB."

  # Simulate final usage in test mode
  if [ "$DRY_RUN" -eq 1 ]; then
    local final_usage=$((simulated_usage * 100 / media_cache_total_space))
    log "DRY RUN: Simulated media_cache usage after demotion: ${final_usage}%"
  else
    usage_mc=$(calculate_usage "$MEDIA_CACHE_PATH")
    if (( usage_mc > MEDIA_CACHE_MAX )); then
      log "WARNING: Unable to reduce media_cache usage to ${MEDIA_CACHE_MAX}%. Remaining usage: ${usage_mc}%."
    fi
  fi
}

#################################
# MAIN EXECUTION
#################################

mkdir -p "$LOG_DIR"
rotate_logs

log "===== Starting 3-Tier Mover Script ====="
adjust_cache_max
demote_from_media_cache
log "===== Script Completed ====="

```

 

made a script with chatgpt to attempt to do this manually.  it looks like FUSE can still see things in directories that aren't in the primary/secondary listing, so no issues there.  goal here is to make mover a less burdensome process and only move down files to the array that are truly cold.  would like to be able to move them up if they are commonly accessed, but baby steps.

Edited January 13, 20251 yr by user2579