mrcake

updated the script again, if anyone has a better way, let me know. Fixed some permission issues and added the ability to run it for multiple transmission instances. #!/bin/bash # This script gets torrent list from Transmission and tells rsync to copy them from array to cache drive. # It also cleans oldest modified files by rsyncing them back to array (in case of modification). # Hardlink are preserved # # By Reynald - 06 may 2020 - mailto:[email protected] # modified by quinto to work with plex v1.29 or later # modified by MrCake 21 may 2024: # - setup of multiple transmission instances # - corrected file permissions # v.0.6.0 # settings { #Rsync path STORAGE_PATH="/mnt/user0/Media/" CACHE_PATH="/mnt/cache/Media/" CACHE_MIN_FREE_SPACE_PCT="70" CACHE_MAX_FREE_SPACE_PCT="50" #Parameters LOG_MAX_SIZE=5000000 NOISY_HOUR_START=9 NOISY_HOUR_STOP=24 #Options (set to true or false) DRY_RUN=false WRITE_ON_STORAGE=false VERBOSE=1 #0=Error; 1=Info; 2=More_Info; 3=Debug } ##### No modification below this line ##### CACHE_DISK=$(df -lah "$CACHE_PATH" | awk 'FNR == 2 {print $1}' | tr -d '\n') sys_checks() { # lock if [[ -f /var/lock/smart-cache ]] then echo "Error: Script already running" exit 1 else touch /var/lock/smart-cache [[ $VERBOSE -ge 2 ]] && echo "Welcome to $0" fi # check that path are mounted if [[ ! -d $STORAGE_PATH ]] || [[ ! -d $CACHE_PATH ]]; then echo "Error: Paths are not accessibles" rm /var/lock/smart-cache exit 1 fi # cut log LOG_FILE=$(echo $0 | sed 's|\/script|\/log.txt|') echo "log= $LOG_FILE" LOG_SIZE=$(stat -c %s "$LOG_FILE") [[ $VERBOSE -ge 1 ]] && echo "Info: Log size is $LOG_SIZE" if [[ $LOG_SIZE -ge $LOG_MAX_SIZE ]] then [[ $VERBOSE -ge 1 ]] && echo "Info: Emptying log file" echo "" > "$LOG_FILE" fi [[ $VERBOSE -ge 1 ]] && echo "" } ####################### # Transfers functions # ####################### noisy_hours() { # return 0 if time in noisy hour range if [[ $(date '+%-H') -ge $NOISY_HOUR_START ]] && [[ $(date +%-H) -le $NOISY_HOUR_STOP ]] then return 0 else return 1 fi } rsync_transfer() { # get files and path SOURCE_FILE=$1 DEST_FILE=$2 SOURCE_PATH=$3 DEST_PATH=$4 RS_OPTIONS=$5 [[ $VERBOSE -ge 3 ]] && echo " --- Debug:Rsync_transfer function parameters:" [[ $VERBOSE -ge 3 ]] && echo " ---- Debug: Source file: $SOURCE_FILE" [[ $VERBOSE -ge 3 ]] && echo " ---- Debug: Dest. file: $DEST_FILE" [[ $VERBOSE -ge 3 ]] && echo " ---- Debug: Source path: $SOURCE_PATH" [[ $VERBOSE -ge 3 ]] && echo " ---- Debug: Dest. path: $DEST_PATH" [[ $VERBOSE -ge 3 ]] && echo " ---- Debug: Options : $RS_OPTIONS" # check if original file exist if [[ ! -f "${SOURCE_FILE}" ]] && [[ ! -f "${DEST_FILE}" ]] then echo " --- Error: Files:" echo " ${SOURCE_FILE}" echo " ${DEST_FILE}" echo " does not exist" return 1 elif [[ "${DEST_FILE}" = "${DEST_PATH}" ]] || [[ "${SOURCE_FILE}" = "${SOURCE_PATH}" ]] then echo " --- Error: Cannot sync root path!" return 1 elif [[ ! -f "${SOURCE_FILE}" ]] && [[ "${DEST_PATH}" = "${CACHE_PATH}" ]] && [[ -f "${DEST_FILE}" ]] then if noisy_hours then if [[ ! WRITE_ON_STORAGE ]] then [[ $VERBOSE -ge 2 ]] && echo " --- Info: File is on cache only. Inside noisy hours, sending to storage" rsync_transfer "${DEST_FILE}" "${SOURCE_FILE}" "${DEST_PATH}" "${SOURCE_PATH}" fi else [[ $VERBOSE -ge 2 ]] && echo " --- Warning: File is on cache only. Outside of noisy hours, doing nothing" fi return elif [[ -f "${DEST_FILE}" ]] && [[ "${DEST_PATH}" = "${CACHE_PATH}" ]] then [[ $VERBOSE -ge 2 ]] && echo " --- Info: File already cached" return fi # get dir SOURCE_DIR=$(dirname "${SOURCE_FILE}") DEST_DIR=$(dirname "${DEST_FILE}") if ! $DRY_RUN then # sync file sudo -u nobody mkdir -p -m u-x,go-rwx,go+u,ugo+X "${DEST_DIR}" [[ $VERBOSE -ge 1 ]] && echo " --- Info: Syncing ${SOURCE_FILE} to ${DEST_FILE}" sudo -u nobody nice -n 13 ionice -c 3 /usr/bin/rsync --bwlimit=190000 -aHq "${SOURCE_FILE}" "${DEST_FILE}" if [[ ! $? -eq 0 ]] then echo " --- Error: cannot rsync ${SOURCE_FILE}" echo " to ${DEST_FILE}" return 1 fi # remove original file if requested if [[ "${RS_OPTIONS}" = "--remove-source-files" ]] && $RSYNC_RESULT then [[ $VERBOSE -ge 2 ]] && echo " --- Info: Delete ${SOURCE_FILE}" rm "${SOURCE_FILE}" fi else echo " --- Running in DRY RUN mode : Files aren't really copied, doing nothing" fi } ################ # Transmission # ################ transmission_cache() { # get full torrent list parameters=("RPC_USER", "RPC_PASSWORD", "RPC_HOST", "RPC_MAX_CACHED_SESSIONS", "RPC_PATH") for ARGUMENT in "$@" do KEY=$(echo $ARGUMENT | cut -f1 -d=) KEY_LENGTH=${#KEY} VALUE="${ARGUMENT:$KEY_LENGTH+1}" if [[ ${parameters[@]} =~ $KEY ]] then export "$KEY"="$VALUE" else echo "value not found" fi done # get header for this Transmission RPC session RPC_LOGIN=" --user ${RPC_USER}:${RPC_PASSWORD}" SESSION_HEADER=$(curl --max-time 5 --silent --anyauth${RPC_LOGIN} ${RPC_HOST}/transmission/rpc/ | sed 's/.*<code>//g;s/<\/code>.*//g') if [[ -z $SESSION_HEADER ]]; then echo "Error: Cannot connect to transmission" return 1 fi # get torrent list TORRENT_LIST=$(curl --silent --anyauth${RPC_LOGIN} --header "${SESSION_HEADER}" "http://${RPC_HOST}/transmission/rpc" \ -d "{\"method\":\"torrent-get\",\"arguments\": {\"ids\":\"recently-active\",\"fields\":[\"id\",\"activityDate\",\"name\",\"downloadDir\",\"files\",\"status\"]}}" \ | jq '.arguments.torrents|=sort_by(-.activityDate)') NB_TORRENTS=$(echo ${TORRENT_LIST} | jq '.arguments.torrents | length') echo "-----------------------" echo "$NB_TORRENTS active(s) torrent(s):" echo "-----------------------" if [[ $NB_TORRENTS -gt $RPC_MAX_CACHED_SESSIONS ]] then NB_TORRENTS=$RPC_MAX_CACHED_SESSIONS echo "Warning: caching is limited to $RPC_MAX_CACHED_SESSIONS torrents (user setting)" fi for i in `seq $NB_TORRENTS` do # get torrent path TORRENT_PATH=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].downloadDir' | sed 's|\"||g' | sed 's|\/data|data|') TORRENT_NAME=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].name'| sed 's|\"||g') TORRENT_STATUS=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].status') NB_FILES=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].files | length') TORRENT_PROGRESS=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].files | map(.length - .bytesCompleted) | add') #echo "${TORRENT_PROGRESS}" if [[ $TORRENT_STATUS == 4 ]] then echo " - $i/$NB_TORRENTS: $TORRENT_NAME (downloading)" [[ $VERBOSE -ge 1 ]] && echo " - Info: Doing nothing" else echo " - $i/$NB_TORRENTS: ${TORRENT_NAME} (seeding)" if [[ $TORRENT_PROGRESS == 0 ]] then # for each torrent that is seeding for j in `seq $NB_FILES` do # get each file path TORRENT_FILE=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].files['$(($j - 1))'].name' | sed 's|\"||g') FILE_TO_CACHE=$(echo ${TORRENT_PATH}/${TORRENT_FILE} | sed 's|\"\"|\/|g' | sed 's|\"||g' | sed 's|data\/||g' | cut -d'/' -f3-) [[ $VERBOSE -ge 2 ]] && echo " -- Info: File $j/$NB_FILES: ${TORRENT_FILE}" STORAGE_FILE="${STORAGE_PATH}${RPC_PATH}/${FILE_TO_CACHE}" CACHE_FILE="${CACHE_PATH}${RPC_PATH}/${FILE_TO_CACHE}" # and send to rsync rsync_transfer "${STORAGE_FILE}" "${CACHE_FILE}" "${STORAGE_PATH}" "${CACHE_PATH}" done #/usr/local/sbin/newperms "${CACHE_PATH}${RPC_PATH}/${TORRENT_PATH}" else [[ $VERBOSE -ge 1 ]] && echo " - Info: Doing nothing" fi fi done [[ $VERBOSE -ge 1 ]] && echo "" } #################### # Delete old files # #################### cleanup() { # get free space a=$(df -h | grep $CACHE_DISK | awk '{ printf "%d", $5 }') b=$CACHE_MIN_FREE_SPACE_PCT echo "---------------------" echo "Cache disk usage: ${a}%" echo "---------------------" if [[ "$a" -ge "$b" ]]; then echo "$a% space used, quota is $b%, cleaning" [[ $VERBOSE -ge 1 ]] && echo "Info: Scanning files..." # get oldest accessed files find "${CACHE_PATH}" -type f -printf "%C@ %p\n" | sort -n | sed "s|`echo ${CACHE_PATH}`|%|g" | cut -d'%' -f2 | while read FILE_TO_CLEAN do # loop start: get free space again a=$(df -h | grep $CACHE_DISK | awk '{ printf "%d", $5 }') b=$CACHE_MAX_FREE_SPACE_PCT # if free space not enough if [[ "$a" -ge "$b" ]]; then [[ $VERBOSE -ge 1 ]] && echo " - Info: $a% space used, target $b%, uncaching $FILE_TO_CLEAN" STORAGE_FILE="${STORAGE_PATH}${FILE_TO_CLEAN}" CACHE_FILE="${CACHE_PATH}${FILE_TO_CLEAN}" # sync back cache to storage rsync_transfer "${CACHE_FILE}" "${STORAGE_FILE}" "${CACHE_PATH}" "${STORAGE_PATH}" "--remove-source-files" fi # loop done fi a=$(df -h | grep $CACHE_DISK | awk '{ printf "%d", $5 }') b=$CACHE_MIN_FREE_SPACE_PCT [[ $VERBOSE -ge 1 ]] && echo " - Info: $a% space used, quota is $b%, nothing to do" # prune empty directories from source dir [[ $VERBOSE -ge 2 ]] && echo " -- Info: Cleaning empty directories..." find "${CACHE_PATH}" -type d -not -path '*/\.*' -empty -prune -exec rmdir --ignore-fail-on-non-empty {} \; [[ $VERBOSE -ge 1 ]] && echo "" } sys_checks cleanup transmission_cache RPC_USER="username" RPC_PASSWORD="password" RPC_HOST="192.168.1.2:9091" RPC_MAX_CACHED_SESSIONS=20 RPC_PATH="Download" transmission_cache RPC_USER="username" RPC_PASSWORD="password" RPC_HOST="192.168.1.2:9091" RPC_MAX_CACHED_SESSIONS=20 RPC_PATH="Download" echo "" rm /var/lock/smart-cache exit 0

Thank you for the script! using the newest code I needed to do some adjustments before it worked. (also the transmission code seems to have been removed in the latest version which is what I was interested in) Also if you are running into problems, ensure Host access to custom networks is enabled in your docker settings. I was not able to reach my docker containers from custom scripts if I did not enable it. [edit] had an issue with it filling my cache drive up until it's locked in readonly. I think it's an issue with unraid when you add a cache drive to a raid 0 config that is bigger than the combined disks. I think unraid misreported the usable size so watch out if you just did the same thing (still trying to fix it, had to add another cache drive to get it out of readonly, convert the pool to btrfs single mode, and now trying to remove the cache drive I added, [edit2] fixed by doing those steps) [edit2] I am getting more permission issues recently while running sonarr v4. I noticed that this script does not preserve permissions so I am experimenting by adding the 'p' flag to rsync 1. the docker inspect error is because a variable is not defined at the top. The value should be the name of your docker image in unraid. PLEX_DOCKER="PlexMediaServer" 2. transmission reports the wrong filename to cache, I did the following adjustments - added a variable with the name of the parent folder where all my downloads end up (eg sharename is Media, parent folder is "Download") - cut the first folder name from the path and insert the RPC_PATH #variables RPC_PATH="Download" #transmission script ################ # Transmission # ################ transmission_cache() { # get full torrent list # get header for this Transmission RPC session RPC_LOGIN=" --user ${RPC_USER}:${RPC_PASSWORD}" SESSION_HEADER=$(curl --max-time 5 --silent --anyauth${RPC_LOGIN} ${RPC_HOST}/transmission/rpc/ | sed 's/.*<code>//g;s/<\/code>.*//g') if [[ -z $SESSION_HEADER ]]; then echo "Error: Cannot connect to transmission" return 1 fi # get torrent list TORRENT_LIST=$(curl --silent --anyauth${RPC_LOGIN} --header "${SESSION_HEADER}" "http://${RPC_HOST}/transmission/rpc" \ -d "{\"method\":\"torrent-get\",\"arguments\": {\"ids\":\"recently-active\",\"fields\":[\"id\",\"activityDate\",\"name\",\"downloadDir\",\"files\",\"status\"]}}" \ | jq '.arguments.torrents|=sort_by(-.activityDate)') NB_TORRENTS=$(echo ${TORRENT_LIST} | jq '.arguments.torrents | length') echo "-----------------------" echo "$NB_TORRENTS active(s) torrent(s):" echo "-----------------------" # for each torrent if [[ $NB_TORRENTS -gt $RPC_MAX_CACHED_SESSIONS ]] then NB_TORRENTS=$RPC_MAX_CACHED_SESSIONS echo "Warning: caching is limited to $RPC_MAX_CACHED_SESSIONS torrents (user setting)" fi for i in `seq $NB_TORRENTS` do # get torrent path TORRENT_PATH=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].downloadDir' | sed 's|\"||g' | sed 's|\/data|data|') TORRENT_NAME=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].name'| sed 's|\"||g') TORRENT_STATUS=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].status') NB_FILES=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].files | length') if [[ $TORRENT_STATUS == 4 ]] then echo " - $i/$NB_TORRENTS: $TORRENT_NAME (downloading)" [[ $VERBOSE -ge 1 ]] && echo " - Info: Doing nothing" else echo " - $i/$NB_TORRENTS: ${TORRENT_NAME} (seeding)" # for each file not downloading for j in `seq $NB_FILES` do # get each file path TORRENT_FILE=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].files['$(($j - 1))'].name' | sed 's|\"||g') FILE_TO_CACHE=$(echo ${TORRENT_PATH}/${TORRENT_FILE} | sed 's|\"\"|\/|g' | sed 's|\"||g' | sed 's|data\/||g' | cut -d'/' -f3-) [[ $VERBOSE -ge 2 ]] && echo " -- Info: File $j/$NB_FILES: ${TORRENT_FILE}" STORAGE_FILE="${STORAGE_PATH}${RPC_PATH}/${FILE_TO_CACHE}" CACHE_FILE="${CACHE_PATH}${RPC_PATH}/${FILE_TO_CACHE}" # and send to rsync rsync_transfer "${STORAGE_FILE}" "${CACHE_FILE}" "${STORAGE_PATH}" "${CACHE_PATH}" done fi done # [[ $NB_TORRENTS != 0 ]] && echo "" [[ $VERBOSE -ge 1 ]] && echo "" } 3. Add the transmission function call at the end of the script again. sys_checks transmission_cache plex_cache cleanup echo "" all modifications: #!/bin/bash # This script gets 19tives torrents list from Transmission, playing sessions from Plex, and tells rsync to copy them from array to cache drive. # It also cleans oldest modified files by rsyncing them back to array (in case of modification). # Hardlink are preserved # # By Reynald - 06 may 2020 - mailto:[email protected] # modified by quinto to work with plex v1.29 or later # v.0.5.19 # settings { #Transmission RPC_USER="****" RPC_PASSWORD="****" RPC_HOST="****:9091" RPC_MAX_CACHED_SESSIONS=15 RPC_PATH="Download" #Plex PLEX_TOKEN="****" PLEX_HOST="****:32400" PLEX_MAX_CACHED_SESSIONS=5 PLEX_CONTAINER_PATH="Media" PLEX_DOCKER="PlexMediaServer" #Rsync path STORAGE_PATH="/mnt/user0/Media/" CACHE_PATH="/mnt/cache/Media/" CACHE_MIN_FREE_SPACE_PCT="90" CACHE_MAX_FREE_SPACE_PCT="70" #Parameters LOG_MAX_SIZE=5000000 NOISY_HOUR_START=9 NOISY_HOUR_STOP=24 #if you're not using Youtube-dl agent for plex don't touch this line, it must only be changed when you're using yt-dl custom agent YOUTUBE_DL_AGENT="com.plexapp.agents.youtube-dl" #Options (set to true or false) DRY_RUN=false WRITE_ON_STORAGE=false #number of episodes to cache PLEX_CACHE_NB_EPISODES=6 PLEX_CACHE_SEASON_TILL_END=false VERBOSE=1 #0=Error; 1=Info; 2=More_Info; 3=Debug } ##### No modification below this line ##### CACHE_DISK=$(df -lah "$CACHE_PATH" | awk 'FNR == 2 {print $1}' | tr -d '\n') sys_checks() { # check if plex docker is running plex_running=`docker inspect -f '{{.State.Running}}' $PLEX_DOCKER` if [[ ! $plex_running ]] then echo "Error: Plex docker is not running" exit 1 fi # lock if [[ -f /var/lock/smart-cache_plex_transmission ]] then echo "Error: Script already running" exit 1 else touch /var/lock/smart-cache_plex_transmission [[ $VERBOSE -ge 2 ]] && echo "Welcome to $0" fi # check that path are mounted if [[ ! -d $STORAGE_PATH ]] || [[ ! -d $CACHE_PATH ]]; then echo "Error: Paths are not accessibles" rm /var/lock/smart-cache_plex_transmission exit 1 fi # cut log LOG_FILE=$(echo $0 | sed 's|\/script|\/log.txt|') echo "log= $LOG_FILE" LOG_SIZE=$(stat -c %s "$LOG_FILE") [[ $VERBOSE -ge 1 ]] && echo "Info: Log size is $LOG_SIZE" if [[ $LOG_SIZE -ge $LOG_MAX_SIZE ]] then [[ $VERBOSE -ge 1 ]] && echo "Info: Emptying log file" echo "" > "$LOG_FILE" fi [[ $VERBOSE -ge 1 ]] && echo "" } ####################### # Transfers functions # ####################### noisy_hours() { # return 0 if time in noisy hour range if [[ $(date '+%-H') -ge $NOISY_HOUR_START ]] && [[ $(date +%-H) -le $NOISY_HOUR_STOP ]] then return 0 else return 1 fi } rsync_transfer() { # get files and path SOURCE_FILE=$1 DEST_FILE=$2 SOURCE_PATH=$3 DEST_PATH=$4 RS_OPTIONS=$5 [[ $VERBOSE -ge 3 ]] && echo " --- Debug:Rsync_transfer function parameters:" [[ $VERBOSE -ge 3 ]] && echo " ---- Debug: Source file: $SOURCE_FILE" [[ $VERBOSE -ge 3 ]] && echo " ---- Debug: Dest. file: $DEST_FILE" [[ $VERBOSE -ge 3 ]] && echo " ---- Debug: Source path: $SOURCE_PATH" [[ $VERBOSE -ge 3 ]] && echo " ---- Debug: Dest. path: $DEST_PATH" [[ $VERBOSE -ge 3 ]] && echo " ---- Debug: Options : $RS_OPTIONS" # check if original file exist if [[ ! -f "${SOURCE_FILE}" ]] && [[ ! -f "${DEST_FILE}" ]] then echo " --- Error: Files:" echo " ${SOURCE_FILE}" echo " ${DEST_FILE}" echo " does not exist" return 1 elif [[ "${DEST_FILE}" = "${DEST_PATH}" ]] || [[ "${SOURCE_FILE}" = "${SOURCE_PATH}" ]] then echo " --- Error: Cannot sync root path!" return 1 elif [[ ! -f "${SOURCE_FILE}" ]] && [[ "${DEST_PATH}" = "${CACHE_PATH}" ]] && [[ -f "${DEST_FILE}" ]] then if noisy_hours then if [[ ! WRITE_ON_STORAGE ]] then [[ $VERBOSE -ge 2 ]] && echo " --- Info: File is on cache only. Inside noisy hours, sending to storage" rsync_transfer "${DEST_FILE}" "${SOURCE_FILE}" "${DEST_PATH}" "${SOURCE_PATH}" fi else [[ $VERBOSE -ge 2 ]] && echo " --- Warning: File is on cache only. Outside of noisy hours, doing nothing" fi return elif [[ -f "${DEST_FILE}" ]] && [[ "${DEST_PATH}" = "${CACHE_PATH}" ]] then [[ $VERBOSE -ge 2 ]] && echo " --- Info: File already cached" return fi # get dir SOURCE_DIR=$(dirname "${SOURCE_FILE}") DEST_DIR=$(dirname "${DEST_FILE}") if ! $DRY_RUN then # sync file mkdir -p "${DEST_DIR}" [[ $VERBOSE -ge 1 ]] && echo " --- Info: Syncing ${SOURCE_FILE} to ${DEST_FILE}" nice -n 13 ionice -c 3 /usr/bin/rsync --bwlimit=190000 -aHqp "${SOURCE_FILE}" "${DEST_FILE}" if [[ ! $? -eq 0 ]] then echo " --- Error: cannot rsync ${SOURCE_FILE}" echo " to ${DEST_FILE}" return 1 fi # remove original file if requested if [[ "${RS_OPTIONS}" = "--remove-source-files" ]] && $RSYNC_RESULT then [[ $VERBOSE -ge 2 ]] && echo " --- Info: Delete ${SOURCE_FILE}" rm "${SOURCE_FILE}" fi else echo " --- Running in DRY RUN mode : Files aren't really copied, doing nothing" fi } ################ # Transmission # ################ transmission_cache() { # get full torrent list # get header for this Transmission RPC session RPC_LOGIN=" --user ${RPC_USER}:${RPC_PASSWORD}" SESSION_HEADER=$(curl --max-time 5 --silent --anyauth${RPC_LOGIN} ${RPC_HOST}/transmission/rpc/ | sed 's/.*<code>//g;s/<\/code>.*//g') if [[ -z $SESSION_HEADER ]]; then echo "Error: Cannot connect to transmission" return 1 fi # get torrent list TORRENT_LIST=$(curl --silent --anyauth${RPC_LOGIN} --header "${SESSION_HEADER}" "http://${RPC_HOST}/transmission/rpc" \ -d "{\"method\":\"torrent-get\",\"arguments\": {\"ids\":\"recently-active\",\"fields\":[\"id\",\"activityDate\",\"name\",\"downloadDir\",\"files\",\"status\"]}}" \ | jq '.arguments.torrents|=sort_by(-.activityDate)') NB_TORRENTS=$(echo ${TORRENT_LIST} | jq '.arguments.torrents | length') echo "-----------------------" echo "$NB_TORRENTS active(s) torrent(s):" echo "-----------------------" # for each torrent if [[ $NB_TORRENTS -gt $RPC_MAX_CACHED_SESSIONS ]] then NB_TORRENTS=$RPC_MAX_CACHED_SESSIONS echo "Warning: caching is limited to $RPC_MAX_CACHED_SESSIONS torrents (user setting)" fi for i in `seq $NB_TORRENTS` do # get torrent path TORRENT_PATH=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].downloadDir' | sed 's|\"||g' | sed 's|\/data|data|') TORRENT_NAME=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].name'| sed 's|\"||g') TORRENT_STATUS=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].status') NB_FILES=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].files | length') if [[ $TORRENT_STATUS == 4 ]] then echo " - $i/$NB_TORRENTS: $TORRENT_NAME (downloading)" [[ $VERBOSE -ge 1 ]] && echo " - Info: Doing nothing" else echo " - $i/$NB_TORRENTS: ${TORRENT_NAME} (seeding)" # for each file not downloading for j in `seq $NB_FILES` do # get each file path TORRENT_FILE=$(echo ${TORRENT_LIST} | jq '.arguments.torrents['$(($i - 1))'].files['$(($j - 1))'].name' | sed 's|\"||g') FILE_TO_CACHE=$(echo ${TORRENT_PATH}/${TORRENT_FILE} | sed 's|\"\"|\/|g' | sed 's|\"||g' | sed 's|data\/||g' | cut -d'/' -f3-) [[ $VERBOSE -ge 2 ]] && echo " -- Info: File $j/$NB_FILES: ${TORRENT_FILE}" STORAGE_FILE="${STORAGE_PATH}${RPC_PATH}/${FILE_TO_CACHE}" CACHE_FILE="${CACHE_PATH}${RPC_PATH}/${FILE_TO_CACHE}" # and send to rsync rsync_transfer "${STORAGE_FILE}" "${CACHE_FILE}" "${STORAGE_PATH}" "${CACHE_PATH}" done fi done # [[ $NB_TORRENTS != 0 ]] && echo "" [[ $VERBOSE -ge 1 ]] && echo "" } ######## # Plex # ######## plex_cache() { # get Plex sessions STATUS_SESSIONS=$(nice -n 13 ionice -c 3 curl --limit-rate 200k --silent http://${PLEX_HOST}/status/sessions -H "X-Plex-Token: $PLEX_TOKEN") if [[ -z $STATUS_SESSIONS ]]; then echo "Error: Cannot connect to plex" return 1 fi NB_SESSIONS=$(echo $STATUS_SESSIONS | xmllint --xpath 'string(//MediaContainer/@size)' -) echo "----------------------------" echo "$NB_SESSIONS active(s) plex session(s):" echo "----------------------------" # for each session if [[ $NB_SESSIONS -gt $PLEX_MAX_CACHED_SESSIONS ]] then NB_SESSIONS=$PLEX_MAX_CACHED_SESSIONS echo "Warning: Caching is limited to $PLEX_MAX_CACHED_SESSIONS plex sessions (user setting)" fi for i in `seq $NB_SESSIONS` do # get title ID=$(echo $STATUS_SESSIONS | xmllint --xpath 'string(//MediaContainer/Video['$i']/@ratingKey)' -) TYPE=$(echo $STATUS_SESSIONS | xmllint --xpath 'string(//MediaContainer/Video['$i']/@type)' -) MKEY=$(echo $STATUS_SESSIONS | xmllint --xpath 'string(//MediaContainer/Video['$i']/@key)' -) # eventually get serie info if [[ $TYPE = "episode" ]] then TYPE="Serie" GRANDPARENTTITLE=$(echo $STATUS_SESSIONS | xmllint --xpath 'string(//MediaContainer/Video['$i']/@grandparentTitle)' -) SEASON=$(echo $STATUS_SESSIONS | xmllint --xpath 'string(//MediaContainer/Video['$i']/@parentIndex)' -) TITLE=$(echo $STATUS_SESSIONS | xmllint --xpath 'string(//MediaContainer/Video['$i']/@title)' -) EPISODE=$(echo $STATUS_SESSIONS | xmllint --xpath 'string(//MediaContainer/Video['$i']/@index)' -) PARENT_ID=$(echo $STATUS_SESSIONS | xmllint --xpath 'string(//MediaContainer/Video['$i']/@parentRatingKey)' -) PARENT_SESS=$(nice -n 13 ionice -c 3 curl --limit-rate 200k --silent http://${PLEX_HOST}/library/metadata/$PARENT_ID/children) PARENT_NB_EPISODES=$(echo $PARENT_SESS | xmllint --xpath 'string(//MediaContainer/@size)' -) PARENT_START_EPISODE=$(echo $PARENT_SESS | xmllint --xpath 'string(//MediaContainer/Video[1]/@index)' -) SECTION_ID=$(nice -n 13 ionice -c 3 curl --limit-rate 200k --silent http://${PLEX_HOST}/library/metadata/$PARENT_ID | xmllint --xpath 'string(//MediaContainer/@librarySectionID)' -) SECTION_AGENT=$(nice -n 13 ionice -c 3 curl --limit-rate 200k --silent http://${PLEX_HOST}/library/sections?X-Plex-Token= | xmllint --xpath 'string(//MediaContainer/Directory[@key='$SECTION_ID']/@agent)' -) if [[ ${SECTION_AGENT} = "com.plexapp.agents.youtube-dl" ]]; then TYPE="Youtube"; fi TITLE="$TYPE: ${GRANDPARENTTITLE} Season ${SEASON} - Episode ${EPISODE}/${PARENT_NB_EPISODES}: $TITLE" # update nb file to cache START_FILE="$EPISODE" if [[ ${SECTION_AGENT} == ${YOUTUBE_DL_AGENT} ]] then NB_FILES=$(( $EPISODE )) else if [[ $PLEX_CACHE_NB_EPISODES -gt 0 ]]; then NB_FILES=$(( ($EPISODE + $PLEX_CACHE_NB_EPISODES) < $PARENT_NB_EPISODES ? ($EPISODE + $PLEX_CACHE_NB_EPISODES) : $PARENT_NB_EPISODES )); fi $PLEX_CACHE_SEASON_TILL_END && NB_FILES=$(( $PARENT_NB_EPISODES )) fi elif [[ $TYPE = "movie" ]] then NB_SESS=$(nice -n 13 ionice -c 3 curl --limit-rate 200k --silent http://${PLEX_HOST}${MKEY}?checkFiles=1?includeChildren=1?X-Plex-Token=) NB_MEDIAS=$(echo $NB_SESS | xmllint --xpath 'count(//Media)' -) NB_FILES=$(echo $NB_SESS | xmllint --xpath 'count(//Part)' -) TYPE="Movie" TITLE="$(echo $STATUS_SESSIONS | xmllint --xpath 'string(//MediaContainer/Video['$i']/@title)' -)" TITLE="$TYPE: $TITLE" START_FILE=1 else TYPE="Audio" TITLE="track caching not implemented" TITLE="$TYPE: $TITLE" START_FILE=0 NB_FILES=0 fi echo " - $i / $NB_SESSIONS: $TITLE" PLEX_FILE_SESS_MOVIE=$(nice -n 13 ionice -c 3 curl --limit-rate 200k --silent http://${PLEX_HOST}/library/metadata/$PARENT_ID/children) PLEX_FILE_SESS_TVSHOW=$(nice -n 13 ionice -c 3 curl --limit-rate 200k --silent http://${PLEX_HOST}/library/metadata/$ID) for j in `seq $START_FILE $NB_FILES` do # get file path if [[ $TYPE = "Audio" ]] then [[ $VERBOSE -ge 2 ]] && echo " -- Info: Skipping" else if [[ $TYPE != "Movie" ]] then if [[ $TYPE = "Youtube" ]] then # media is a youtube episode PLEX_FILE=$(echo "${PLEX_FILE_SESS_MOVIE}" | xmllint --xpath 'string(//MediaContainer/Video['$(($PARENT_START_EPISODE - $j + 1))']/Media/Part/@file)' -) else # media is a tv show PLEX_FILE=$(echo "${PLEX_FILE_SESS_MOVIE}" | xmllint --xpath 'string(//MediaContainer/Video['$(($j - $PARENT_START_EPISODE + 1))']/Media/Part/@file)' -) fi m=$PARENT_NB_EPISODES else # media is a movie if [[ $NB_FILES -gt 1 ]] then if [[ $NB_FILES -gt $NB_MEDIAS ]] then # multiple parts movie (part1, part2, etc) PLEX_FILE=$(echo "${PLEX_FILE_SESS_TVSHOW}" | xmllint --xpath 'string(//MediaContainer/Video/Media/Part['$j']/@file)' -) else # multiple medias movie (4k, 1080p, 720p, etc) PLEX_FILE=$(echo "${PLEX_FILE_SESS_TVSHOW}" | xmllint --xpath 'string(//MediaContainer/Video/Media['$j']/Part/@file)' -) fi else # movie in single part / format PLEX_FILE=$(echo "${PLEX_FILE_SESS_TVSHOW}" | xmllint --xpath 'string(//MediaContainer/Video/Media/Part/@file)' -) fi m=$NB_FILES fi FILE_TO_CACHE="$(sed -e 's|\"\"|\/|g' -e 's/%/\%/g' <<<${PLEX_FILE} | sed 's|\"||g' | sed 's|\/'${PLEX_CONTAINER_PATH}'\/||')" [[ $VERBOSE -ge 2 ]] && echo " -- Info: Streaming File $j/$m: ${FILE_TO_CACHE}" STORAGE_FILE="${STORAGE_PATH}${FILE_TO_CACHE}" CACHE_FILE="${CACHE_PATH}${FILE_TO_CACHE}" # and send to rsync rsync_transfer "${STORAGE_FILE}" "${CACHE_FILE}" "${STORAGE_PATH}" "${CACHE_PATH}" fi sleep .8 done sleep .8 done sleep .8 # [[ $NB_SESSIONS != 0 ]] && echo "" [[ $VERBOSE -ge 1 ]] && echo "" } #################### # Delete old files # #################### cleanup() { # get free space a=$(df -h | grep $CACHE_DISK | awk '{ printf "%d", $5 }') b=$CACHE_MIN_FREE_SPACE_PCT echo "---------------------" echo "Cache disk usage: ${a}%" echo "---------------------" if [[ "$a" -ge "$b" ]]; then echo "$a% space used, quota is $b%, cleaning" [[ $VERBOSE -ge 1 ]] && echo "Info: Scanning files..." # get oldest accessed files find "${CACHE_PATH}" -type f -printf "%C@ %p\n" | sort -n | sed "s|`echo ${CACHE_PATH}`|%|g" | cut -d'%' -f2 | while read FILE_TO_CLEAN do # loop start: get free space again a=$(df -h | grep $CACHE_DISK | awk '{ printf "%d", $5 }') b=$CACHE_MAX_FREE_SPACE_PCT # if free space not enough if [[ "$a" -ge "$b" ]]; then [[ $VERBOSE -ge 1 ]] && echo " - Info: $a% space used, target $b%, uncaching $FILE_TO_CLEAN" STORAGE_FILE="${STORAGE_PATH}${FILE_TO_CLEAN}" CACHE_FILE="${CACHE_PATH}${FILE_TO_CLEAN}" # sync back cache to storage rsync_transfer "${CACHE_FILE}" "${STORAGE_FILE}" "${CACHE_PATH}" "${STORAGE_PATH}" "--remove-source-files" fi # loop done fi a=$(df -h | grep $CACHE_DISK | awk '{ printf "%d", $5 }') b=$CACHE_MIN_FREE_SPACE_PCT [[ $VERBOSE -ge 1 ]] && echo " - Info: $a% space used, quota is $b%, nothing to do" # prune empty directories from source dir [[ $VERBOSE -ge 2 ]] && echo " -- Info: Cleaning empty directories..." find "${CACHE_PATH}" -type d -not -path '*/\.*' -empty -prune -exec rmdir --ignore-fail-on-non-empty {} \; [[ $VERBOSE -ge 1 ]] && echo "" } sys_checks transmission_cache plex_cache cleanup echo "" rm /var/lock/smart-cache_plex_transmission exit 0

It is working now. What i did now is disable the adblock on dd-wrt, after that steam had some trouble connecting and I left the pc on while I was asleep. In the morning the downloads started and I am now observing that the share is growing in size, access log is showing the misses. and now tried to reinstall the game and that hit the cache. Thanks for the help

@mlebjerg I had an adblocker running on dd wrt, disabled that (did not help) Steam is failing its connections now, so that is better i think? No vpns active or used on my network. Here is the docker log: root@Tower:~# docker logs SteamCacheBundle Running bootstrap.sh... nginx: the configuration file /etc/nginx/nginx.conf syntax is ok nginx: configuration file /etc/nginx/nginx.conf test is successful Bind9 { Enabling cache for Blizzard 2018-08-20 14:31:57 Failed to set file handle limit: Operation not permitted Enabling cache for Frontier Enabling cache for Origin Enabling cache for Riot Enabling cache for Steam Enabling cache for Uplay Enabling cache for Windows Enabling custom DNS forwarder bootstrap finished. checking Bind9 config Running Bind9 Running bootstrap.sh... nginx: the configuration file /etc/nginx/nginx.conf syntax is ok nginx: configuration file /etc/nginx/nginx.conf test is successful Bind9 { Enabling cache for Blizzard 2018-08-20 15:24:04 Failed to set file handle limit: Operation not permitted Enabling cache for Frontier Enabling cache for Origin Enabling cache for Riot Enabling cache for Steam Enabling cache for Uplay Enabling cache for Windows Enabling custom DNS forwarder bootstrap finished. checking Bind9 config Running Bind9 Running bootstrap.sh... nginx: the configuration file /etc/nginx/nginx.conf syntax is ok nginx: configuration file /etc/nginx/nginx.conf test is successful Bind9 { Enabling cache for Blizzard 2018-08-21 08:39:09 Failed to set file handle limit: Operation not permitted Enabling cache for Frontier Enabling cache for Origin Enabling cache for Riot Enabling cache for Steam Enabling cache for Uplay Enabling cache for Windows Enabling custom DNS forwarder bootstrap finished. checking Bind9 config Running Bind9 Running bootstrap.sh... nginx: the configuration file /etc/nginx/nginx.conf syntax is ok nginx: configuration file /etc/nginx/nginx.conf test is successful Bind9 { Enabling cache for Blizzard 2018-08-21 13:31:56 Failed to set file handle limit: Operation not permitted Enabling cache for Frontier Enabling cache for Origin Enabling cache for Riot Enabling cache for Steam Enabling cache for Uplay Enabling cache for Windows Enabling custom DNS forwarder bootstrap finished. checking Bind9 config Running Bind9 Running bootstrap.sh... nginx: the configuration file /etc/nginx/nginx.conf syntax is ok nginx: configuration file /etc/nginx/nginx.conf test is successful Bind9 { Enabling cache for Blizzard 2018-08-21 15:31:52 Failed to set file handle limit: Operation not permitted Enabling cache for Frontier Enabling cache for Origin Enabling cache for Riot Enabling cache for Steam Enabling cache for Uplay Enabling cache for Windows Enabling custom DNS forwarder bootstrap finished. checking Bind9 config Running Bind9 Running bootstrap.sh... nginx: the configuration file /etc/nginx/nginx.conf syntax is ok nginx: configuration file /etc/nginx/nginx.conf test is successful Bind9 { Enabling cache for Blizzard 2018-08-21 15:33:55 Failed to set file handle limit: Operation not permitted Enabling cache for Frontier Enabling cache for Origin Enabling cache for Riot Enabling cache for Steam Enabling cache for Uplay Enabling cache for Windows Enabling custom DNS forwarder bootstrap finished. checking Bind9 config Running Bind9

Hi @nuhll, @mlebjerg Thanks for the tips! I think i have given it half an hour or something, i'll leave it on while i'm sleeping to see if it changes something. I have tried to be as thorough as possible when testing it: windows: 1. check the hosts file, nothing strange there 2. set dns ip to 192.168.1.90 3. flushdns 4. clear download cache on steam 5. clear the steam content log unraid: 1. compute steamcache share: 0B 2. make sure that steamcache share is not using the cache drive 3. clear the logfiles 4. open 2 command windows that tail the /data/logs/access.log and error.log windows: 1. check dns in powershell --> gives me 192.168.1.90 2. start a download on steam unraid: 1. check the steamcache share size: 0 B windows: 1. uninstall game 2. reinstall game 3. try another game dd-wrt router: 1. check if there is traffic going outside of the LAN: yes unraid: 1. observe the tails that i had started on error.log and access.log. they are empty 2. double check on the docker container, logs are empty I have added the steam content log, maybe that can help. Are there other ways I can test if it is working properly? Thanks for the help! this is something i found in the steam connection log: [2018-08-21 23:51:35] [0,0] Existing WebSocket ping data (cm-03-ams1.cm.steampowered.com:27020) results: 52.0000ms + load 66.0000 (757 seconds ago) [2018-08-21 23:51:35] [0,0] Existing WebSocket ping data (cm-04-ams1.cm.steampowered.com:27020) results: 52.0000ms + load 63.0000 (757 seconds ago) [2018-08-21 23:51:35] [0,0] Existing WebSocket ping data (CM02-LUX.cm.steampowered.com:27021) results: 50.0000ms + load 61.0000 (719 seconds ago) [2018-08-21 23:51:35] [0,0] Existing WebSocket ping data (CM01-FRA.cm.steampowered.com:27021) results: 49.0000ms + load 63.0000 (719 seconds ago) [2018-08-21 23:51:35] [0,0] Existing WebSocket ping data (CM02-STO.cm.steampowered.com:27021) results: 49.0000ms + load 61.0000 (751 seconds ago) [2018-08-21 23:51:35] [0,0] Existing WebSocket ping data (cm-02-ams1.cm.steampowered.com:443) results: 39.0000ms + load 62.0000 (756 seconds ago) [2018-08-21 23:51:35] [0,0] Existing WebSocket ping data (cm-02-ams1.cm.steampowered.com:27021) results: 39.0000ms + load 63.0000 (756 seconds ago) [2018-08-21 23:51:35] [0,0] Existing WebSocket ping data (cm-02-ams1.cm.steampowered.com:27020) results: 32.0000ms + load 63.0000 (756 seconds ago) [2018-08-21 23:51:35] [0,0] PingWebSocketCM() (cm-03-ams1.cm.steampowered.com:443) starting... [2018-08-21 23:51:35] [0,0] PingWebSocketCM() (CM03-STO.cm.steampowered.com:443) starting... [2018-08-21 23:51:35] [0,0] PingWebSocketCM() (CM02-FRA.cm.steampowered.com:27021) starting... [2018-08-21 23:51:35] [0,0] PingWebSocketCM() (CM06-IAD.cm.steampowered.com:27021) starting... [2018-08-21 23:51:36] [0,0] PingWebSocketCM() (cm-03-ams1.cm.steampowered.com:443) failed talking to cm (timeout/neterror - OK) [2018-08-21 23:51:36] [0,0] PingWebSocketCM() (CM06-IAD.cm.steampowered.com:27021) failed talking to cm (timeout/neterror - Invalid) [2018-08-21 23:51:36] [0,0] PingWebSocketCM() (CM02-FRA.cm.steampowered.com:27021) failed talking to cm (timeout/neterror - Invalid) [2018-08-21 23:51:36] [0,0] PingWebSocketCM() (CM03-STO.cm.steampowered.com:443) failed talking to cm (timeout/neterror - Invalid) [2018-08-21 23:51:36] [1,3] Connect() starting connection (eNetQOSLevelHigh, cm-02-ams1.cm.steampowered.com:27020, WebSocket) content_log.txt

Hi I am having some issues getting the steam cache bundle going. Steps to get where I am at now: - set unraid ports to something else than 80 and 443 - install steamcachebundle from community applications - cleared download cache in steam - in windows manually set the dns ip to '192.168.1.90' (no secondary server) - in windows disable ipv6 config: Unraid version: Version 6.5.3 2018-06-12 Powershell output: PS C:\WINDOWS\system32> Resolve-DnsName steamcache.cs.steampowered.com -server 192.168.1.90 Name Type TTL Section IPAddress ---- ---- --- ------- --------- steamcache.cs.steampowered.com A 600 Answer 192.168.1.90 Name : cs.steampowered.com QueryType : NS TTL : 600 Section : Authority NameHost : ns1.cs.steampowered.com ns1.cs.steampowered.com A 600 Additional 192.168.1.90 I do not get any hits in the /data/logs/access.log. I appreciate any help