From dc431197265f64ef442fb5a7565de16f65330356 Mon Sep 17 00:00:00 2001 From: Jackmaninov Date: Sat, 14 Oct 2023 21:00:20 +0300 Subject: [PATCH 1/2] feat: faster filtering of previously searched Album IDs using GNU comm --- lidarr/Audio.service.bash | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lidarr/Audio.service.bash b/lidarr/Audio.service.bash index de159bb..557ba7f 100644 --- a/lidarr/Audio.service.bash +++ b/lidarr/Audio.service.bash @@ -996,13 +996,16 @@ GetMissingCutOffList () { dlnumber="$lidarrMissingTotalRecords" fi log "$page :: missing :: Downloading page $page... ($offset - $dlnumber of $lidarrMissingTotalRecords Results)" - lidarrRecords=$(wget --timeout=0 -q -O - "$arrUrl/api/v1/wanted/missing?page=$page&pagesize=$amountPerPull&sortKey=$searchOrder&sortDirection=$searchDirection&apikey=${arrApiKey}" | jq -r '.records[].id') + wget --timeout=0 -q -O - "$arrUrl/api/v1/wanted/missing?page=$page&pagesize=$amountPerPull&sortKey=$searchOrder&sortDirection=$searchDirection&apikey=${arrApiKey}" | jq -r '.records[].id' | sort > /config/extended/cache/tocheck.txt log "$page :: missing :: Filtering Album IDs by removing previously searched Album IDs (/config/extended/logs/notfound/)" - for lidarrRecordId in $(echo $lidarrRecords); do + ls /config/extended/cache/notfound/ | sed "s/--.*//" > /config/extended/cache/notfound.txt + + for lidarrRecordId in $(comm -13 /config/extended/cache/notfound.txt /config/extended/cache/tocheck.txt); do if [ ! -f /config/extended/logs/notfound/$lidarrRecordId--* ]; then touch "/config/extended/cache/lidarr/list/${lidarrRecordId}-missing" fi done + rm /config/extended/cache/notfound.txt /config/extended/cache/tocheck.txt lidarrMissingRecords=$(ls /config/extended/cache/lidarr/list 2>/dev/null | wc -l) log "$page :: missing :: ${lidarrMissingRecords} albums found to process!" @@ -1034,14 +1037,18 @@ GetMissingCutOffList () { fi log "$page :: cutoff :: Downloading page $page... ($offset - $dlnumber of $lidarrCutoffTotalRecords Results)" - lidarrRecords=$(wget --timeout=0 -q -O - "$arrUrl/api/v1/wanted/cutoff?page=$page&pagesize=$amountPerPull&sortKey=$searchOrder&sortDirection=$searchDirection&apikey=${arrApiKey}" | jq -r '.records[].id') + # lidarrRecords=$(wget --timeout=0 -q -O - "$arrUrl/api/v1/wanted/cutoff?page=$page&pagesize=$amountPerPull&sortKey=$searchOrder&sortDirection=$searchDirection&apikey=${arrApiKey}" | jq -r '.records[].id') + wget --timeout=0 -q -O - "$arrUrl/api/v1/wanted/cutoff?page=$page&pagesize=$amountPerPull&sortKey=$searchOrder&sortDirection=$searchDirection&apikey=${arrApiKey}" | jq -r '.records[].id' | sort > /config/extended/cache/tocheck.txt + log "$page :: cutoff :: Filtering Album IDs by removing previously searched Album IDs (/config/extended/logs/notfound/)" - - for lidarrRecordId in $(echo $lidarrRecords); do + ls /config/extended/cache/notfound/ | sed "s/--.*//" > /config/extended/cache/notfound.txt + + for lidarrRecordId in $(comm -13 /config/extended/cache/notfound.txt /config/extended/cache/tocheck.txt); do if [ ! -f /config/extended/logs/notfound/$lidarrRecordId--* ]; then touch /config/extended/cache/lidarr/list/${lidarrRecordId}-cutoff fi done + rm /config/extended/cache/notfound.txt /config/extended/cache/tocheck.txt lidarrCutoffRecords=$(ls /config/extended/cache/lidarr/list/*-cutoff 2>/dev/null | wc -l) log "$page :: cutoff :: ${lidarrCutoffRecords} ablums found to process!" From 4a9f05c094a86a91ad9a4b636c5a141dd4fe1577 Mon Sep 17 00:00:00 2001 From: Jackmaninov Date: Sat, 14 Oct 2023 21:45:29 +0300 Subject: [PATCH 2/2] fix: broke a path --- lidarr/Audio.service.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lidarr/Audio.service.bash b/lidarr/Audio.service.bash index 557ba7f..7f4920c 100644 --- a/lidarr/Audio.service.bash +++ b/lidarr/Audio.service.bash @@ -998,7 +998,7 @@ GetMissingCutOffList () { log "$page :: missing :: Downloading page $page... ($offset - $dlnumber of $lidarrMissingTotalRecords Results)" wget --timeout=0 -q -O - "$arrUrl/api/v1/wanted/missing?page=$page&pagesize=$amountPerPull&sortKey=$searchOrder&sortDirection=$searchDirection&apikey=${arrApiKey}" | jq -r '.records[].id' | sort > /config/extended/cache/tocheck.txt log "$page :: missing :: Filtering Album IDs by removing previously searched Album IDs (/config/extended/logs/notfound/)" - ls /config/extended/cache/notfound/ | sed "s/--.*//" > /config/extended/cache/notfound.txt + ls /config/extended/logs/notfound/ | sed "s/--.*//" > /config/extended/cache/notfound.txt for lidarrRecordId in $(comm -13 /config/extended/cache/notfound.txt /config/extended/cache/tocheck.txt); do if [ ! -f /config/extended/logs/notfound/$lidarrRecordId--* ]; then @@ -1041,7 +1041,7 @@ GetMissingCutOffList () { wget --timeout=0 -q -O - "$arrUrl/api/v1/wanted/cutoff?page=$page&pagesize=$amountPerPull&sortKey=$searchOrder&sortDirection=$searchDirection&apikey=${arrApiKey}" | jq -r '.records[].id' | sort > /config/extended/cache/tocheck.txt log "$page :: cutoff :: Filtering Album IDs by removing previously searched Album IDs (/config/extended/logs/notfound/)" - ls /config/extended/cache/notfound/ | sed "s/--.*//" > /config/extended/cache/notfound.txt + ls /config/extended/logs/notfound/ | sed "s/--.*//" > /config/extended/cache/notfound.txt for lidarrRecordId in $(comm -13 /config/extended/cache/notfound.txt /config/extended/cache/tocheck.txt); do if [ ! -f /config/extended/logs/notfound/$lidarrRecordId--* ]; then