johno/decypharr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Torrent Queuing for Botched torrent (#83)

Browse Source 
* Implement a queue for handling failed torrent

* Add checks for getting slots

* Few other cleanups, change some function names
This commit is contained in:
Mukhtar Akere
2025-06-07 17:23:41 +01:00
committed by GitHub
parent 84603b084b
commit 5bf1dab5e6
30 changed files with 556 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
pkg/store/torrent.go
View File

@@ -3,6 +3,7 @@ package store
import (
"cmp"
"context"
"errors"
"fmt"
"github.com/sirrobot01/decypharr/internal/request"
"github.com/sirrobot01/decypharr/internal/utils"
@@ -15,22 +16,125 @@ import (
func (s *Store) AddTorrent(ctx context.Context, importReq *ImportRequest) error {
torrent := createTorrentFromMagnet(importReq)
debridTorrent, err := debridTypes.ProcessTorrent(ctx, s.debrid, importReq.Debrid, importReq.Magnet, importReq.Arr, importReq.IsSymlink, importReq.DownloadUncached)
if err != nil || debridTorrent == nil {
if err == nil {
err = fmt.Errorf("failed to process torrent")
debridTorrent, err := debridTypes.Process(ctx, s.debrid, importReq.SelectedDebrid, importReq.Magnet, importReq.Arr, importReq.IsSymlink, importReq.DownloadUncached)
if err != nil {
var httpErr *utils.HTTPError
if ok := errors.As(err, &httpErr); ok {
switch httpErr.Code {
case "too_many_active_downloads":
// Handle too much active downloads error
s.logger.Warn().Msgf("Too many active downloads for %s, adding to queue", importReq.Magnet.Name)
err := s.addToQueue(importReq)
if err != nil {
s.logger.Error().Err(err).Msgf("Failed to add %s to queue", importReq.Magnet.Name)
return err
}
torrent.State = "queued"
default:
// Unhandled error, return it, caller logs it
return err
}
} else {
// Unhandled error, return it, caller logs it
return err
}
// This error is returned immediately to the user(no need for callback)
return err
}
torrent = s.UpdateTorrentMin(torrent, debridTorrent)
torrent = s.partialTorrentUpdate(torrent, debridTorrent)
s.torrents.AddOrUpdate(torrent)
go s.processFiles(torrent, debridTorrent, importReq) // We can send async for file processing not to delay the response
return nil
}
func (s *Store) addToQueue(importReq *ImportRequest) error {
if importReq.Magnet == nil {
return fmt.Errorf("magnet is required")
}
if importReq.Arr == nil {
return fmt.Errorf("arr is required")
}
importReq.Status = "queued"
importReq.CompletedAt = time.Time{}
importReq.Error = nil
err := s.importsQueue.Push(importReq)
if err != nil {
return err
}
return nil
}
func (s *Store) processFromQueue(ctx context.Context, selectedDebrid string) error {
// Pop the next import request from the queue
importReq, err := s.importsQueue.TryPop(selectedDebrid)
if err != nil {
return err
}
if importReq == nil {
return nil
}
return s.AddTorrent(ctx, importReq)
}
func (s *Store) StartQueueSchedule(ctx context.Context) error {
s.trackAvailableSlots(ctx) // Initial tracking of available slots
ticker := time.NewTicker(time.Minute)
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
s.trackAvailableSlots(ctx)
}
}
}
func (s *Store) trackAvailableSlots(ctx context.Context) {
// This function tracks the available slots for each debrid client
availableSlots := make(map[string]int)
for name, deb := range s.debrid.Debrids() {
slots, err := deb.Client().GetAvailableSlots()
if err != nil {
continue
}
availableSlots[name] = slots
}
for name, slots := range availableSlots {
if s.importsQueue.Size(name) <= 0 {
continue
}
s.logger.Debug().Msgf("Available slots for %s: %d", name, slots)
// If slots are available, process the next import request from the queue
for slots > 0 {
select {
case <-ctx.Done():
return // Exit if context is done
default:
if err := s.processFromQueue(ctx, name); err != nil {
s.logger.Error().Err(err).Msg("Error processing from queue")
return // Exit on error
}
slots-- // Decrease the available slots after processing
}
}
}
}
func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, importReq *ImportRequest) {
client := s.debrid.GetClient(debridTorrent.Debrid)
if debridTorrent == nil {
// Early return if debridTorrent is nil
return
}
deb := s.debrid.Debrid(debridTorrent.Debrid)
client := deb.Client()
downloadingStatuses := client.GetDownloadingStatus()
_arr := importReq.Arr
for debridTorrent.Status != "downloaded" {
@@ -53,7 +157,7 @@ func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, imp
}
debridTorrent = dbT
torrent = s.UpdateTorrentMin(torrent, debridTorrent)
torrent = s.partialTorrentUpdate(torrent, debridTorrent)
// Exit the loop for downloading statuses to prevent memory buildup
if debridTorrent.Status == "downloaded" || !utils.Contains(downloadingStatuses, debridTorrent.Status) {
@@ -71,9 +175,8 @@ func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, imp
// Check if debrid supports webdav by checking cache
timer := time.Now()
if importReq.IsSymlink {
caches := s.debrid.GetCaches()
cache, useWebdav := caches[debridTorrent.Debrid]
if useWebdav {
cache := deb.Cache()
if cache != nil {
s.logger.Info().Msgf("Using internal webdav for %s", debridTorrent.Debrid)
// Use webdav to download the file
@@ -91,10 +194,10 @@ func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, imp
} else {
// User is using either zurg or debrid webdav
torrentSymlinkPath, err = s.ProcessSymlink(torrent) // /mnt/symlinks/{category}/MyTVShow/
torrentSymlinkPath, err = s.processSymlink(torrent) // /mnt/symlinks/{category}/MyTVShow/
}
} else {
torrentSymlinkPath, err = s.ProcessManualFile(torrent)
torrentSymlinkPath, err = s.processDownload(torrent)
}
if err != nil {
s.markTorrentAsFailed(torrent)
@@ -106,7 +209,7 @@ func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, imp
return
}
torrent.TorrentPath = torrentSymlinkPath
s.UpdateTorrent(torrent, debridTorrent)
s.updateTorrent(torrent, debridTorrent)
s.logger.Info().Msgf("Adding %s took %s", debridTorrent.Name, time.Since(timer))
go importReq.markAsCompleted(torrent, debridTorrent) // Mark the import request as completed, send callback if needed
@@ -129,7 +232,7 @@ func (s *Store) markTorrentAsFailed(t *Torrent) *Torrent {
return t
}
func (s *Store) UpdateTorrentMin(t *Torrent, debridTorrent *types.Torrent) *Torrent {
func (s *Store) partialTorrentUpdate(t *Torrent, debridTorrent *types.Torrent) *Torrent {
if debridTorrent == nil {
return t
}
@@ -170,17 +273,17 @@ func (s *Store) UpdateTorrentMin(t *Torrent, debridTorrent *types.Torrent) *Torr
return t
}
func (s *Store) UpdateTorrent(t *Torrent, debridTorrent *types.Torrent) *Torrent {
func (s *Store) updateTorrent(t *Torrent, debridTorrent *types.Torrent) *Torrent {
if debridTorrent == nil {
return t
}
if debridClient := s.debrid.GetClients()[debridTorrent.Debrid]; debridClient != nil {
if debridClient := s.debrid.Clients()[debridTorrent.Debrid]; debridClient != nil {
if debridTorrent.Status != "downloaded" {
_ = debridClient.UpdateTorrent(debridTorrent)
}
}
t = s.UpdateTorrentMin(t, debridTorrent)
t = s.partialTorrentUpdate(t, debridTorrent)
t.ContentPath = t.TorrentPath + string(os.PathSeparator)
if t.IsReady() {
@@ -200,7 +303,7 @@ func (s *Store) UpdateTorrent(t *Torrent, debridTorrent *types.Torrent) *Torrent
s.torrents.Update(t)
return t
}
updatedT := s.UpdateTorrent(t, debridTorrent)
updatedT := s.updateTorrent(t, debridTorrent)
t = updatedT
case <-time.After(10 * time.Minute): // Add a timeout