- Deprecate proxy

- Add Proxy for each debrid
- Add support for multiple-API keys
- Use internal http.Client for streaming
- Bug fixes etc

pkg/debrid/alldebrid/alldebrid.go

@@ -14,7 +14,6 @@ import (
 	"path/filepath"
 	"slices"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 )
@@ -23,7 +22,7 @@ type AllDebrid struct {
 	Name             string
 	Host             string `json:"host"`
 	APIKey           string
-	ExtraAPIKeys     []string
+	DownloadKeys     []string
 	DownloadUncached bool
 	client           *request.Client
 
@@ -34,26 +33,22 @@ type AllDebrid struct {
 
 func New(dc config.Debrid) *AllDebrid {
 	rl := request.ParseRateLimit(dc.RateLimit)
-	apiKeys := strings.Split(dc.APIKey, ",")
-	extraKeys := make([]string, 0)
-	if len(apiKeys) > 1 {
-		extraKeys = apiKeys[1:]
-	}
-	mainKey := apiKeys[0]
+
 	headers := map[string]string{
-		"Authorization": fmt.Sprintf("Bearer %s", mainKey),
+		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 	}
 	_log := logger.New(dc.Name)
 	client := request.New(
 		request.WithHeaders(headers),
 		request.WithLogger(_log),
 		request.WithRateLimiter(rl),
+		request.WithProxy(dc.Proxy),
 	)
 	return &AllDebrid{
 		Name:             "alldebrid",
 		Host:             dc.Host,
-		APIKey:           mainKey,
-		ExtraAPIKeys:     extraKeys,
+		APIKey:           dc.APIKey,
+		DownloadKeys:     dc.DownloadAPIKeys,
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
 		MountPath:        dc.Folder,
@@ -256,7 +251,7 @@ func (ad *AllDebrid) GenerateDownloadLinks(t *types.Torrent) error {
 	for _, file := range t.Files {
 		go func(file types.File) {
 			defer wg.Done()
-			link, err := ad.GetDownloadLink(t, &file)
+			link, err := ad.GetDownloadLink(t, &file, 0)
 			if err != nil {
 				errCh <- err
 				return
@@ -291,7 +286,7 @@ func (ad *AllDebrid) GenerateDownloadLinks(t *types.Torrent) error {
 	return nil
 }
 
-func (ad *AllDebrid) GetDownloadLink(t *types.Torrent, file *types.File) (string, error) {
+func (ad *AllDebrid) GetDownloadLink(t *types.Torrent, file *types.File, index int) (string, error) {
 	url := fmt.Sprintf("%s/link/unlock", ad.Host)
 	query := gourl.Values{}
 	query.Add("link", file.Link)
@@ -367,3 +362,7 @@ func (ad *AllDebrid) CheckLink(link string) error {
 func (ad *AllDebrid) GetMountPath() string {
 	return ad.MountPath
 }
+
+func (ad *AllDebrid) GetDownloadKeys() []string {
+	return ad.DownloadKeys
+}

pkg/debrid/debrid/cache.go

@@ -46,6 +46,7 @@ type CachedTorrent struct {
 
 type downloadLinkCache struct {
 	Link      string
+	KeyIndex  int
 	ExpiresAt time.Time
 }
 
@@ -68,12 +69,13 @@ type Cache struct {
 	client types.Client
 	logger zerolog.Logger
 
-	torrents      *xsync.MapOf[string, *CachedTorrent] // key: torrent.Id, value: *CachedTorrent
-	torrentsNames *xsync.MapOf[string, *CachedTorrent] // key: torrent.Name, value: torrent
-	listings      atomic.Value
-	downloadLinks *xsync.MapOf[string, downloadLinkCache]
-	PropfindResp  *xsync.MapOf[string, PropfindResponse]
-	folderNaming  WebDavFolderNaming
+	torrents             *xsync.MapOf[string, *CachedTorrent] // key: torrent.Id, value: *CachedTorrent
+	torrentsNames        *xsync.MapOf[string, *CachedTorrent] // key: torrent.Name, value: torrent
+	listings             atomic.Value
+	downloadLinks        *xsync.MapOf[string, downloadLinkCache]
+	invalidDownloadLinks *xsync.MapOf[string, string]
+	PropfindResp         *xsync.MapOf[string, PropfindResponse]
+	folderNaming         WebDavFolderNaming
 
 	// repair
 	repairChan        chan RepairRequest
@@ -116,6 +118,7 @@ func New(dc config.Debrid, client types.Client) *Cache {
 		dir:                          filepath.Join(cfg.Path, "cache", dc.Name), // path to save cache files
 		torrents:                     xsync.NewMapOf[string, *CachedTorrent](),
 		torrentsNames:                xsync.NewMapOf[string, *CachedTorrent](),
+		invalidDownloadLinks:         xsync.NewMapOf[string, string](),
 		client:                       client,
 		logger:                       logger.New(fmt.Sprintf("%s-webdav", client.GetName())),
 		workers:                      workers,
@@ -161,6 +164,8 @@ func (c *Cache) Start(ctx context.Context) error {
 
 func (c *Cache) load() (map[string]*CachedTorrent, error) {
 	torrents := make(map[string]*CachedTorrent)
+	var results sync.Map
+
 	if err := os.MkdirAll(c.dir, 0755); err != nil {
 		return torrents, fmt.Errorf("failed to create cache directory: %w", err)
 	}
@@ -170,54 +175,102 @@ func (c *Cache) load() (map[string]*CachedTorrent, error) {
 		return torrents, fmt.Errorf("failed to read cache directory: %w", err)
 	}
 
-	now := time.Now()
+	// Get only json files
+	var jsonFiles []os.DirEntry
 	for _, file := range files {
-		fileName := file.Name()
-		if file.IsDir() || filepath.Ext(fileName) != ".json" {
-			continue
-		}
-
-		filePath := filepath.Join(c.dir, fileName)
-		data, err := os.ReadFile(filePath)
-		if err != nil {
-			c.logger.Debug().Err(err).Msgf("Failed to read file: %s", filePath)
-			continue
-		}
-
-		var ct CachedTorrent
-		if err := json.Unmarshal(data, &ct); err != nil {
-			c.logger.Debug().Err(err).Msgf("Failed to unmarshal file: %s", filePath)
-			continue
-		}
-		isComplete := true
-		if len(ct.Files) != 0 {
-			// Check if all files are valid, if not, delete the file.json and remove from cache.
-			for _, f := range ct.Files {
-				if !f.IsValid() {
-					isComplete = false
-					break
-				}
-			}
-			if isComplete {
-				addedOn, err := time.Parse(time.RFC3339, ct.Added)
-				if err != nil {
-					addedOn = now
-				}
-				ct.AddedOn = addedOn
-				ct.IsComplete = true
-				torrents[ct.Id] = &ct
-			} else {
-				// Delete the file if it's not complete
-				_ = os.Remove(filePath)
-			}
-
+		if !file.IsDir() && filepath.Ext(file.Name()) == ".json" {
+			jsonFiles = append(jsonFiles, file)
 		}
 	}
 
+	if len(jsonFiles) == 0 {
+		return torrents, nil
+	}
+
+	// Create channels with appropriate buffering
+	workChan := make(chan os.DirEntry, min(c.workers, len(jsonFiles)))
+
+	// Create a wait group for workers
+	var wg sync.WaitGroup
+
+	// Start workers
+	for i := 0; i < c.workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			now := time.Now()
+
+			for {
+				file, ok := <-workChan
+				if !ok {
+					return // Channel closed, exit goroutine
+				}
+
+				fileName := file.Name()
+				filePath := filepath.Join(c.dir, fileName)
+				data, err := os.ReadFile(filePath)
+				if err != nil {
+					c.logger.Debug().Err(err).Msgf("Failed to read file: %s", filePath)
+					continue
+				}
+
+				var ct CachedTorrent
+				if err := json.Unmarshal(data, &ct); err != nil {
+					c.logger.Debug().Err(err).Msgf("Failed to unmarshal file: %s", filePath)
+					continue
+				}
+
+				isComplete := true
+				if len(ct.Files) != 0 {
+					// Check if all files are valid, if not, delete the file.json and remove from cache.
+					for _, f := range ct.Files {
+						if f.Link == "" {
+							isComplete = false
+							break
+						}
+					}
+
+					if isComplete {
+						addedOn, err := time.Parse(time.RFC3339, ct.Added)
+						if err != nil {
+							addedOn = now
+						}
+						ct.AddedOn = addedOn
+						ct.IsComplete = true
+						results.Store(ct.Id, &ct)
+					} else {
+						// Delete the file if it's not complete
+						_ = os.Remove(filePath)
+					}
+				}
+			}
+		}()
+	}
+
+	// Feed work to workers
+	for _, file := range jsonFiles {
+		workChan <- file
+	}
+
+	// Signal workers that no more work is coming
+	close(workChan)
+
+	// Wait for all workers to complete
+	wg.Wait()
+
+	// Convert sync.Map to regular map
+	results.Range(func(key, value interface{}) bool {
+		id, _ := key.(string)
+		torrent, _ := value.(*CachedTorrent)
+		torrents[id] = torrent
+		return true
+	})
+
 	return torrents, nil
 }
 
 func (c *Cache) Sync() error {
+	defer c.logger.Info().Msg("WebDav server sync complete")
 	cachedTorrents, err := c.load()
 	if err != nil {
 		c.logger.Debug().Err(err).Msg("Failed to load cache")
@@ -486,38 +539,45 @@ func (c *Cache) saveTorrent(id string, data []byte) {
 }
 
 func (c *Cache) ProcessTorrent(t *types.Torrent, refreshRclone bool) error {
-	if len(t.Files) == 0 {
+
+	isComplete := func(files map[string]types.File) bool {
+		_complete := len(files) > 0
+		for _, file := range files {
+			if file.Link == "" {
+				_complete = false
+				break
+			}
+		}
+		return _complete
+	}
+
+	if !isComplete(t.Files) {
 		if err := c.client.UpdateTorrent(t); err != nil {
 			return fmt.Errorf("failed to update torrent: %w", err)
 		}
 	}
-	// Validate each file in the torrent
-	isComplete := true
-	for _, file := range t.Files {
-		if file.Link == "" {
-			isComplete = false
-			continue
-		}
-	}
 
-	if !isComplete {
-		c.logger.Debug().Msgf("Torrent %s is not complete, missing link for file %s. Triggering a reinsert", t.Id)
-		if err := c.ReInsertTorrent(t); err != nil {
-			c.logger.Error().Err(err).Msgf("Failed to reinsert torrent %s", t.Id)
-			return fmt.Errorf("failed to reinsert torrent: %w", err)
-		}
-	}
+	if !isComplete(t.Files) {
+		c.logger.Debug().Msgf("Torrent %s is still not complete. Triggering a reinsert(disabled)", t.Id)
+		//ct, err := c.reInsertTorrent(t)
+		//if err != nil {
+		//	c.logger.Debug().Err(err).Msgf("Failed to reinsert torrent %s", t.Id)
+		//	return err
+		//}
+		//c.logger.Debug().Msgf("Reinserted torrent %s", ct.Id)
 
-	addedOn, err := time.Parse(time.RFC3339, t.Added)
-	if err != nil {
-		addedOn = time.Now()
+	} else {
+		addedOn, err := time.Parse(time.RFC3339, t.Added)
+		if err != nil {
+			addedOn = time.Now()
+		}
+		ct := &CachedTorrent{
+			Torrent:    t,
+			IsComplete: len(t.Files) > 0,
+			AddedOn:    addedOn,
+		}
+		c.setTorrent(ct)
 	}
-	ct := &CachedTorrent{
-		Torrent:    t,
-		IsComplete: len(t.Files) > 0,
-		AddedOn:    addedOn,
-	}
-	c.setTorrent(ct)
 
 	if refreshRclone {
 		c.refreshListings()
@@ -525,7 +585,7 @@ func (c *Cache) ProcessTorrent(t *types.Torrent, refreshRclone bool) error {
 	return nil
 }
 
-func (c *Cache) GetDownloadLink(torrentId, filename, fileLink string) string {
+func (c *Cache) GetDownloadLink(torrentId, filename, fileLink string, index int) string {
 
 	// Check link cache
 	if dl := c.checkDownloadLink(fileLink); dl != "" {
@@ -548,44 +608,60 @@ func (c *Cache) GetDownloadLink(torrentId, filename, fileLink string) string {
 		}
 	}
 
+	// If file.Link is still empty, return
+	if file.Link == "" {
+		c.logger.Debug().Msgf("File link is empty for %s. Release is probably nerfed", filename)
+		// Try to reinsert the torrent?
+		ct, err := c.reInsertTorrent(ct.Torrent)
+		if err != nil {
+			c.logger.Debug().Err(err).Msgf("Failed to reinsert torrent %s", ct.Name)
+			return ""
+		}
+		file = ct.Files[filename]
+		c.logger.Debug().Msgf("Reinserted torrent %s", ct.Name)
+	}
+
 	c.logger.Trace().Msgf("Getting download link for %s", filename)
-	downloadLink, err := c.client.GetDownloadLink(ct.Torrent, &file)
+	downloadLink, err := c.client.GetDownloadLink(ct.Torrent, &file, index)
 	if err != nil {
 		if errors.Is(err, request.HosterUnavailableError) {
-			c.logger.Debug().Err(err).Msgf("Hoster is unavailable for %s/%s", ct.Name, filename)
+			c.logger.Debug().Err(err).Msgf("Hoster is unavailable. Triggering repair for %s", ct.Name)
+			ct, err := c.reInsertTorrent(ct.Torrent)
+			if err != nil {
+				c.logger.Debug().Err(err).Msgf("Failed to reinsert torrent %s", ct.Name)
+				return ""
+			}
+			c.logger.Debug().Msgf("Reinserted torrent %s", ct.Name)
+			file = ct.Files[filename]
+			// Retry getting the download link
+			downloadLink, err = c.client.GetDownloadLink(ct.Torrent, &file, index)
+			if err != nil {
+				c.logger.Debug().Err(err).Msgf("Failed to get download link for %s", file.Link)
+				return ""
+			}
+			if downloadLink == "" {
+				c.logger.Debug().Msgf("Download link is empty for %s", file.Link)
+				return ""
+			}
+			file.DownloadLink = downloadLink
+			file.Generated = time.Now()
+			ct.Files[filename] = file
+			go func() {
+				c.updateDownloadLink(file.Link, downloadLink, 0)
+				c.setTorrent(ct)
+			}()
+			return file.DownloadLink
+		} else {
+			c.logger.Debug().Err(err).Msgf("Failed to get download link for %s", file.Link)
 			return ""
-			// This code is commented iut due to the fact that if a torrent link is uncached, it's likely that we can't redownload it again
-			// Do not attempt to repair the torrent if the hoster is unavailable
-			// Check link here??
-			//c.logger.Debug().Err(err).Msgf("Hoster is unavailable. Triggering repair for %s", ct.Name)
-			//if err := c.repairTorrent(ct); err != nil {
-			//	c.logger.Error().Err(err).Msgf("Failed to trigger repair for %s", ct.Name)
-			//	return ""
-			//}
-			//// Generate download link for the file then
-			//f := ct.Files[filename]
-			//downloadLink, _ = c.client.GetDownloadLink(ct.Torrent, &f)
-			//f.DownloadLink = downloadLink
-			//file.Generated = time.Now()
-			//ct.Files[filename] = f
-			//c.updateDownloadLink(file.Link, downloadLink)
-			//
-			//go func() {
-			//	go c.setTorrent(ct)
-			//}()
-			//
-			//return downloadLink // Gets download link in the next pass
 		}
-
-		c.logger.Debug().Err(err).Msgf("Failed to get download link for :%s", file.Link)
-		return ""
 	}
 	file.DownloadLink = downloadLink
 	file.Generated = time.Now()
 	ct.Files[filename] = file
 
 	go func() {
-		c.updateDownloadLink(file.Link, downloadLink)
+		c.updateDownloadLink(file.Link, downloadLink, 0)
 		c.setTorrent(ct)
 	}()
 	return file.DownloadLink
@@ -596,7 +672,7 @@ func (c *Cache) GenerateDownloadLinks(t *CachedTorrent) {
 		c.logger.Error().Err(err).Msg("Failed to generate download links")
 	}
 	for _, file := range t.Files {
-		c.updateDownloadLink(file.Link, file.DownloadLink)
+		c.updateDownloadLink(file.Link, file.DownloadLink, 0)
 	}
 
 	c.SaveTorrent(t)
@@ -624,22 +700,39 @@ func (c *Cache) AddTorrent(t *types.Torrent) error {
 
 }
 
-func (c *Cache) updateDownloadLink(link, downloadLink string) {
+func (c *Cache) updateDownloadLink(link, downloadLink string, keyIndex int) {
 	c.downloadLinks.Store(link, downloadLinkCache{
 		Link:      downloadLink,
-		ExpiresAt: time.Now().Add(c.autoExpiresLinksAfter), // Expires in 24 hours
+		ExpiresAt: time.Now().Add(c.autoExpiresLinksAfter),
+		KeyIndex:  keyIndex,
 	})
 }
 
 func (c *Cache) checkDownloadLink(link string) string {
 	if dl, ok := c.downloadLinks.Load(link); ok {
-		if dl.ExpiresAt.After(time.Now()) {
+		if dl.ExpiresAt.After(time.Now()) && !c.IsDownloadLinkInvalid(dl.Link) {
 			return dl.Link
 		}
 	}
 	return ""
 }
 
+func (c *Cache) RemoveDownloadLink(link string) {
+	c.downloadLinks.Delete(link)
+}
+
+func (c *Cache) MarkDownloadLinkAsInvalid(downloadLink, reason string) {
+	c.invalidDownloadLinks.Store(downloadLink, reason)
+}
+
+func (c *Cache) IsDownloadLinkInvalid(downloadLink string) bool {
+	if reason, ok := c.invalidDownloadLinks.Load(downloadLink); ok {
+		c.logger.Debug().Msgf("Download link %s is invalid: %s", downloadLink, reason)
+		return true
+	}
+	return false
+}
+
 func (c *Cache) GetClient() types.Client {
 	return c.client
 }
@@ -688,3 +781,7 @@ func (c *Cache) OnRemove(torrentId string) {
 func (c *Cache) GetLogger() zerolog.Logger {
 	return c.logger
 }
+
+func (c *Cache) TotalDownloadKeys() int {
+	return len(c.client.GetDownloadKeys())
+}

pkg/debrid/debrid/refresh.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"path/filepath"
 	"slices"
 	"sort"
 	"strings"
@@ -67,29 +66,6 @@ func (c *Cache) refreshListings() {
 	}
 }
 
-func (c *Cache) resetPropfindResponse() {
-	// Right now, parents are hardcoded
-	parents := []string{"__all__", "torrents"}
-	// Reset only the parent directories
-	// Convert the parents to a keys
-	// This is a bit hacky, but it works
-	// Instead of deleting all the keys, we only delete the parent keys, e.g __all__/ or torrents/
-	keys := make([]string, 0, len(parents))
-	for _, p := range parents {
-		// Construct the key
-		// construct url
-		url := filepath.Clean(filepath.Join("/webdav", c.client.GetName(), p))
-		key0 := fmt.Sprintf("propfind:%s:0", url)
-		key1 := fmt.Sprintf("propfind:%s:1", url)
-		keys = append(keys, key0, key1)
-	}
-
-	// Delete the keys
-	for _, k := range keys {
-		c.PropfindResp.Delete(k)
-	}
-}
-
 func (c *Cache) refreshTorrents() {
 	if c.torrentsRefreshMu.TryLock() {
 		defer c.torrentsRefreshMu.Unlock()
@@ -127,7 +103,7 @@ func (c *Cache) refreshTorrents() {
 
 	// Check for deleted torrents
 	deletedTorrents := make([]string, 0)
-	for id, _ := range torrents {
+	for id := range torrents {
 		if _, ok := idStore[id]; !ok {
 			deletedTorrents = append(deletedTorrents, id)
 		}
@@ -264,8 +240,7 @@ func (c *Cache) refreshDownloadLinks() {
 				ExpiresAt: v.Generated.Add(c.autoExpiresLinksAfter - timeSince),
 			})
 		} else {
-			//c.downloadLinks.Delete(k) don't delete, just log
-			c.logger.Trace().Msgf("Download link for %s expired", k)
+			c.downloadLinks.Delete(k)
 		}
 	}
 

pkg/debrid/debrid/repair.go

@@ -3,6 +3,7 @@ package debrid
 import (
 	"errors"
 	"fmt"
+	"github.com/puzpuzpuz/xsync/v3"
 	"github.com/sirrobot01/debrid-blackhole/internal/request"
 	"github.com/sirrobot01/debrid-blackhole/internal/utils"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/types"
@@ -35,6 +36,8 @@ func (c *Cache) IsTorrentBroken(t *CachedTorrent, filenames []string) bool {
 		}
 	}
 
+	files = t.Files
+
 	for _, f := range files {
 		// Check if file link is still missing
 		if f.Link == "" {
@@ -46,11 +49,7 @@ func (c *Cache) IsTorrentBroken(t *CachedTorrent, filenames []string) bool {
 				if errors.Is(err, request.HosterUnavailableError) {
 					isBroken = true
 					break
-				} else {
-					// This might just be a temporary error
 				}
-			} else {
-				// Generate a new download link?
 			}
 		}
 	}
@@ -59,70 +58,48 @@ func (c *Cache) IsTorrentBroken(t *CachedTorrent, filenames []string) bool {
 
 func (c *Cache) repairWorker() {
 	// This watches a channel for torrents to repair
-	for {
-		select {
-		case req := <-c.repairChan:
-			torrentId := req.TorrentID
-			if _, inProgress := c.repairsInProgress.Load(torrentId); inProgress {
-				c.logger.Debug().Str("torrentId", torrentId).Msg("Skipping duplicate repair request")
-				continue
-			}
-
-			// Mark as in progress
-			c.repairsInProgress.Store(torrentId, true)
-			c.logger.Debug().Str("torrentId", req.TorrentID).Msg("Received repair request")
-
-			// Get the torrent from the cache
-			cachedTorrent, ok := c.torrents.Load(torrentId)
-			if !ok || cachedTorrent == nil {
-				c.logger.Warn().Str("torrentId", torrentId).Msg("Torrent not found in cache")
-				continue
-			}
-
-			switch req.Type {
-			case RepairTypeReinsert:
-				c.logger.Debug().Str("torrentId", torrentId).Msg("Reinserting torrent")
-				c.reInsertTorrent(cachedTorrent)
-			case RepairTypeDelete:
-				c.logger.Debug().Str("torrentId", torrentId).Msg("Deleting torrent")
-				if err := c.DeleteTorrent(torrentId); err != nil {
-					c.logger.Error().Err(err).Str("torrentId", torrentId).Msg("Failed to delete torrent")
-					continue
-				}
-
-			}
-			c.repairsInProgress.Delete(torrentId)
+	for req := range c.repairChan {
+		torrentId := req.TorrentID
+		if _, inProgress := c.repairsInProgress.Load(torrentId); inProgress {
+			c.logger.Debug().Str("torrentId", torrentId).Msg("Skipping duplicate repair request")
+			continue
 		}
+
+		// Mark as in progress
+		c.repairsInProgress.Store(torrentId, true)
+		c.logger.Debug().Str("torrentId", req.TorrentID).Msg("Received repair request")
+
+		// Get the torrent from the cache
+		cachedTorrent, ok := c.torrents.Load(torrentId)
+		if !ok || cachedTorrent == nil {
+			c.logger.Warn().Str("torrentId", torrentId).Msg("Torrent not found in cache")
+			continue
+		}
+
+		switch req.Type {
+		case RepairTypeReinsert:
+			c.logger.Debug().Str("torrentId", torrentId).Msg("Reinserting torrent")
+			var err error
+			cachedTorrent, err = c.reInsertTorrent(cachedTorrent.Torrent)
+			if err != nil {
+				c.logger.Error().Err(err).Str("torrentId", cachedTorrent.Id).Msg("Failed to reinsert torrent")
+				continue
+			}
+		case RepairTypeDelete:
+			c.logger.Debug().Str("torrentId", torrentId).Msg("Deleting torrent")
+			if err := c.DeleteTorrent(torrentId); err != nil {
+				c.logger.Error().Err(err).Str("torrentId", torrentId).Msg("Failed to delete torrent")
+				continue
+			}
+		}
+		c.repairsInProgress.Delete(torrentId)
 	}
 }
 
-func (c *Cache) reInsertTorrent(t *CachedTorrent) {
-	// Reinsert the torrent into the cache
-	c.torrents.Store(t.Id, t)
-	c.logger.Debug().Str("torrentId", t.Id).Msg("Reinserted torrent into cache")
-}
-
-func (c *Cache) submitForRepair(repairType RepairType, torrentId, fileName string) {
-	// Submitting a torrent for repair.Not used yet
-
-	// Check if already in progress before even submitting
-	if _, inProgress := c.repairsInProgress.Load(torrentId); inProgress {
-		c.logger.Debug().Str("torrentID", torrentId).Msg("Repair already in progress")
-		return
-	}
-
-	select {
-	case c.repairChan <- RepairRequest{TorrentID: torrentId, FileName: fileName}:
-		c.logger.Debug().Str("torrentID", torrentId).Msg("Submitted for repair")
-	default:
-		c.logger.Warn().Str("torrentID", torrentId).Msg("Repair channel full, skipping repair request")
-	}
-}
-
-func (c *Cache) ReInsertTorrent(torrent *types.Torrent) error {
+func (c *Cache) reInsertTorrent(torrent *types.Torrent) (*CachedTorrent, error) {
 	// Check if Magnet is not empty, if empty, reconstruct the magnet
 	if _, ok := c.repairsInProgress.Load(torrent.Id); ok {
-		return fmt.Errorf("repair already in progress for torrent %s", torrent.Id)
+		return nil, fmt.Errorf("repair already in progress for torrent %s", torrent.Id)
 	}
 
 	if torrent.Magnet == nil {
@@ -130,8 +107,12 @@ func (c *Cache) ReInsertTorrent(torrent *types.Torrent) error {
 	}
 
 	oldID := torrent.Id
-	defer c.repairsInProgress.Delete(oldID)
-	defer c.DeleteTorrent(oldID)
+	defer func() {
+		err := c.DeleteTorrent(oldID)
+		if err != nil {
+			c.logger.Error().Err(err).Str("torrentId", oldID).Msg("Failed to delete old torrent")
+		}
+	}()
 
 	// Submit the magnet to the debrid service
 	torrent.Id = ""
@@ -139,12 +120,12 @@ func (c *Cache) ReInsertTorrent(torrent *types.Torrent) error {
 	torrent, err = c.client.SubmitMagnet(torrent)
 	if err != nil {
 		// Remove the old torrent from the cache and debrid service
-		return fmt.Errorf("failed to submit magnet: %w", err)
+		return nil, fmt.Errorf("failed to submit magnet: %w", err)
 	}
 
 	// Check if the torrent was submitted
 	if torrent == nil || torrent.Id == "" {
-		return fmt.Errorf("failed to submit magnet: empty torrent")
+		return nil, fmt.Errorf("failed to submit magnet: empty torrent")
 	}
 	torrent.DownloadUncached = false // Set to false, avoid re-downloading
 	torrent, err = c.client.CheckStatus(torrent, true)
@@ -152,24 +133,22 @@ func (c *Cache) ReInsertTorrent(torrent *types.Torrent) error {
 		// Torrent is likely in progress
 		_ = c.DeleteTorrent(torrent.Id)
 
-		return fmt.Errorf("failed to check status: %w", err)
+		return nil, fmt.Errorf("failed to check status: %w", err)
 	}
 
 	if torrent == nil {
-		return fmt.Errorf("failed to check status: empty torrent")
-	}
-
-	for _, file := range torrent.Files {
-		if file.Link == "" {
-			c.logger.Debug().Msgf("Torrent %s is still not complete, missing link for file %s.", torrent.Name, file.Name)
-			// Delete the torrent from the cache
-			_ = c.DeleteTorrent(torrent.Id)
-			return fmt.Errorf("torrent %s is still not complete, missing link for file %s", torrent.Name, file.Name)
-		}
+		return nil, fmt.Errorf("failed to check status: empty torrent")
 	}
 
 	// Update the torrent in the cache
 	addedOn, err := time.Parse(time.RFC3339, torrent.Added)
+	for _, f := range torrent.Files {
+		if f.Link == "" {
+			// Delete the new torrent
+			_ = c.DeleteTorrent(torrent.Id)
+			return nil, fmt.Errorf("failed to reinsert torrent: empty link")
+		}
+	}
 	if err != nil {
 		addedOn = time.Now()
 	}
@@ -180,5 +159,17 @@ func (c *Cache) ReInsertTorrent(torrent *types.Torrent) error {
 	}
 	c.setTorrent(ct)
 	c.refreshListings()
+	return ct, nil
+}
+
+func (c *Cache) refreshDownloadLink(link string) error {
+	// A generated download link has being limited
+	// Generate a new one with other API keys
+	// Temporarily remove the old one
 	return nil
 }
+
+func (c *Cache) resetDownloadLinks() {
+	c.invalidDownloadLinks = xsync.NewMapOf[string, string]()
+	c.downloadLinks = xsync.NewMapOf[string, downloadLinkCache]()
+}

pkg/debrid/debrid/worker.go

@@ -13,11 +13,8 @@ func (c *Cache) refreshDownloadLinksWorker() {
 	refreshTicker := time.NewTicker(c.downloadLinksRefreshInterval)
 	defer refreshTicker.Stop()
 
-	for {
-		select {
-		case <-refreshTicker.C:
-			c.refreshDownloadLinks()
-		}
+	for range refreshTicker.C {
+		c.refreshDownloadLinks()
 	}
 }
 
@@ -25,10 +22,7 @@ func (c *Cache) refreshTorrentsWorker() {
 	refreshTicker := time.NewTicker(c.torrentRefreshInterval)
 	defer refreshTicker.Stop()
 
-	for {
-		select {
-		case <-refreshTicker.C:
-			c.refreshTorrents()
-		}
+	for range refreshTicker.C {
+		c.refreshTorrents()
 	}
 }

pkg/debrid/debrid_link/debrid_link.go

@@ -21,7 +21,7 @@ type DebridLink struct {
 	Name             string
 	Host             string `json:"host"`
 	APIKey           string
-	ExtraAPIKeys     []string
+	DownloadKeys     []string
 	DownloadUncached bool
 	client           *request.Client
 
@@ -243,7 +243,7 @@ func (dl *DebridLink) GetDownloads() (map[string]types.DownloadLinks, error) {
 	return nil, nil
 }
 
-func (dl *DebridLink) GetDownloadLink(t *types.Torrent, file *types.File) (string, error) {
+func (dl *DebridLink) GetDownloadLink(t *types.Torrent, file *types.File, index int) (string, error) {
 	return file.DownloadLink, nil
 }
 
@@ -261,14 +261,9 @@ func (dl *DebridLink) GetDownloadUncached() bool {
 
 func New(dc config.Debrid) *DebridLink {
 	rl := request.ParseRateLimit(dc.RateLimit)
-	apiKeys := strings.Split(dc.APIKey, ",")
-	extraKeys := make([]string, 0)
-	if len(apiKeys) > 1 {
-		extraKeys = apiKeys[1:]
-	}
-	mainKey := apiKeys[0]
+
 	headers := map[string]string{
-		"Authorization": fmt.Sprintf("Bearer %s", mainKey),
+		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 		"Content-Type":  "application/json",
 	}
 	_log := logger.New(dc.Name)
@@ -276,12 +271,13 @@ func New(dc config.Debrid) *DebridLink {
 		request.WithHeaders(headers),
 		request.WithLogger(_log),
 		request.WithRateLimiter(rl),
+		request.WithProxy(dc.Proxy),
 	)
 	return &DebridLink{
 		Name:             "debridlink",
 		Host:             dc.Host,
-		APIKey:           mainKey,
-		ExtraAPIKeys:     extraKeys,
+		APIKey:           dc.APIKey,
+		DownloadKeys:     dc.DownloadAPIKeys,
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
 		MountPath:        dc.Folder,
@@ -371,3 +367,7 @@ func (dl *DebridLink) CheckLink(link string) error {
 func (dl *DebridLink) GetMountPath() string {
 	return dl.MountPath
 }
+
+func (dl *DebridLink) GetDownloadKeys() []string {
+	return dl.DownloadKeys
+}

pkg/debrid/realdebrid/realdebrid.go

@@ -1,6 +1,7 @@
 package realdebrid
 
 import (
+	"errors"
 	"fmt"
 	"github.com/goccy/go-json"
 	"github.com/rs/zerolog"
@@ -25,7 +26,7 @@ type RealDebrid struct {
 	Host string `json:"host"`
 
 	APIKey       string
-	ExtraAPIKeys []string // This is used for bandwidth
+	DownloadKeys []string // This is used for bandwidth
 
 	DownloadUncached bool
 	client           *request.Client
@@ -43,45 +44,58 @@ func (r *RealDebrid) GetLogger() zerolog.Logger {
 	return r.logger
 }
 
+func getSelectedFiles(t *types.Torrent, data TorrentInfo) map[string]types.File {
+	selectedFiles := make([]types.File, 0)
+	for _, f := range data.Files {
+		if f.Selected == 1 {
+			name := filepath.Base(f.Path)
+			file := types.File{
+				Name: name,
+				Path: name,
+				Size: f.Bytes,
+				Id:   strconv.Itoa(f.ID),
+			}
+			selectedFiles = append(selectedFiles, file)
+		}
+	}
+	files := make(map[string]types.File)
+	for index, f := range selectedFiles {
+		if index >= len(data.Links) {
+			break
+		}
+		f.Link = data.Links[index]
+		files[f.Name] = f
+	}
+	return files
+}
+
 // getTorrentFiles returns a list of torrent files from the torrent info
 // validate is used to determine if the files should be validated
 // if validate is false, selected files will be returned
-func getTorrentFiles(t *types.Torrent, data TorrentInfo, validate bool) map[string]types.File {
+func getTorrentFiles(t *types.Torrent, data TorrentInfo) map[string]types.File {
 	files := make(map[string]types.File)
 	cfg := config.Get()
 	idx := 0
+
 	for _, f := range data.Files {
 		name := filepath.Base(f.Path)
-		if validate {
-			if utils.IsSampleFile(f.Path) {
-				// Skip sample files
-				continue
-			}
-
-			if !cfg.IsAllowedFile(name) {
-				continue
-			}
-			if !cfg.IsSizeAllowed(f.Bytes) {
-				continue
-			}
-		} else {
-			if f.Selected == 0 {
-				continue
-			}
+		if utils.IsSampleFile(f.Path) {
+			// Skip sample files
+			continue
 		}
 
-		fileId := f.ID
-		_link := ""
-		if len(data.Links) > idx {
-			_link = data.Links[idx]
+		if !cfg.IsAllowedFile(name) {
+			continue
+		}
+		if !cfg.IsSizeAllowed(f.Bytes) {
+			continue
 		}
 
 		file := types.File{
 			Name: name,
 			Path: name,
 			Size: f.Bytes,
-			Id:   strconv.Itoa(fileId),
-			Link: _link,
+			Id:   strconv.Itoa(f.ID),
 		}
 		files[name] = file
 		idx++
@@ -182,7 +196,7 @@ func (r *RealDebrid) UpdateTorrent(t *types.Torrent) error {
 	t.MountPath = r.MountPath
 	t.Debrid = r.Name
 	t.Added = data.Added
-	t.Files = getTorrentFiles(t, data, false) // Get selected files
+	t.Files = getSelectedFiles(t, data) // Get selected files
 	return nil
 }
 
@@ -213,7 +227,7 @@ func (r *RealDebrid) CheckStatus(t *types.Torrent, isSymlink bool) (*types.Torre
 		t.Debrid = r.Name
 		t.MountPath = r.MountPath
 		if status == "waiting_files_selection" {
-			t.Files = getTorrentFiles(t, data, true)
+			t.Files = getTorrentFiles(t, data)
 			if len(t.Files) == 0 {
 				return t, fmt.Errorf("no video files found")
 			}
@@ -231,7 +245,7 @@ func (r *RealDebrid) CheckStatus(t *types.Torrent, isSymlink bool) (*types.Torre
 				return t, err
 			}
 		} else if status == "downloaded" {
-			t.Files = getTorrentFiles(t, data, false) // Get selected files
+			t.Files = getSelectedFiles(t, data) // Get selected files
 			r.logger.Info().Msgf("Torrent: %s downloaded to RD", t.Name)
 			if !isSymlink {
 				err = r.GenerateDownloadLinks(t)
@@ -273,7 +287,7 @@ func (r *RealDebrid) GenerateDownloadLinks(t *types.Torrent) error {
 		go func(file types.File) {
 			defer wg.Done()
 
-			link, err := r.GetDownloadLink(t, &file)
+			link, err := r.GetDownloadLink(t, &file, 0)
 			if err != nil {
 				errCh <- err
 				return
@@ -333,6 +347,7 @@ func (r *RealDebrid) _getDownloadLink(file *types.File) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		// Read the response body to get the error message
 		b, err := io.ReadAll(resp.Body)
@@ -348,12 +363,12 @@ func (r *RealDebrid) _getDownloadLink(file *types.File) (string, error) {
 			return "", request.TrafficExceededError
 		case 24:
 			return "", request.HosterUnavailableError // Link has been nerfed
+		case 19:
+			return "", request.HosterUnavailableError // File has been removed
 		default:
 			return "", fmt.Errorf("realdebrid API error: %d", resp.StatusCode)
 		}
-
 	}
-	defer resp.Body.Close()
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return "", err
@@ -366,20 +381,28 @@ func (r *RealDebrid) _getDownloadLink(file *types.File) (string, error) {
 
 }
 
-func (r *RealDebrid) GetDownloadLink(t *types.Torrent, file *types.File) (string, error) {
+func (r *RealDebrid) GetDownloadLink(t *types.Torrent, file *types.File, index int) (string, error) {
+
+	if index >= len(r.DownloadKeys) {
+		// Reset to first key
+		index = 0
+	}
+	r.client.SetHeader("Authorization", fmt.Sprintf("Bearer %s", r.DownloadKeys[index]))
 	link, err := r._getDownloadLink(file)
-	if err == nil {
+	if err == nil && link != "" {
 		return link, nil
 	}
-	for _, key := range r.ExtraAPIKeys {
-		r.client.SetHeader("Authorization", fmt.Sprintf("Bearer %s", key))
-		if link, err := r._getDownloadLink(file); err == nil {
-			return link, nil
+	if err != nil && errors.Is(err, request.HosterUnavailableError) {
+		// Try the next key
+		if index+1 < len(r.DownloadKeys) {
+			return r.GetDownloadLink(t, file, index+1)
 		}
 	}
-	// Reset to main API key
+	// If we reach here, it means we have tried all keys
+	// and none of them worked, or the error is not related to the keys
+	// Reset to the first key
 	r.client.SetHeader("Authorization", fmt.Sprintf("Bearer %s", r.APIKey))
-	return "", err
+	return link, err
 }
 
 func (r *RealDebrid) GetCheckCached() bool {
@@ -553,11 +576,7 @@ func (r *RealDebrid) GetMountPath() string {
 
 func New(dc config.Debrid) *RealDebrid {
 	rl := request.ParseRateLimit(dc.RateLimit)
-	apiKeys := strings.Split(dc.DownloadAPIKeys, ",")
-	extraKeys := make([]string, 0)
-	if len(apiKeys) > 1 {
-		extraKeys = apiKeys[1:]
-	}
+
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 	}
@@ -568,12 +587,13 @@ func New(dc config.Debrid) *RealDebrid {
 		request.WithLogger(_log),
 		request.WithMaxRetries(5),
 		request.WithRetryableStatus(429),
+		request.WithProxy(dc.Proxy),
 	)
 	return &RealDebrid{
 		Name:             "realdebrid",
 		Host:             dc.Host,
 		APIKey:           dc.APIKey,
-		ExtraAPIKeys:     extraKeys,
+		DownloadKeys:     dc.DownloadAPIKeys,
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
 		MountPath:        dc.Folder,
@@ -581,3 +601,7 @@ func New(dc config.Debrid) *RealDebrid {
 		CheckCached:      dc.CheckCached,
 	}
 }
+
+func (r *RealDebrid) GetDownloadKeys() []string {
+	return r.DownloadKeys
+}

pkg/debrid/torbox/torbox.go

@@ -25,7 +25,7 @@ type Torbox struct {
 	Name             string
 	Host             string `json:"host"`
 	APIKey           string
-	ExtraAPIKeys     []string
+	DownloadKeys     []string
 	DownloadUncached bool
 	client           *request.Client
 
@@ -36,27 +36,23 @@ type Torbox struct {
 
 func New(dc config.Debrid) *Torbox {
 	rl := request.ParseRateLimit(dc.RateLimit)
-	apiKeys := strings.Split(dc.APIKey, ",")
-	extraKeys := make([]string, 0)
-	if len(apiKeys) > 1 {
-		extraKeys = apiKeys[1:]
-	}
-	mainKey := apiKeys[0]
+
 	headers := map[string]string{
-		"Authorization": fmt.Sprintf("Bearer %s", mainKey),
+		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 	}
 	_log := logger.New(dc.Name)
 	client := request.New(
 		request.WithHeaders(headers),
 		request.WithRateLimiter(rl),
 		request.WithLogger(_log),
+		request.WithProxy(dc.Proxy),
 	)
 
 	return &Torbox{
 		Name:             "torbox",
 		Host:             dc.Host,
-		APIKey:           mainKey,
-		ExtraAPIKeys:     extraKeys,
+		APIKey:           dc.APIKey,
+		DownloadKeys:     dc.DownloadAPIKeys,
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
 		MountPath:        dc.Folder,
@@ -284,7 +280,7 @@ func (tb *Torbox) GenerateDownloadLinks(t *types.Torrent) error {
 	for _, file := range t.Files {
 		go func() {
 			defer wg.Done()
-			link, err := tb.GetDownloadLink(t, &file)
+			link, err := tb.GetDownloadLink(t, &file, 0)
 			if err != nil {
 				errCh <- err
 				return
@@ -316,7 +312,7 @@ func (tb *Torbox) GenerateDownloadLinks(t *types.Torrent) error {
 	return nil
 }
 
-func (tb *Torbox) GetDownloadLink(t *types.Torrent, file *types.File) (string, error) {
+func (tb *Torbox) GetDownloadLink(t *types.Torrent, file *types.File, index int) (string, error) {
 	url := fmt.Sprintf("%s/api/torrents/requestdl/", tb.Host)
 	query := gourl.Values{}
 	query.Add("torrent_id", t.Id)
@@ -366,3 +362,7 @@ func (tb *Torbox) CheckLink(link string) error {
 func (tb *Torbox) GetMountPath() string {
 	return tb.MountPath
 }
+
+func (tb *Torbox) GetDownloadKeys() []string {
+	return tb.DownloadKeys
+}

pkg/debrid/types/debrid.go

@@ -8,7 +8,7 @@ type Client interface {
 	SubmitMagnet(tr *Torrent) (*Torrent, error)
 	CheckStatus(tr *Torrent, isSymlink bool) (*Torrent, error)
 	GenerateDownloadLinks(tr *Torrent) error
-	GetDownloadLink(tr *Torrent, file *File) (string, error)
+	GetDownloadLink(tr *Torrent, file *File, index int) (string, error)
 	DeleteTorrent(torrentId string) error
 	IsAvailable(infohashes []string) map[string]bool
 	GetCheckCached() bool
@@ -21,4 +21,5 @@ type Client interface {
 	GetDownloads() (map[string]DownloadLinks, error)
 	CheckLink(link string) error
 	GetMountPath() string
+	GetDownloadKeys() []string
 }