initializing webdav server

pkg/debrid/alldebrid/alldebrid.go

@@ -1,20 +1,19 @@
 package alldebrid
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"github.com/rs/zerolog"
-	"github.com/sirrobot01/debrid-blackhole/internal/cache"
 	"github.com/sirrobot01/debrid-blackhole/internal/config"
 	"github.com/sirrobot01/debrid-blackhole/internal/logger"
 	"github.com/sirrobot01/debrid-blackhole/internal/request"
 	"github.com/sirrobot01/debrid-blackhole/internal/utils"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent"
 	"slices"
+	"time"
 
 	"net/http"
 	gourl "net/url"
-	"os"
 	"path/filepath"
 	"strconv"
 )
@@ -24,11 +23,11 @@ type AllDebrid struct {
 	Host             string `json:"host"`
 	APIKey           string
 	DownloadUncached bool
-	client           *request.RLHTTPClient
-	cache            *cache.Cache
-	MountPath        string
-	logger           zerolog.Logger
-	CheckCached      bool
+	client           *request.Client
+
+	MountPath   string
+	logger      zerolog.Logger
+	CheckCached bool
 }
 
 func (ad *AllDebrid) GetName() string {
@@ -39,15 +38,9 @@ func (ad *AllDebrid) GetLogger() zerolog.Logger {
 	return ad.logger
 }
 
-func (ad *AllDebrid) IsAvailable(infohashes []string) map[string]bool {
+func (ad *AllDebrid) IsAvailable(hashes []string) map[string]bool {
 	// Check if the infohashes are available in the local cache
-	hashes, result := torrent.GetLocalCache(infohashes, ad.cache)
-
-	if len(hashes) == 0 {
-		// Either all the infohashes are locally cached or none are
-		ad.cache.AddMultiple(result)
-		return result
-	}
+	result := make(map[string]bool)
 
 	// Divide hashes into groups of 100
 	// AllDebrid does not support checking cached infohashes
@@ -91,8 +84,8 @@ func getAlldebridStatus(statusCode int) string {
 	}
 }
 
-func flattenFiles(files []MagnetFile, parentPath string, index *int) []torrent.File {
-	result := make([]torrent.File, 0)
+func flattenFiles(files []MagnetFile, parentPath string, index *int) map[string]torrent.File {
+	result := make(map[string]torrent.File)
 
 	cfg := config.GetConfig()
 
@@ -104,7 +97,15 @@ func flattenFiles(files []MagnetFile, parentPath string, index *int) []torrent.F
 
 		if f.Elements != nil {
 			// This is a folder, recurse into it
-			result = append(result, flattenFiles(f.Elements, currentPath, index)...)
+			subFiles := flattenFiles(f.Elements, currentPath, index)
+			for k, v := range subFiles {
+				if _, ok := result[k]; ok {
+					// File already exists, use path as key
+					result[v.Path] = v
+				} else {
+					result[k] = v
+				}
+			}
 		} else {
 			// This is a file
 			fileName := filepath.Base(f.Name)
@@ -128,25 +129,25 @@ func flattenFiles(files []MagnetFile, parentPath string, index *int) []torrent.F
 				Size: f.Size,
 				Path: currentPath,
 			}
-			result = append(result, file)
+			result[file.Name] = file
 		}
 	}
 
 	return result
 }
 
-func (ad *AllDebrid) GetTorrent(t *torrent.Torrent) (*torrent.Torrent, error) {
+func (ad *AllDebrid) UpdateTorrent(t *torrent.Torrent) error {
 	url := fmt.Sprintf("%s/magnet/status?id=%s", ad.Host, t.Id)
 	req, _ := http.NewRequest(http.MethodGet, url, nil)
 	resp, err := ad.client.MakeRequest(req)
 	if err != nil {
-		return t, err
+		return err
 	}
 	var res TorrentInfoResponse
 	err = json.Unmarshal(resp, &res)
 	if err != nil {
 		ad.logger.Info().Msgf("Error unmarshalling torrent info: %s", err)
-		return t, err
+		return err
 	}
 	data := res.Data.Magnets
 	status := getAlldebridStatus(data.StatusCode)
@@ -158,7 +159,6 @@ func (ad *AllDebrid) GetTorrent(t *torrent.Torrent) (*torrent.Torrent, error) {
 	t.Folder = name
 	t.MountPath = ad.MountPath
 	t.Debrid = ad.Name
-	t.DownloadLinks = make(map[string]torrent.DownloadLinks)
 	if status == "downloaded" {
 		t.Bytes = data.Size
 
@@ -169,23 +169,21 @@ func (ad *AllDebrid) GetTorrent(t *torrent.Torrent) (*torrent.Torrent, error) {
 		files := flattenFiles(data.Files, "", &index)
 		t.Files = files
 	}
-	return t, nil
+	return nil
 }
 
 func (ad *AllDebrid) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*torrent.Torrent, error) {
 	for {
-		tb, err := ad.GetTorrent(torrent)
+		err := ad.UpdateTorrent(torrent)
 
-		torrent = tb
-
-		if err != nil || tb == nil {
-			return tb, err
+		if err != nil || torrent == nil {
+			return torrent, err
 		}
 		status := torrent.Status
 		if status == "downloaded" {
 			ad.logger.Info().Msgf("Torrent: %s downloaded", torrent.Name)
 			if !isSymlink {
-				err = ad.GetDownloadLinks(torrent)
+				err = ad.GenerateDownloadLinks(torrent)
 				if err != nil {
 					return torrent, err
 				}
@@ -217,8 +215,7 @@ func (ad *AllDebrid) DeleteTorrent(torrent *torrent.Torrent) {
 	}
 }
 
-func (ad *AllDebrid) GetDownloadLinks(t *torrent.Torrent) error {
-	downloadLinks := make(map[string]torrent.DownloadLinks)
+func (ad *AllDebrid) GenerateDownloadLinks(t *torrent.Torrent) error {
 	for _, file := range t.Files {
 		url := fmt.Sprintf("%s/link/unlock", ad.Host)
 		query := gourl.Values{}
@@ -234,19 +231,15 @@ func (ad *AllDebrid) GetDownloadLinks(t *torrent.Torrent) error {
 			return err
 		}
 		link := data.Data.Link
+		file.DownloadLink = link
+		file.Generated = time.Now()
+		t.Files[file.Name] = file
 
-		dl := torrent.DownloadLinks{
-			Link:         file.Link,
-			Filename:     data.Data.Filename,
-			DownloadLink: link,
-		}
-		downloadLinks[file.Id] = dl
 	}
-	t.DownloadLinks = downloadLinks
 	return nil
 }
 
-func (ad *AllDebrid) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.DownloadLinks {
+func (ad *AllDebrid) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.File {
 	url := fmt.Sprintf("%s/link/unlock", ad.Host)
 	query := gourl.Values{}
 	query.Add("link", file.Link)
@@ -261,11 +254,9 @@ func (ad *AllDebrid) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *to
 		return nil
 	}
 	link := data.Data.Link
-	return &torrent.DownloadLinks{
-		DownloadLink: link,
-		Link:         file.Link,
-		Filename:     data.Data.Filename,
-	}
+	file.DownloadLink = link
+	file.Generated = time.Now()
+	return file
 }
 
 func (ad *AllDebrid) GetCheckCached() bool {
@@ -276,6 +267,10 @@ func (ad *AllDebrid) GetTorrents() ([]*torrent.Torrent, error) {
 	return nil, nil
 }
 
+func (ad *AllDebrid) GetDownloads() (map[string]torrent.DownloadLinks, error) {
+	return nil, nil
+}
+
 func (ad *AllDebrid) GetDownloadingStatus() []string {
 	return []string{"downloading"}
 }
@@ -284,21 +279,27 @@ func (ad *AllDebrid) GetDownloadUncached() bool {
 	return ad.DownloadUncached
 }
 
-func New(dc config.Debrid, cache *cache.Cache) *AllDebrid {
+func (ad *AllDebrid) ConvertLinksToFiles(links []string) []torrent.File {
+	return nil
+}
+
+func New(dc config.Debrid) *AllDebrid {
 	rl := request.ParseRateLimit(dc.RateLimit)
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 	}
-	client := request.NewRLHTTPClient(rl, headers)
+	_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
+	client := request.New().
+		WithHeaders(headers).
+		WithRateLimiter(rl).WithLogger(_log)
 	return &AllDebrid{
 		Name:             "alldebrid",
 		Host:             dc.Host,
 		APIKey:           dc.APIKey,
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
-		cache:            cache,
 		MountPath:        dc.Folder,
-		logger:           logger.NewLogger(dc.Name, config.GetConfig().LogLevel, os.Stdout),
+		logger:           logger.NewLogger(dc.Name, config.GetConfig().LogLevel),
 		CheckCached:      dc.CheckCached,
 	}
 }

pkg/debrid/cache/cache.go

@@ -1,452 +0,0 @@
-package cache
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/dgraph-io/badger/v4"
-	"github.com/rs/zerolog"
-	"github.com/sirrobot01/debrid-blackhole/internal/logger"
-	"os"
-	"path/filepath"
-	"runtime"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/sirrobot01/debrid-blackhole/internal/config"
-	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/engine"
-	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent"
-)
-
-type DownloadLinkCache struct {
-	Link string `json:"download_link"`
-}
-
-type CachedTorrent struct {
-	*torrent.Torrent
-	LastRead      time.Time                    `json:"last_read"`
-	IsComplete    bool                         `json:"is_complete"`
-	DownloadLinks map[string]DownloadLinkCache `json:"download_links"`
-}
-
-var (
-	_logInstance zerolog.Logger
-	once         sync.Once
-)
-
-func getLogger() zerolog.Logger {
-	once.Do(func() {
-		cfg := config.GetConfig()
-		_logInstance = logger.NewLogger("cache", cfg.LogLevel, os.Stdout)
-	})
-	return _logInstance
-}
-
-type Cache struct {
-	dir               string
-	client            engine.Service
-	db                *badger.DB
-	torrents          map[string]*CachedTorrent // key: torrent.Id, value: *CachedTorrent
-	torrentsMutex     sync.RWMutex
-	torrentsNames     map[string]*CachedTorrent // key: torrent.Name, value: torrent
-	torrentNamesMutex sync.RWMutex
-	LastUpdated       time.Time `json:"last_updated"`
-}
-
-func (c *Cache) SetTorrent(t *CachedTorrent) {
-	c.torrentsMutex.Lock()
-	defer c.torrentsMutex.Unlock()
-	c.torrents[t.Id] = t
-}
-
-func (c *Cache) SetTorrentName(name string, t *CachedTorrent) {
-	c.torrentNamesMutex.Lock()
-	defer c.torrentNamesMutex.Unlock()
-	c.torrentsNames[name] = t
-}
-
-func (c *Cache) GetTorrents() map[string]*CachedTorrent {
-	c.torrentsMutex.RLock()
-	defer c.torrentsMutex.RUnlock()
-	return c.torrents
-}
-
-func (c *Cache) GetTorrentNames() map[string]*CachedTorrent {
-	c.torrentNamesMutex.RLock()
-	defer c.torrentNamesMutex.RUnlock()
-	return c.torrentsNames
-}
-
-type Manager struct {
-	caches map[string]*Cache
-}
-
-func NewManager(debridService *engine.Engine) *Manager {
-	cfg := config.GetConfig()
-	cm := &Manager{
-		caches: make(map[string]*Cache),
-	}
-	for _, debrid := range debridService.GetDebrids() {
-		c := New(debrid, cfg.Path)
-		cm.caches[debrid.GetName()] = c
-	}
-	return cm
-}
-
-func (m *Manager) GetCaches() map[string]*Cache {
-	return m.caches
-}
-
-func (m *Manager) GetCache(debridName string) *Cache {
-	return m.caches[debridName]
-}
-
-func New(debridService engine.Service, basePath string) *Cache {
-	dbPath := filepath.Join(basePath, "cache", debridService.GetName(), "db")
-	return &Cache{
-		dir:           dbPath,
-		torrents:      make(map[string]*CachedTorrent),
-		torrentsNames: make(map[string]*CachedTorrent),
-		client:        debridService,
-	}
-}
-
-func (c *Cache) Start() error {
-	_logger := getLogger()
-	_logger.Info().Msg("Starting cache for: " + c.client.GetName())
-
-	// Make sure the directory exists
-	if err := os.MkdirAll(c.dir, 0755); err != nil {
-		return fmt.Errorf("failed to create cache directory: %w", err)
-	}
-
-	// Open BadgerDB
-	opts := badger.DefaultOptions(c.dir)
-	opts.Logger = nil // Disable Badger's internal logger
-
-	var err error
-	c.db, err = badger.Open(opts)
-	if err != nil {
-		return fmt.Errorf("failed to open BadgerDB: %w", err)
-	}
-
-	if err := c.Load(); err != nil {
-		return fmt.Errorf("failed to load cache: %v", err)
-	}
-
-	if err := c.Sync(); err != nil {
-		return fmt.Errorf("failed to sync cache: %v", err)
-	}
-
-	return nil
-}
-
-func (c *Cache) Close() error {
-	if c.db != nil {
-		return c.db.Close()
-	}
-	return nil
-}
-
-func (c *Cache) Load() error {
-	_logger := getLogger()
-
-	err := c.db.View(func(txn *badger.Txn) error {
-		opts := badger.DefaultIteratorOptions
-		it := txn.NewIterator(opts)
-		defer it.Close()
-
-		prefix := []byte("torrent:")
-		for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
-			item := it.Item()
-
-			err := item.Value(func(val []byte) error {
-				var ct CachedTorrent
-				if err := json.Unmarshal(val, &ct); err != nil {
-					_logger.Debug().Err(err).Msgf("Failed to unmarshal torrent")
-					return nil // Continue to next item
-				}
-
-				if len(ct.Files) > 0 {
-					c.SetTorrent(&ct)
-					c.SetTorrentName(ct.Name, &ct)
-				}
-				return nil
-			})
-
-			if err != nil {
-				_logger.Debug().Err(err).Msg("Error reading torrent value")
-			}
-		}
-		return nil
-	})
-
-	return err
-}
-
-func (c *Cache) GetTorrent(id string) *CachedTorrent {
-	if t, ok := c.GetTorrents()[id]; ok {
-		return t
-	}
-	return nil
-}
-
-func (c *Cache) GetTorrentByName(name string) *CachedTorrent {
-	if t, ok := c.GetTorrentNames()[name]; ok {
-		return t
-	}
-	return nil
-}
-
-func (c *Cache) SaveTorrent(ct *CachedTorrent) error {
-	data, err := json.Marshal(ct)
-	if err != nil {
-		return fmt.Errorf("failed to marshal torrent: %w", err)
-	}
-
-	key := []byte(fmt.Sprintf("torrent:%s", ct.Torrent.Id))
-
-	err = c.db.Update(func(txn *badger.Txn) error {
-		return txn.Set(key, data)
-	})
-
-	if err != nil {
-		return fmt.Errorf("failed to save torrent to BadgerDB: %w", err)
-	}
-
-	// Also create an index by name for quick lookups
-	nameKey := []byte(fmt.Sprintf("name:%s", ct.Torrent.Name))
-	err = c.db.Update(func(txn *badger.Txn) error {
-		return txn.Set(nameKey, []byte(ct.Torrent.Id))
-	})
-
-	if err != nil {
-		return fmt.Errorf("failed to save torrent name index: %w", err)
-	}
-
-	return nil
-}
-
-func (c *Cache) SaveAll() error {
-	const batchSize = 100
-	var wg sync.WaitGroup
-	_logger := getLogger()
-
-	tasks := make(chan *CachedTorrent, batchSize)
-
-	for i := 0; i < runtime.NumCPU(); i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for ct := range tasks {
-				if err := c.SaveTorrent(ct); err != nil {
-					_logger.Error().Err(err).Msg("failed to save torrent")
-				}
-			}
-		}()
-	}
-
-	for _, value := range c.GetTorrents() {
-		tasks <- value
-	}
-
-	close(tasks)
-	wg.Wait()
-	c.LastUpdated = time.Now()
-
-	// Run value log garbage collection when appropriate
-	// This helps reclaim space from deleted/updated values
-	go func() {
-		err := c.db.RunValueLogGC(0.5) // Run GC if 50% of the value log can be discarded
-		if err != nil && err != badger.ErrNoRewrite {
-			_logger.Debug().Err(err).Msg("BadgerDB value log GC")
-		}
-	}()
-
-	return nil
-}
-
-func (c *Cache) Sync() error {
-	_logger := getLogger()
-	torrents, err := c.client.GetTorrents()
-	if err != nil {
-		return fmt.Errorf("failed to sync torrents: %v", err)
-	}
-	_logger.Info().Msgf("Syncing %d torrents", len(torrents))
-
-	// Calculate optimal workers - balance between CPU and IO
-	workers := runtime.NumCPU() * 4 // A more balanced multiplier for BadgerDB
-
-	// Create channels with appropriate buffering
-	workChan := make(chan *torrent.Torrent, workers*2)
-
-	// Use an atomic counter for progress tracking
-	var processed int64
-	var errorCount int64
-
-	// Create a context with cancellation in case of critical errors
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// Create a wait group for workers
-	var wg sync.WaitGroup
-
-	// Start workers
-	for i := 0; i < workers; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for {
-				select {
-				case t, ok := <-workChan:
-					if !ok {
-						return // Channel closed, exit goroutine
-					}
-
-					if err := c.processTorrent(t); err != nil {
-						_logger.Error().Err(err).Str("torrent", t.Name).Msg("sync error")
-						atomic.AddInt64(&errorCount, 1)
-					}
-
-					count := atomic.AddInt64(&processed, 1)
-					if count%1000 == 0 {
-						_logger.Info().Msgf("Progress: %d/%d torrents processed", count, len(torrents))
-					}
-
-				case <-ctx.Done():
-					return // Context cancelled, exit goroutine
-				}
-			}
-		}()
-	}
-
-	// Feed work to workers
-	for _, t := range torrents {
-		select {
-		case workChan <- t:
-			// Work sent successfully
-		case <-ctx.Done():
-			break // Context cancelled
-		}
-	}
-
-	// Signal workers that no more work is coming
-	close(workChan)
-
-	// Wait for all workers to complete
-	wg.Wait()
-
-	_logger.Info().Msgf("Sync complete: %d torrents processed, %d errors", len(torrents), errorCount)
-	return nil
-}
-
-func (c *Cache) processTorrent(t *torrent.Torrent) error {
-	if ct := c.GetTorrent(t.Id); ct != nil {
-		if ct.IsComplete {
-			return nil
-		}
-	}
-	c.AddTorrent(t)
-	return nil
-}
-
-func (c *Cache) AddTorrent(t *torrent.Torrent) {
-	_logger := getLogger()
-
-	if len(t.Files) == 0 {
-		tNew, err := c.client.GetTorrent(t)
-		_logger.Debug().Msgf("Getting torrent files for %s", t.Id)
-		if err != nil {
-			_logger.Debug().Msgf("Failed to get torrent files for %s: %v", t.Id, err)
-			return
-		}
-		t = tNew
-	}
-
-	if len(t.Files) == 0 {
-		_logger.Debug().Msgf("No files found for %s", t.Id)
-		return
-	}
-
-	ct := &CachedTorrent{
-		Torrent:       t,
-		LastRead:      time.Now(),
-		IsComplete:    len(t.Files) > 0,
-		DownloadLinks: make(map[string]DownloadLinkCache),
-	}
-
-	c.SetTorrent(ct)
-	c.SetTorrentName(t.Name, ct)
-
-	go func() {
-		if err := c.SaveTorrent(ct); err != nil {
-			_logger.Debug().Err(err).Msgf("Failed to save torrent %s", t.Id)
-		}
-	}()
-}
-
-func (c *Cache) RefreshTorrent(torrent *CachedTorrent) *CachedTorrent {
-	_logger := getLogger()
-
-	t, err := c.client.GetTorrent(torrent.Torrent)
-	if err != nil {
-		_logger.Debug().Msgf("Failed to get torrent files for %s: %v", torrent.Id, err)
-		return nil
-	}
-	if len(t.Files) == 0 {
-		return nil
-	}
-
-	ct := &CachedTorrent{
-		Torrent:       t,
-		LastRead:      time.Now(),
-		IsComplete:    len(t.Files) > 0,
-		DownloadLinks: make(map[string]DownloadLinkCache),
-	}
-
-	c.SetTorrent(ct)
-	c.SetTorrentName(t.Name, ct)
-
-	go func() {
-		if err := c.SaveTorrent(ct); err != nil {
-			_logger.Debug().Err(err).Msgf("Failed to save torrent %s", t.Id)
-		}
-	}()
-
-	return ct
-}
-
-func (c *Cache) GetFileDownloadLink(t *CachedTorrent, file *torrent.File) (string, error) {
-	_logger := getLogger()
-
-	if linkCache, ok := t.DownloadLinks[file.Id]; ok {
-		return linkCache.Link, nil
-	}
-
-	if file.Link == "" {
-		t = c.RefreshTorrent(t)
-		if t == nil {
-			return "", fmt.Errorf("torrent not found")
-		}
-		file = t.Torrent.GetFile(file.Id)
-	}
-
-	_logger.Debug().Msgf("Getting download link for %s", t.Name)
-	link := c.client.GetDownloadLink(t.Torrent, file)
-	if link == nil {
-		return "", fmt.Errorf("download link not found")
-	}
-
-	t.DownloadLinks[file.Id] = DownloadLinkCache{
-		Link: link.DownloadLink,
-	}
-
-	go func() {
-		if err := c.SaveTorrent(t); err != nil {
-			_logger.Debug().Err(err).Msgf("Failed to save torrent %s", t.Id)
-		}
-	}()
-
-	return link.DownloadLink, nil
-}

pkg/debrid/debrid.go

@@ -1,13 +1,12 @@
 package debrid
 
 import (
-	"cmp"
 	"fmt"
-	"github.com/sirrobot01/debrid-blackhole/internal/cache"
 	"github.com/sirrobot01/debrid-blackhole/internal/config"
 	"github.com/sirrobot01/debrid-blackhole/internal/utils"
 	"github.com/sirrobot01/debrid-blackhole/pkg/arr"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/alldebrid"
+	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid_link"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/engine"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/realdebrid"
@@ -17,33 +16,33 @@ import (
 
 func New() *engine.Engine {
 	cfg := config.GetConfig()
-	maxCachedSize := cmp.Or(cfg.MaxCacheSize, 1000)
-	debrids := make([]engine.Service, 0)
-	// Divide the cache size by the number of debrids
-	maxCacheSize := maxCachedSize / len(cfg.Debrids)
+	debrids := make([]debrid.Client, 0)
 
 	for _, dc := range cfg.Debrids {
-		d := createDebrid(dc, cache.New(maxCacheSize))
-		logger := d.GetLogger()
+		client := createDebridClient(dc)
+		logger := client.GetLogger()
 		logger.Info().Msg("Debrid Service started")
-		debrids = append(debrids, d)
+		debrids = append(debrids, client)
+	}
+	d := &engine.Engine{
+		Debrids:  debrids,
+		LastUsed: 0,
 	}
-	d := &engine.Engine{Debrids: debrids, LastUsed: 0}
 	return d
 }
 
-func createDebrid(dc config.Debrid, cache *cache.Cache) engine.Service {
+func createDebridClient(dc config.Debrid) debrid.Client {
 	switch dc.Name {
 	case "realdebrid":
-		return realdebrid.New(dc, cache)
+		return realdebrid.New(dc)
 	case "torbox":
-		return torbox.New(dc, cache)
+		return torbox.New(dc)
 	case "debridlink":
-		return debrid_link.New(dc, cache)
+		return debrid_link.New(dc)
 	case "alldebrid":
-		return alldebrid.New(dc, cache)
+		return alldebrid.New(dc)
 	default:
-		return realdebrid.New(dc, cache)
+		return realdebrid.New(dc)
 	}
 }
 
@@ -55,6 +54,7 @@ func ProcessTorrent(d *engine.Engine, magnet *utils.Magnet, a *arr.Arr, isSymlin
 		Name:     magnet.Name,
 		Arr:      a,
 		Size:     magnet.Size,
+		Files:    make(map[string]torrent.File),
 	}
 
 	errs := make([]error, 0)

pkg/debrid/debrid/debrid.go

@@ -1,22 +1,24 @@
-package engine
+package debrid
 
 import (
 	"github.com/rs/zerolog"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent"
 )
 
-type Service interface {
+type Client interface {
 	SubmitMagnet(tr *torrent.Torrent) (*torrent.Torrent, error)
 	CheckStatus(tr *torrent.Torrent, isSymlink bool) (*torrent.Torrent, error)
-	GetDownloadLinks(tr *torrent.Torrent) error
-	GetDownloadLink(tr *torrent.Torrent, file *torrent.File) *torrent.DownloadLinks
+	GenerateDownloadLinks(tr *torrent.Torrent) error
+	GetDownloadLink(tr *torrent.Torrent, file *torrent.File) *torrent.File
+	ConvertLinksToFiles(links []string) []torrent.File
 	DeleteTorrent(tr *torrent.Torrent)
 	IsAvailable(infohashes []string) map[string]bool
 	GetCheckCached() bool
 	GetDownloadUncached() bool
-	GetTorrent(torrent *torrent.Torrent) (*torrent.Torrent, error)
+	UpdateTorrent(torrent *torrent.Torrent) error
 	GetTorrents() ([]*torrent.Torrent, error)
 	GetName() string
 	GetLogger() zerolog.Logger
 	GetDownloadingStatus() []string
+	GetDownloads() (map[string]torrent.DownloadLinks, error)
 }

pkg/debrid/debrid_link/debrid_link.go

@@ -2,19 +2,18 @@ package debrid_link
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"github.com/rs/zerolog"
-	"github.com/sirrobot01/debrid-blackhole/internal/cache"
 	"github.com/sirrobot01/debrid-blackhole/internal/config"
 	"github.com/sirrobot01/debrid-blackhole/internal/logger"
 	"github.com/sirrobot01/debrid-blackhole/internal/request"
 	"github.com/sirrobot01/debrid-blackhole/internal/utils"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent"
 	"slices"
+	"time"
 
 	"net/http"
-	"os"
 	"strings"
 )
 
@@ -23,11 +22,11 @@ type DebridLink struct {
 	Host             string `json:"host"`
 	APIKey           string
 	DownloadUncached bool
-	client           *request.RLHTTPClient
-	cache            *cache.Cache
-	MountPath        string
-	logger           zerolog.Logger
-	CheckCached      bool
+	client           *request.Client
+
+	MountPath   string
+	logger      zerolog.Logger
+	CheckCached bool
 }
 
 func (dl *DebridLink) GetName() string {
@@ -38,15 +37,9 @@ func (dl *DebridLink) GetLogger() zerolog.Logger {
 	return dl.logger
 }
 
-func (dl *DebridLink) IsAvailable(infohashes []string) map[string]bool {
+func (dl *DebridLink) IsAvailable(hashes []string) map[string]bool {
 	// Check if the infohashes are available in the local cache
-	hashes, result := torrent.GetLocalCache(infohashes, dl.cache)
-
-	if len(hashes) == 0 {
-		// Either all the infohashes are locally cached or none are
-		dl.cache.AddMultiple(result)
-		return result
-	}
+	result := make(map[string]bool)
 
 	// Divide hashes into groups of 100
 	for i := 0; i < len(hashes); i += 100 {
@@ -93,32 +86,31 @@ func (dl *DebridLink) IsAvailable(infohashes []string) map[string]bool {
 			}
 		}
 	}
-	dl.cache.AddMultiple(result) // Add the results to the cache
 	return result
 }
 
-func (dl *DebridLink) GetTorrent(t *torrent.Torrent) (*torrent.Torrent, error) {
+func (dl *DebridLink) UpdateTorrent(t *torrent.Torrent) error {
 	url := fmt.Sprintf("%s/seedbox/list?ids=%s", dl.Host, t.Id)
 	req, _ := http.NewRequest(http.MethodGet, url, nil)
 	resp, err := dl.client.MakeRequest(req)
 	if err != nil {
-		return t, err
+		return err
 	}
 	var res TorrentInfo
 	err = json.Unmarshal(resp, &res)
 	if err != nil {
-		return t, err
+		return err
 	}
 	if !res.Success {
-		return t, fmt.Errorf("error getting torrent")
+		return fmt.Errorf("error getting torrent")
 	}
 	if res.Value == nil {
-		return t, fmt.Errorf("torrent not found")
+		return fmt.Errorf("torrent not found")
 	}
 	dt := *res.Value
 
 	if len(dt) == 0 {
-		return t, fmt.Errorf("torrent not found")
+		return fmt.Errorf("torrent not found")
 	}
 	data := dt[0]
 	status := "downloading"
@@ -136,21 +128,22 @@ func (dl *DebridLink) GetTorrent(t *torrent.Torrent) (*torrent.Torrent, error) {
 	t.Seeders = data.PeersConnected
 	t.Filename = name
 	t.OriginalFilename = name
-	files := make([]torrent.File, len(data.Files))
 	cfg := config.GetConfig()
-	for i, f := range data.Files {
+	for _, f := range data.Files {
 		if !cfg.IsSizeAllowed(f.Size) {
 			continue
 		}
-		files[i] = torrent.File{
-			Id:   f.ID,
-			Name: f.Name,
-			Size: f.Size,
-			Path: f.Name,
+		file := torrent.File{
+			Id:           f.ID,
+			Name:         f.Name,
+			Size:         f.Size,
+			Path:         f.Name,
+			DownloadLink: f.DownloadURL,
+			Link:         f.DownloadURL,
 		}
+		t.Files[f.Name] = file
 	}
-	t.Files = files
-	return t, nil
+	return nil
 }
 
 func (dl *DebridLink) SubmitMagnet(t *torrent.Torrent) (*torrent.Torrent, error) {
@@ -185,33 +178,32 @@ func (dl *DebridLink) SubmitMagnet(t *torrent.Torrent) (*torrent.Torrent, error)
 	t.OriginalFilename = name
 	t.MountPath = dl.MountPath
 	t.Debrid = dl.Name
-	t.DownloadLinks = make(map[string]torrent.DownloadLinks)
-	files := make([]torrent.File, len(data.Files))
-	for i, f := range data.Files {
-		files[i] = torrent.File{
-			Id:   f.ID,
-			Name: f.Name,
-			Size: f.Size,
-			Path: f.Name,
-			Link: f.DownloadURL,
+	for _, f := range data.Files {
+		file := torrent.File{
+			Id:           f.ID,
+			Name:         f.Name,
+			Size:         f.Size,
+			Path:         f.Name,
+			Link:         f.DownloadURL,
+			DownloadLink: f.DownloadURL,
+			Generated:    time.Now(),
 		}
+		t.Files[f.Name] = file
 	}
-	t.Files = files
 
 	return t, nil
 }
 
 func (dl *DebridLink) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*torrent.Torrent, error) {
 	for {
-		t, err := dl.GetTorrent(torrent)
-		torrent = t
+		err := dl.UpdateTorrent(torrent)
 		if err != nil || torrent == nil {
 			return torrent, err
 		}
 		status := torrent.Status
 		if status == "downloaded" {
 			dl.logger.Info().Msgf("Torrent: %s downloaded", torrent.Name)
-			err = dl.GetDownloadLinks(torrent)
+			err = dl.GenerateDownloadLinks(torrent)
 			if err != nil {
 				return torrent, err
 			}
@@ -242,25 +234,16 @@ func (dl *DebridLink) DeleteTorrent(torrent *torrent.Torrent) {
 	}
 }
 
-func (dl *DebridLink) GetDownloadLinks(t *torrent.Torrent) error {
-	downloadLinks := make(map[string]torrent.DownloadLinks)
-	for _, f := range t.Files {
-		dl := torrent.DownloadLinks{
-			Link:     f.Link,
-			Filename: f.Name,
-		}
-		downloadLinks[f.Id] = dl
-	}
-	t.DownloadLinks = downloadLinks
+func (dl *DebridLink) GenerateDownloadLinks(t *torrent.Torrent) error {
 	return nil
 }
 
-func (dl *DebridLink) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.DownloadLinks {
-	dlLink, ok := t.DownloadLinks[file.Id]
-	if !ok {
-		return nil
-	}
-	return &dlLink
+func (dl *DebridLink) GetDownloads() (map[string]torrent.DownloadLinks, error) {
+	return nil, nil
+}
+
+func (dl *DebridLink) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.File {
+	return file
 }
 
 func (dl *DebridLink) GetDownloadingStatus() []string {
@@ -275,22 +258,24 @@ func (dl *DebridLink) GetDownloadUncached() bool {
 	return dl.DownloadUncached
 }
 
-func New(dc config.Debrid, cache *cache.Cache) *DebridLink {
+func New(dc config.Debrid) *DebridLink {
 	rl := request.ParseRateLimit(dc.RateLimit)
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 		"Content-Type":  "application/json",
 	}
-	client := request.NewRLHTTPClient(rl, headers)
+	_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
+	client := request.New().
+		WithHeaders(headers).
+		WithRateLimiter(rl).WithLogger(_log)
 	return &DebridLink{
 		Name:             "debridlink",
 		Host:             dc.Host,
 		APIKey:           dc.APIKey,
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
-		cache:            cache,
 		MountPath:        dc.Folder,
-		logger:           logger.NewLogger(dc.Name, config.GetConfig().LogLevel, os.Stdout),
+		logger:           logger.NewLogger(dc.Name, config.GetConfig().LogLevel),
 		CheckCached:      dc.CheckCached,
 	}
 }
@@ -298,3 +283,7 @@ func New(dc config.Debrid, cache *cache.Cache) *DebridLink {
 func (dl *DebridLink) GetTorrents() ([]*torrent.Torrent, error) {
 	return nil, nil
 }
+
+func (dl *DebridLink) ConvertLinksToFiles(links []string) []torrent.File {
+	return nil
+}

pkg/debrid/engine/engine.go

@@ -1,18 +1,22 @@
 package engine
 
+import (
+	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid"
+)
+
 type Engine struct {
-	Debrids  []Service
+	Debrids  []debrid.Client
 	LastUsed int
 }
 
-func (d *Engine) Get() Service {
+func (d *Engine) Get() debrid.Client {
 	if d.LastUsed == 0 {
 		return d.Debrids[0]
 	}
 	return d.Debrids[d.LastUsed]
 }
 
-func (d *Engine) GetByName(name string) Service {
+func (d *Engine) GetByName(name string) debrid.Client {
 	for _, deb := range d.Debrids {
 		if deb.GetName() == name {
 			return deb
@@ -21,6 +25,6 @@ func (d *Engine) GetByName(name string) Service {
 	return nil
 }
 
-func (d *Engine) GetDebrids() []Service {
+func (d *Engine) GetDebrids() []debrid.Client {
 	return d.Debrids
 }

pkg/debrid/realdebrid/realdebrid.go

@@ -1,22 +1,23 @@
 package realdebrid
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"github.com/rs/zerolog"
-	"github.com/sirrobot01/debrid-blackhole/internal/cache"
 	"github.com/sirrobot01/debrid-blackhole/internal/config"
 	"github.com/sirrobot01/debrid-blackhole/internal/logger"
 	"github.com/sirrobot01/debrid-blackhole/internal/request"
 	"github.com/sirrobot01/debrid-blackhole/internal/utils"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent"
+	"io"
 	"net/http"
 	gourl "net/url"
-	"os"
 	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
+	"time"
 )
 
 type RealDebrid struct {
@@ -24,11 +25,11 @@ type RealDebrid struct {
 	Host             string `json:"host"`
 	APIKey           string
 	DownloadUncached bool
-	client           *request.RLHTTPClient
-	cache            *cache.Cache
-	MountPath        string
-	logger           zerolog.Logger
-	CheckCached      bool
+	client           *request.Client
+
+	MountPath   string
+	logger      zerolog.Logger
+	CheckCached bool
 }
 
 func (r *RealDebrid) GetName() string {
@@ -39,11 +40,11 @@ func (r *RealDebrid) GetLogger() zerolog.Logger {
 	return r.logger
 }
 
-// GetTorrentFiles returns a list of torrent files from the torrent info
+// getTorrentFiles returns a list of torrent files from the torrent info
 // validate is used to determine if the files should be validated
 // if validate is false, selected files will be returned
-func GetTorrentFiles(data TorrentInfo, validate bool) []torrent.File {
-	files := make([]torrent.File, 0)
+func getTorrentFiles(t *torrent.Torrent, data TorrentInfo, validate bool) map[string]torrent.File {
+	files := make(map[string]torrent.File)
 	cfg := config.GetConfig()
 	idx := 0
 	for _, f := range data.Files {
@@ -72,6 +73,13 @@ func GetTorrentFiles(data TorrentInfo, validate bool) []torrent.File {
 		if len(data.Links) > idx {
 			_link = data.Links[idx]
 		}
+
+		if a, ok := t.Files[name]; ok {
+			a.Link = _link
+			files[name] = a
+			continue
+		}
+
 		file := torrent.File{
 			Name: name,
 			Path: name,
@@ -79,21 +87,15 @@ func GetTorrentFiles(data TorrentInfo, validate bool) []torrent.File {
 			Id:   strconv.Itoa(fileId),
 			Link: _link,
 		}
-		files = append(files, file)
+		files[name] = file
 		idx++
 	}
 	return files
 }
 
-func (r *RealDebrid) IsAvailable(infohashes []string) map[string]bool {
+func (r *RealDebrid) IsAvailable(hashes []string) map[string]bool {
 	// Check if the infohashes are available in the local cache
-	hashes, result := torrent.GetLocalCache(infohashes, r.cache)
-
-	if len(hashes) == 0 {
-		// Either all the infohashes are locally cached or none are
-		r.cache.AddMultiple(result)
-		return result
-	}
+	result := make(map[string]bool)
 
 	// Divide hashes into groups of 100
 	for i := 0; i < len(hashes); i += 200 {
@@ -136,7 +138,6 @@ func (r *RealDebrid) IsAvailable(infohashes []string) map[string]bool {
 			}
 		}
 	}
-	r.cache.AddMultiple(result) // Add the results to the cache
 	return result
 }
 
@@ -160,17 +161,17 @@ func (r *RealDebrid) SubmitMagnet(t *torrent.Torrent) (*torrent.Torrent, error)
 	return t, nil
 }
 
-func (r *RealDebrid) GetTorrent(t *torrent.Torrent) (*torrent.Torrent, error) {
+func (r *RealDebrid) UpdateTorrent(t *torrent.Torrent) error {
 	url := fmt.Sprintf("%s/torrents/info/%s", r.Host, t.Id)
 	req, _ := http.NewRequest(http.MethodGet, url, nil)
 	resp, err := r.client.MakeRequest(req)
 	if err != nil {
-		return t, err
+		return err
 	}
 	var data TorrentInfo
 	err = json.Unmarshal(resp, &data)
 	if err != nil {
-		return t, err
+		return err
 	}
 	name := utils.RemoveInvalidChars(data.OriginalFilename)
 	t.Name = name
@@ -185,10 +186,8 @@ func (r *RealDebrid) GetTorrent(t *torrent.Torrent) (*torrent.Torrent, error) {
 	t.Links = data.Links
 	t.MountPath = r.MountPath
 	t.Debrid = r.Name
-	t.DownloadLinks = make(map[string]torrent.DownloadLinks)
-	files := GetTorrentFiles(data, false) // Get selected files
-	t.Files = files
-	return t, nil
+	t.Files = getTorrentFiles(t, data, false) // Get selected files
+	return nil
 }
 
 func (r *RealDebrid) CheckStatus(t *torrent.Torrent, isSymlink bool) (*torrent.Torrent, error) {
@@ -219,13 +218,12 @@ func (r *RealDebrid) CheckStatus(t *torrent.Torrent, isSymlink bool) (*torrent.T
 		t.Debrid = r.Name
 		t.MountPath = r.MountPath
 		if status == "waiting_files_selection" {
-			files := GetTorrentFiles(data, true) // Validate files to be selected
-			t.Files = files
-			if len(files) == 0 {
+			t.Files = getTorrentFiles(t, data, true)
+			if len(t.Files) == 0 {
 				return t, fmt.Errorf("no video files found")
 			}
 			filesId := make([]string, 0)
-			for _, f := range files {
+			for _, f := range t.Files {
 				filesId = append(filesId, f.Id)
 			}
 			p := gourl.Values{
@@ -238,11 +236,10 @@ func (r *RealDebrid) CheckStatus(t *torrent.Torrent, isSymlink bool) (*torrent.T
 				return t, err
 			}
 		} else if status == "downloaded" {
-			files := GetTorrentFiles(data, false) // Get selected files
-			t.Files = files
+			t.Files = getTorrentFiles(t, data, false) // Get selected files
 			r.logger.Info().Msgf("Torrent: %s downloaded to RD", t.Name)
 			if !isSymlink {
-				err = r.GetDownloadLinks(t)
+				err = r.GenerateDownloadLinks(t)
 				if err != nil {
 					return t, err
 				}
@@ -271,12 +268,11 @@ func (r *RealDebrid) DeleteTorrent(torrent *torrent.Torrent) {
 	}
 }
 
-func (r *RealDebrid) GetDownloadLinks(t *torrent.Torrent) error {
+func (r *RealDebrid) GenerateDownloadLinks(t *torrent.Torrent) error {
 	url := fmt.Sprintf("%s/unrestrict/link/", r.Host)
-	downloadLinks := make(map[string]torrent.DownloadLinks)
 	for _, f := range t.Files {
-		dlLink := t.DownloadLinks[f.Id]
-		if f.Link == "" || dlLink.DownloadLink != "" {
+		if f.DownloadLink != "" {
+			// Or check the generated link
 			continue
 		}
 		payload := gourl.Values{
@@ -291,18 +287,41 @@ func (r *RealDebrid) GetDownloadLinks(t *torrent.Torrent) error {
 		if err = json.Unmarshal(resp, &data); err != nil {
 			return err
 		}
-		download := torrent.DownloadLinks{
-			Link:         data.Link,
-			Filename:     data.Filename,
-			DownloadLink: data.Download,
-		}
-		downloadLinks[f.Id] = download
+		f.DownloadLink = data.Download
+		f.Generated = time.Now()
+		t.Files[f.Name] = f
 	}
-	t.DownloadLinks = downloadLinks
 	return nil
 }
 
-func (r *RealDebrid) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.DownloadLinks {
+func (r *RealDebrid) ConvertLinksToFiles(links []string) []torrent.File {
+	files := make([]torrent.File, 0)
+	for _, l := range links {
+		url := fmt.Sprintf("%s/unrestrict/link/", r.Host)
+		payload := gourl.Values{
+			"link": {l},
+		}
+		req, _ := http.NewRequest(http.MethodPost, url, strings.NewReader(payload.Encode()))
+		resp, err := r.client.MakeRequest(req)
+		if err != nil {
+			continue
+		}
+		var data UnrestrictResponse
+		if err = json.Unmarshal(resp, &data); err != nil {
+			continue
+		}
+		files = append(files, torrent.File{
+			Name:         data.Filename,
+			Size:         data.Filesize,
+			Link:         l,
+			DownloadLink: data.Download,
+			Generated:    time.Now(),
+		})
+	}
+	return files
+}
+
+func (r *RealDebrid) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.File {
 	url := fmt.Sprintf("%s/unrestrict/link/", r.Host)
 	payload := gourl.Values{
 		"link": {file.Link},
@@ -316,32 +335,43 @@ func (r *RealDebrid) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *to
 	if err = json.Unmarshal(resp, &data); err != nil {
 		return nil
 	}
-	return &torrent.DownloadLinks{
-		Link:         data.Link,
-		Filename:     data.Filename,
-		DownloadLink: data.Download,
-	}
+	file.DownloadLink = data.Download
+	file.Generated = time.Now()
+	return file
 }
 
 func (r *RealDebrid) GetCheckCached() bool {
 	return r.CheckCached
 }
 
-func (r *RealDebrid) getTorrents(offset int, limit int) ([]*torrent.Torrent, error) {
+func (r *RealDebrid) getTorrents(offset int, limit int) (int, []*torrent.Torrent, error) {
 	url := fmt.Sprintf("%s/torrents?limit=%d", r.Host, limit)
+	torrents := make([]*torrent.Torrent, 0)
 	if offset > 0 {
 		url = fmt.Sprintf("%s&offset=%d", url, offset)
 	}
 	req, _ := http.NewRequest(http.MethodGet, url, nil)
-	resp, err := r.client.MakeRequest(req)
+	resp, err := r.client.Do(req)
+
 	if err != nil {
-		return nil, err
+		return 0, torrents, err
 	}
+
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return 0, torrents, fmt.Errorf("realdebrid API error: %d", resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return 0, torrents, err
+	}
+	totalItems, _ := strconv.Atoi(resp.Header.Get("X-Total-Count"))
 	var data []TorrentsResponse
-	if err = json.Unmarshal(resp, &data); err != nil {
-		return nil, err
+	if err = json.Unmarshal(body, &data); err != nil {
+		return 0, nil, err
 	}
-	torrents := make([]*torrent.Torrent, 0)
 	filenames := map[string]bool{}
 	for _, t := range data {
 		if _, exists := filenames[t.Filename]; exists {
@@ -356,20 +386,122 @@ func (r *RealDebrid) getTorrents(offset int, limit int) ([]*torrent.Torrent, err
 			Filename:         t.Filename,
 			OriginalFilename: t.Filename,
 			Links:            t.Links,
+			Files:            make(map[string]torrent.File),
+			InfoHash:         t.Hash,
+			Debrid:           r.Name,
+			MountPath:        r.MountPath,
 		})
 	}
-	return torrents, nil
+	return totalItems, torrents, nil
 }
 
 func (r *RealDebrid) GetTorrents() ([]*torrent.Torrent, error) {
-	torrents := make([]*torrent.Torrent, 0)
-	offset := 0
-	limit := 1000
-	ts, _ := r.getTorrents(offset, limit)
-	torrents = append(torrents, ts...)
-	offset = len(torrents)
-	return torrents, nil
+	limit := 5000
 
+	// Get first batch and total count
+	totalItems, firstBatch, err := r.getTorrents(0, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	allTorrents := firstBatch
+
+	// Calculate remaining requests
+	remaining := totalItems - len(firstBatch)
+	if remaining <= 0 {
+		return allTorrents, nil
+	}
+
+	// Prepare for concurrent fetching
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	var fetchError error
+
+	// Calculate how many more requests we need
+	batchCount := (remaining + limit - 1) / limit // ceiling division
+
+	for i := 1; i <= batchCount; i++ {
+		wg.Add(1)
+		go func(batchOffset int) {
+			defer wg.Done()
+
+			_, batch, err := r.getTorrents(batchOffset, limit)
+			if err != nil {
+				mu.Lock()
+				fetchError = err
+				mu.Unlock()
+				return
+			}
+
+			mu.Lock()
+			allTorrents = append(allTorrents, batch...)
+			mu.Unlock()
+		}(i * limit)
+	}
+
+	// Wait for all fetches to complete
+	wg.Wait()
+
+	if fetchError != nil {
+		return nil, fetchError
+	}
+
+	return allTorrents, nil
+}
+
+func (r *RealDebrid) GetDownloads() (map[string]torrent.DownloadLinks, error) {
+	links := make(map[string]torrent.DownloadLinks)
+	offset := 0
+	limit := 5000
+	for {
+		dl, err := r._getDownloads(offset, limit)
+		if err != nil {
+			break
+		}
+		if len(dl) == 0 {
+			break
+		}
+
+		for _, d := range dl {
+			if _, exists := links[d.Link]; exists {
+				// This is ordered by date, so we can skip the rest
+				continue
+			}
+			links[d.Link] = d
+		}
+
+		offset += len(dl)
+	}
+	return links, nil
+}
+
+func (r *RealDebrid) _getDownloads(offset int, limit int) ([]torrent.DownloadLinks, error) {
+	url := fmt.Sprintf("%s/downloads?limit=%d", r.Host, limit)
+	if offset > 0 {
+		url = fmt.Sprintf("%s&offset=%d", url, offset)
+	}
+	req, _ := http.NewRequest(http.MethodGet, url, nil)
+	resp, err := r.client.MakeRequest(req)
+	if err != nil {
+		return nil, err
+	}
+	var data []DownloadsResponse
+	if err = json.Unmarshal(resp, &data); err != nil {
+		return nil, err
+	}
+	links := make([]torrent.DownloadLinks, 0)
+	for _, d := range data {
+		links = append(links, torrent.DownloadLinks{
+			Filename:     d.Filename,
+			Size:         d.Filesize,
+			Link:         d.Link,
+			DownloadLink: d.Download,
+			Generated:    d.Generated,
+			Id:           d.Id,
+		})
+
+	}
+	return links, nil
 }
 
 func (r *RealDebrid) GetDownloadingStatus() []string {
@@ -380,21 +512,23 @@ func (r *RealDebrid) GetDownloadUncached() bool {
 	return r.DownloadUncached
 }
 
-func New(dc config.Debrid, cache *cache.Cache) *RealDebrid {
+func New(dc config.Debrid) *RealDebrid {
 	rl := request.ParseRateLimit(dc.RateLimit)
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 	}
-	client := request.NewRLHTTPClient(rl, headers)
+	_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
+	client := request.New().
+		WithHeaders(headers).
+		WithRateLimiter(rl).WithLogger(_log)
 	return &RealDebrid{
 		Name:             "realdebrid",
 		Host:             dc.Host,
 		APIKey:           dc.APIKey,
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
-		cache:            cache,
 		MountPath:        dc.Folder,
-		logger:           logger.NewLogger(dc.Name, config.GetConfig().LogLevel, os.Stdout),
+		logger:           logger.NewLogger(dc.Name, config.GetConfig().LogLevel),
 		CheckCached:      dc.CheckCached,
 	}
 }

pkg/debrid/realdebrid/types.go

@@ -1,8 +1,8 @@
 package realdebrid
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"time"
 )
 
@@ -98,7 +98,7 @@ type UnrestrictResponse struct {
 	Id         string `json:"id"`
 	Filename   string `json:"filename"`
 	MimeType   string `json:"mimeType"`
-	Filesize   int    `json:"filesize"`
+	Filesize   int64  `json:"filesize"`
 	Link       string `json:"link"`
 	Host       string `json:"host"`
 	Chunks     int    `json:"chunks"`
@@ -120,3 +120,17 @@ type TorrentsResponse struct {
 	Links    []string  `json:"links"`
 	Ended    time.Time `json:"ended"`
 }
+
+type DownloadsResponse struct {
+	Id         string    `json:"id"`
+	Filename   string    `json:"filename"`
+	MimeType   string    `json:"mimeType"`
+	Filesize   int64     `json:"filesize"`
+	Link       string    `json:"link"`
+	Host       string    `json:"host"`
+	HostIcon   string    `json:"host_icon"`
+	Chunks     int64     `json:"chunks"`
+	Download   string    `json:"download"`
+	Streamable int       `json:"streamable"`
+	Generated  time.Time `json:"generated"`
+}

pkg/debrid/torbox/torbox.go

@@ -2,20 +2,19 @@ package torbox
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"github.com/rs/zerolog"
-	"github.com/sirrobot01/debrid-blackhole/internal/cache"
 	"github.com/sirrobot01/debrid-blackhole/internal/config"
 	"github.com/sirrobot01/debrid-blackhole/internal/logger"
 	"github.com/sirrobot01/debrid-blackhole/internal/request"
 	"github.com/sirrobot01/debrid-blackhole/internal/utils"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent"
+	"time"
 
 	"mime/multipart"
 	"net/http"
 	gourl "net/url"
-	"os"
 	"path"
 	"path/filepath"
 	"slices"
@@ -28,11 +27,11 @@ type Torbox struct {
 	Host             string `json:"host"`
 	APIKey           string
 	DownloadUncached bool
-	client           *request.RLHTTPClient
-	cache            *cache.Cache
-	MountPath        string
-	logger           zerolog.Logger
-	CheckCached      bool
+	client           *request.Client
+
+	MountPath   string
+	logger      zerolog.Logger
+	CheckCached bool
 }
 
 func (tb *Torbox) GetName() string {
@@ -43,15 +42,9 @@ func (tb *Torbox) GetLogger() zerolog.Logger {
 	return tb.logger
 }
 
-func (tb *Torbox) IsAvailable(infohashes []string) map[string]bool {
+func (tb *Torbox) IsAvailable(hashes []string) map[string]bool {
 	// Check if the infohashes are available in the local cache
-	hashes, result := torrent.GetLocalCache(infohashes, tb.cache)
-
-	if len(hashes) == 0 {
-		// Either all the infohashes are locally cached or none are
-		tb.cache.AddMultiple(result)
-		return result
-	}
+	result := make(map[string]bool)
 
 	// Divide hashes into groups of 100
 	for i := 0; i < len(hashes); i += 100 {
@@ -91,13 +84,12 @@ func (tb *Torbox) IsAvailable(infohashes []string) map[string]bool {
 			return result
 		}
 
-		for h, cache := range *res.Data {
-			if cache.Size > 0 {
+		for h, c := range *res.Data {
+			if c.Size > 0 {
 				result[strings.ToUpper(h)] = true
 			}
 		}
 	}
-	tb.cache.AddMultiple(result) // Add the results to the cache
 	return result
 }
 
@@ -149,17 +141,17 @@ func getTorboxStatus(status string, finished bool) string {
 	}
 }
 
-func (tb *Torbox) GetTorrent(t *torrent.Torrent) (*torrent.Torrent, error) {
+func (tb *Torbox) UpdateTorrent(t *torrent.Torrent) error {
 	url := fmt.Sprintf("%s/api/torrents/mylist/?id=%s", tb.Host, t.Id)
 	req, _ := http.NewRequest(http.MethodGet, url, nil)
 	resp, err := tb.client.MakeRequest(req)
 	if err != nil {
-		return t, err
+		return err
 	}
 	var res InfoResponse
 	err = json.Unmarshal(resp, &res)
 	if err != nil {
-		return t, err
+		return err
 	}
 	data := res.Data
 	name := data.Name
@@ -174,8 +166,6 @@ func (tb *Torbox) GetTorrent(t *torrent.Torrent) (*torrent.Torrent, error) {
 	t.OriginalFilename = name
 	t.MountPath = tb.MountPath
 	t.Debrid = tb.Name
-	t.DownloadLinks = make(map[string]torrent.DownloadLinks)
-	files := make([]torrent.File, 0)
 	cfg := config.GetConfig()
 	for _, f := range data.Files {
 		fileName := filepath.Base(f.Name)
@@ -196,35 +186,32 @@ func (tb *Torbox) GetTorrent(t *torrent.Torrent) (*torrent.Torrent, error) {
 			Size: f.Size,
 			Path: fileName,
 		}
-		files = append(files, file)
+		t.Files[fileName] = file
 	}
 	var cleanPath string
-	if len(files) > 0 {
+	if len(t.Files) > 0 {
 		cleanPath = path.Clean(data.Files[0].Name)
 	} else {
 		cleanPath = path.Clean(data.Name)
 	}
 
 	t.OriginalFilename = strings.Split(cleanPath, "/")[0]
-	t.Files = files
-	//t.Debrid = tb
-	return t, nil
+	t.Debrid = tb.Name
+	return nil
 }
 
 func (tb *Torbox) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*torrent.Torrent, error) {
 	for {
-		t, err := tb.GetTorrent(torrent)
+		err := tb.UpdateTorrent(torrent)
 
-		torrent = t
-
-		if err != nil || t == nil {
-			return t, err
+		if err != nil || torrent == nil {
+			return torrent, err
 		}
 		status := torrent.Status
 		if status == "downloaded" {
 			tb.logger.Info().Msgf("Torrent: %s downloaded", torrent.Name)
 			if !isSymlink {
-				err = tb.GetDownloadLinks(torrent)
+				err = tb.GenerateDownloadLinks(torrent)
 				if err != nil {
 					return torrent, err
 				}
@@ -258,8 +245,7 @@ func (tb *Torbox) DeleteTorrent(torrent *torrent.Torrent) {
 	}
 }
 
-func (tb *Torbox) GetDownloadLinks(t *torrent.Torrent) error {
-	downloadLinks := make(map[string]torrent.DownloadLinks)
+func (tb *Torbox) GenerateDownloadLinks(t *torrent.Torrent) error {
 	for _, file := range t.Files {
 		url := fmt.Sprintf("%s/api/torrents/requestdl/", tb.Host)
 		query := gourl.Values{}
@@ -279,21 +265,15 @@ func (tb *Torbox) GetDownloadLinks(t *torrent.Torrent) error {
 		if data.Data == nil {
 			return fmt.Errorf("error getting download links")
 		}
-		idx := 0
 		link := *data.Data
-
-		dl := torrent.DownloadLinks{
-			Link:         link,
-			Filename:     t.Files[idx].Name,
-			DownloadLink: link,
-		}
-		downloadLinks[file.Id] = dl
+		file.DownloadLink = link
+		file.Generated = time.Now()
+		t.Files[file.Name] = file
 	}
-	t.DownloadLinks = downloadLinks
 	return nil
 }
 
-func (tb *Torbox) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.DownloadLinks {
+func (tb *Torbox) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.File {
 	url := fmt.Sprintf("%s/api/torrents/requestdl/", tb.Host)
 	query := gourl.Values{}
 	query.Add("torrent_id", t.Id)
@@ -313,11 +293,9 @@ func (tb *Torbox) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torre
 		return nil
 	}
 	link := *data.Data
-	return &torrent.DownloadLinks{
-		Link:         file.Link,
-		Filename:     file.Name,
-		DownloadLink: link,
-	}
+	file.DownloadLink = link
+	file.Generated = time.Now()
+	return file
 }
 
 func (tb *Torbox) GetDownloadingStatus() []string {
@@ -336,21 +314,32 @@ func (tb *Torbox) GetDownloadUncached() bool {
 	return tb.DownloadUncached
 }
 
-func New(dc config.Debrid, cache *cache.Cache) *Torbox {
+func New(dc config.Debrid) *Torbox {
 	rl := request.ParseRateLimit(dc.RateLimit)
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 	}
-	client := request.NewRLHTTPClient(rl, headers)
+	_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
+	client := request.New().
+		WithHeaders(headers).
+		WithRateLimiter(rl).WithLogger(_log)
+
 	return &Torbox{
 		Name:             "torbox",
 		Host:             dc.Host,
 		APIKey:           dc.APIKey,
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
-		cache:            cache,
 		MountPath:        dc.Folder,
-		logger:           logger.NewLogger(dc.Name, config.GetConfig().LogLevel, os.Stdout),
+		logger:           _log,
 		CheckCached:      dc.CheckCached,
 	}
 }
+
+func (tb *Torbox) ConvertLinksToFiles(links []string) []torrent.File {
+	return nil
+}
+
+func (tb *Torbox) GetDownloads() (map[string]torrent.DownloadLinks, error) {
+	return nil, nil
+}

pkg/debrid/torrent/torrent.go

@@ -2,34 +2,33 @@ package torrent
 
 import (
 	"fmt"
-	"github.com/sirrobot01/debrid-blackhole/internal/cache"
 	"github.com/sirrobot01/debrid-blackhole/internal/logger"
 	"github.com/sirrobot01/debrid-blackhole/internal/utils"
 	"github.com/sirrobot01/debrid-blackhole/pkg/arr"
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 )
 
 type Torrent struct {
-	Id               string                   `json:"id"`
-	InfoHash         string                   `json:"info_hash"`
-	Name             string                   `json:"name"`
-	Folder           string                   `json:"folder"`
-	Filename         string                   `json:"filename"`
-	OriginalFilename string                   `json:"original_filename"`
-	Size             int64                    `json:"size"`
-	Bytes            int64                    `json:"bytes"` // Size of only the files that are downloaded
-	Magnet           *utils.Magnet            `json:"magnet"`
-	Files            []File                   `json:"files"`
-	Status           string                   `json:"status"`
-	Added            string                   `json:"added"`
-	Progress         float64                  `json:"progress"`
-	Speed            int64                    `json:"speed"`
-	Seeders          int                      `json:"seeders"`
-	Links            []string                 `json:"links"`
-	DownloadLinks    map[string]DownloadLinks `json:"download_links"`
-	MountPath        string                   `json:"mount_path"`
+	Id               string          `json:"id"`
+	InfoHash         string          `json:"info_hash"`
+	Name             string          `json:"name"`
+	Folder           string          `json:"folder"`
+	Filename         string          `json:"filename"`
+	OriginalFilename string          `json:"original_filename"`
+	Size             int64           `json:"size"`
+	Bytes            int64           `json:"bytes"` // Size of only the files that are downloaded
+	Magnet           *utils.Magnet   `json:"magnet"`
+	Files            map[string]File `json:"files"`
+	Status           string          `json:"status"`
+	Added            string          `json:"added"`
+	Progress         float64         `json:"progress"`
+	Speed            int64           `json:"speed"`
+	Seeders          int             `json:"seeders"`
+	Links            []string        `json:"links"`
+	MountPath        string          `json:"mount_path"`
 
 	Debrid string `json:"debrid"`
 
@@ -40,9 +39,12 @@ type Torrent struct {
 }
 
 type DownloadLinks struct {
-	Filename     string `json:"filename"`
-	Link         string `json:"link"`
-	DownloadLink string `json:"download_link"`
+	Filename     string    `json:"filename"`
+	Link         string    `json:"link"`
+	DownloadLink string    `json:"download_link"`
+	Generated    time.Time `json:"generated"`
+	Size         int64     `json:"size"`
+	Id           string    `json:"id"`
 }
 
 func (t *Torrent) GetSymlinkFolder(parent string) string {
@@ -69,11 +71,13 @@ func (t *Torrent) GetMountFolder(rClonePath string) (string, error) {
 }
 
 type File struct {
-	Id   string `json:"id"`
-	Name string `json:"name"`
-	Size int64  `json:"size"`
-	Path string `json:"path"`
-	Link string `json:"link"`
+	Id           string    `json:"id"`
+	Name         string    `json:"name"`
+	Size         int64     `json:"size"`
+	Path         string    `json:"path"`
+	Link         string    `json:"link"`
+	DownloadLink string    `json:"download_link"`
+	Generated    time.Time `json:"generated"`
 }
 
 func (t *Torrent) Cleanup(remove bool) {
@@ -93,30 +97,3 @@ func (t *Torrent) GetFile(id string) *File {
 	}
 	return nil
 }
-
-func GetLocalCache(infohashes []string, cache *cache.Cache) ([]string, map[string]bool) {
-	result := make(map[string]bool)
-	hashes := make([]string, 0)
-
-	if len(infohashes) == 0 {
-		return hashes, result
-	}
-	if len(infohashes) == 1 {
-		if cache.Exists(infohashes[0]) {
-			return hashes, map[string]bool{infohashes[0]: true}
-		}
-		return infohashes, result
-	}
-
-	cachedHashes := cache.GetMultiple(infohashes)
-	for _, h := range infohashes {
-		_, exists := cachedHashes[h]
-		if !exists {
-			hashes = append(hashes, h)
-		} else {
-			result[h] = true
-		}
-	}
-
-	return infohashes, result
-}