Feature/torbox provider improvements (#100)

- Add Torbox WebDAV implementation
- Fix Issues with sample and extension checks

internal/config/misc.go

@@ -24,7 +24,7 @@ func (c *Config) IsAllowedFile(filename string) bool {
 }
 
 func getDefaultExtensions() []string {
-	videoExts := strings.Split("webm,m4v,3gp,nsv,ty,strm,rm,rmvb,m3u,ifo,mov,qt,divx,xvid,bivx,nrg,pva,wmv,asf,asx,ogm,ogv,m2v,avi,bin,dat,dvr-ms,mpg,mpeg,mp4,avc,vp3,svq3,nuv,viv,dv,fli,flv,wpl,img,iso,vob,mkv,mk3d,ts,wtv,m2ts'", ",")
+	videoExts := strings.Split("webm,m4v,3gp,nsv,ty,strm,rm,rmvb,m3u,ifo,mov,qt,divx,xvid,bivx,nrg,pva,wmv,asf,asx,ogm,ogv,m2v,avi,bin,dat,dvr-ms,mpg,mpeg,mp4,avc,vp3,svq3,nuv,viv,dv,fli,flv,wpl,img,iso,vob,mkv,mk3d,ts,wtv,m2ts", ",")
 	musicExts := strings.Split("MP3,WAV,FLAC,OGG,WMA,AIFF,ALAC,M4A,APE,AC3,DTS,M4P,MID,MIDI,MKA,MP2,MPA,RA,VOC,WV,AMR", ",")
 
 	// Combine both slices

internal/utils/regex.go

@@ -51,7 +51,8 @@ func IsMediaFile(path string) bool {
 }
 
 func IsSampleFile(path string) bool {
-	if strings.HasSuffix(strings.ToLower(path), "sample.mkv") {
+	filename := filepath.Base(path)
+	if strings.HasSuffix(strings.ToLower(filename), "sample.mkv") {
 		return true
 	}
 	return RegexMatch(sampleRegex, path)

pkg/debrid/providers/torbox/torbox.go

@@ -4,13 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/rs/zerolog"
-	"github.com/sirrobot01/decypharr/internal/config"
-	"github.com/sirrobot01/decypharr/internal/logger"
-	"github.com/sirrobot01/decypharr/internal/request"
-	"github.com/sirrobot01/decypharr/internal/utils"
-	"github.com/sirrobot01/decypharr/pkg/debrid/types"
-	"github.com/sirrobot01/decypharr/pkg/version"
 	"mime/multipart"
 	"net/http"
 	gourl "net/url"
@@ -21,6 +14,14 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/rs/zerolog"
+	"github.com/sirrobot01/decypharr/internal/config"
+	"github.com/sirrobot01/decypharr/internal/logger"
+	"github.com/sirrobot01/decypharr/internal/request"
+	"github.com/sirrobot01/decypharr/internal/utils"
+	"github.com/sirrobot01/decypharr/pkg/debrid/types"
+	"github.com/sirrobot01/decypharr/pkg/version"
 )
 
 type Torbox struct {
@@ -168,7 +169,7 @@ func (tb *Torbox) SubmitMagnet(torrent *types.Torrent) (*types.Torrent, error) {
 	return torrent, nil
 }
 
-func getTorboxStatus(status string, finished bool) string {
+func (tb *Torbox) getTorboxStatus(status string, finished bool) string {
 	if finished {
 		return "downloaded"
 	}
@@ -176,12 +177,16 @@ func getTorboxStatus(status string, finished bool) string {
 		"checkingResumeData", "metaDL", "pausedUP", "queuedUP", "checkingUP",
 		"forcedUP", "allocating", "downloading", "metaDL", "pausedDL",
 		"queuedDL", "checkingDL", "forcedDL", "checkingResumeData", "moving"}
+
+	var determinedStatus string
 	switch {
 	case utils.Contains(downloading, status):
-		return "downloading"
+		determinedStatus = "downloading"
 	default:
-		return "error"
+		determinedStatus = "error"
 	}
+
+	return determinedStatus
 }
 
 func (tb *Torbox) GetTorrent(torrentId string) (*types.Torrent, error) {
@@ -206,7 +211,7 @@ func (tb *Torbox) GetTorrent(torrentId string) (*types.Torrent, error) {
 		Bytes:            data.Size,
 		Folder:           data.Name,
 		Progress:         data.Progress * 100,
-		Status:           getTorboxStatus(data.DownloadState, data.DownloadFinished),
+		Status:           tb.getTorboxStatus(data.DownloadState, data.DownloadFinished),
 		Speed:            data.DownloadSpeed,
 		Seeders:          data.Seeds,
 		Filename:         data.Name,
@@ -217,19 +222,33 @@ func (tb *Torbox) GetTorrent(torrentId string) (*types.Torrent, error) {
 		Added:            data.CreatedAt.Format(time.RFC3339),
 	}
 	cfg := config.Get()
+
+	totalFiles := 0
+	skippedSamples := 0
+	skippedFileType := 0
+	skippedSize := 0
+	validFiles := 0
+	filesWithLinks := 0
+
 	for _, f := range data.Files {
+		totalFiles++
 		fileName := filepath.Base(f.Name)
+
 		if !tb.addSamples && utils.IsSampleFile(f.AbsolutePath) {
-			// Skip sample files
+			skippedSamples++
 			continue
 		}
 		if !cfg.IsAllowedFile(fileName) {
+			skippedFileType++
 			continue
 		}
 
 		if !cfg.IsSizeAllowed(f.Size) {
+			skippedSize++
 			continue
 		}
+
+		validFiles++
 		file := types.File{
 			TorrentId: t.Id,
 			Id:        strconv.Itoa(f.Id),
@@ -237,8 +256,26 @@ func (tb *Torbox) GetTorrent(torrentId string) (*types.Torrent, error) {
 			Size:      f.Size,
 			Path:      f.Name,
 		}
+
+		// For downloaded torrents, set a placeholder link to indicate file is available
+		if data.DownloadFinished {
+			file.Link = fmt.Sprintf("torbox://%s/%d", t.Id, f.Id)
+			filesWithLinks++
+		}
+
 		t.Files[fileName] = file
 	}
+
+	// Log summary only if there are issues or for debugging
+	tb.logger.Debug().
+		Str("torrent_id", t.Id).
+		Str("torrent_name", t.Name).
+		Bool("download_finished", data.DownloadFinished).
+		Str("status", t.Status).
+		Int("total_files", totalFiles).
+		Int("valid_files", validFiles).
+		Int("final_file_count", len(t.Files)).
+		Msg("Torrent file processing completed")
 	var cleanPath string
 	if len(t.Files) > 0 {
 		cleanPath = path.Clean(data.Files[0].Name)
@@ -266,24 +303,33 @@ func (tb *Torbox) UpdateTorrent(t *types.Torrent) error {
 	}
 	data := res.Data
 	name := data.Name
+
 	t.Name = name
 	t.Bytes = data.Size
 	t.Folder = name
 	t.Progress = data.Progress * 100
-	t.Status = getTorboxStatus(data.DownloadState, data.DownloadFinished)
+	t.Status = tb.getTorboxStatus(data.DownloadState, data.DownloadFinished)
 	t.Speed = data.DownloadSpeed
 	t.Seeders = data.Seeds
 	t.Filename = name
 	t.OriginalFilename = name
 	t.MountPath = tb.MountPath
 	t.Debrid = tb.name
+
+	// Clear existing files map to rebuild it
+	t.Files = make(map[string]types.File)
+
 	cfg := config.Get()
+	validFiles := 0
+	filesWithLinks := 0
+
 	for _, f := range data.Files {
 		fileName := filepath.Base(f.Name)
+
 		if !tb.addSamples && utils.IsSampleFile(f.AbsolutePath) {
-			// Skip sample files
 			continue
 		}
+
 		if !cfg.IsAllowedFile(fileName) {
 			continue
 		}
@@ -291,6 +337,8 @@ func (tb *Torbox) UpdateTorrent(t *types.Torrent) error {
 		if !cfg.IsSizeAllowed(f.Size) {
 			continue
 		}
+
+		validFiles++
 		file := types.File{
 			TorrentId: t.Id,
 			Id:        strconv.Itoa(f.Id),
@@ -298,8 +346,16 @@ func (tb *Torbox) UpdateTorrent(t *types.Torrent) error {
 			Size:      f.Size,
 			Path:      fileName,
 		}
+
+		// For downloaded torrents, set a placeholder link to indicate file is available
+		if data.DownloadFinished {
+			file.Link = fmt.Sprintf("torbox://%s/%s", t.Id, strconv.Itoa(f.Id))
+			filesWithLinks++
+		}
+
 		t.Files[fileName] = file
 	}
+
 	var cleanPath string
 	if len(t.Files) > 0 {
 		cleanPath = path.Clean(data.Files[0].Name)
@@ -409,30 +465,58 @@ func (tb *Torbox) GetDownloadLink(t *types.Torrent, file *types.File) (*types.Do
 	query.Add("token", tb.APIKey)
 	query.Add("file_id", file.Id)
 	url += "?" + query.Encode()
+
 	req, _ := http.NewRequest(http.MethodGet, url, nil)
 	resp, err := tb.client.MakeRequest(req)
 	if err != nil {
+		tb.logger.Error().
+			Err(err).
+			Str("torrent_id", t.Id).
+			Str("file_id", file.Id).
+			Msg("Failed to make request to Torbox API")
 		return nil, err
 	}
+
 	var data DownloadLinksResponse
 	if err = json.Unmarshal(resp, &data); err != nil {
+		tb.logger.Error().
+			Err(err).
+			Str("torrent_id", t.Id).
+			Str("file_id", file.Id).
+			Msg("Failed to unmarshal Torbox API response")
 		return nil, err
 	}
+
 	if data.Data == nil {
+		tb.logger.Error().
+			Str("torrent_id", t.Id).
+			Str("file_id", file.Id).
+			Bool("success", data.Success).
+			Interface("error", data.Error).
+			Str("detail", data.Detail).
+			Msg("Torbox API returned no data")
 		return nil, fmt.Errorf("error getting download links")
 	}
+
 	link := *data.Data
 	if link == "" {
+		tb.logger.Error().
+			Str("torrent_id", t.Id).
+			Str("file_id", file.Id).
+			Msg("Torbox API returned empty download link")
 		return nil, fmt.Errorf("error getting download links")
 	}
+
 	now := time.Now()
-	return &types.DownloadLink{
+	downloadLink := &types.DownloadLink{
 		Link:         file.Link,
 		DownloadLink: link,
 		Id:           file.Id,
 		Generated:    now,
 		ExpiresAt:    now.Add(tb.autoExpiresLinksAfter),
-	}, nil
+	}
+
+	return downloadLink, nil
 }
 
 func (tb *Torbox) GetDownloadingStatus() []string {
@@ -440,7 +524,87 @@ func (tb *Torbox) GetDownloadingStatus() []string {
 }
 
 func (tb *Torbox) GetTorrents() ([]*types.Torrent, error) {
-	return nil, nil
+	url := fmt.Sprintf("%s/api/torrents/mylist", tb.Host)
+	req, _ := http.NewRequest(http.MethodGet, url, nil)
+	resp, err := tb.client.MakeRequest(req)
+	if err != nil {
+		return nil, err
+	}
+
+	var res TorrentsListResponse
+	err = json.Unmarshal(resp, &res)
+	if err != nil {
+		return nil, err
+	}
+
+	if !res.Success || res.Data == nil {
+		return nil, fmt.Errorf("torbox API error: %v", res.Error)
+	}
+
+	torrents := make([]*types.Torrent, 0, len(*res.Data))
+	cfg := config.Get()
+
+	for _, data := range *res.Data {
+		t := &types.Torrent{
+			Id:               strconv.Itoa(data.Id),
+			Name:             data.Name,
+			Bytes:            data.Size,
+			Folder:           data.Name,
+			Progress:         data.Progress * 100,
+			Status:           tb.getTorboxStatus(data.DownloadState, data.DownloadFinished),
+			Speed:            data.DownloadSpeed,
+			Seeders:          data.Seeds,
+			Filename:         data.Name,
+			OriginalFilename: data.Name,
+			MountPath:        tb.MountPath,
+			Debrid:           tb.name,
+			Files:            make(map[string]types.File),
+			Added:            data.CreatedAt.Format(time.RFC3339),
+			InfoHash:         data.Hash,
+		}
+
+		// Process files
+		for _, f := range data.Files {
+			fileName := filepath.Base(f.Name)
+			if !tb.addSamples && utils.IsSampleFile(f.AbsolutePath) {
+				// Skip sample files
+				continue
+			}
+			if !cfg.IsAllowedFile(fileName) {
+				continue
+			}
+			if !cfg.IsSizeAllowed(f.Size) {
+				continue
+			}
+			file := types.File{
+				TorrentId: t.Id,
+				Id:        strconv.Itoa(f.Id),
+				Name:      fileName,
+				Size:      f.Size,
+				Path:      f.Name,
+			}
+
+			// For downloaded torrents, set a placeholder link to indicate file is available
+			if data.DownloadFinished {
+				file.Link = fmt.Sprintf("torbox://%s/%d", t.Id, f.Id)
+			}
+
+			t.Files[fileName] = file
+		}
+
+		// Set original filename based on first file or torrent name
+		var cleanPath string
+		if len(t.Files) > 0 {
+			cleanPath = path.Clean(data.Files[0].Name)
+		} else {
+			cleanPath = path.Clean(data.Name)
+		}
+		t.OriginalFilename = strings.Split(cleanPath, "/")[0]
+
+		torrents = append(torrents, t)
+	}
+
+	return torrents, nil
 }
 
 func (tb *Torbox) GetDownloadUncached() bool {

pkg/debrid/providers/torbox/types.go

@@ -57,7 +57,7 @@ type torboxInfo struct {
 	} `json:"files"`
 	DownloadPath     string      `json:"download_path"`
 	InactiveCheck    int         `json:"inactive_check"`
-	Availability     int         `json:"availability"`
+	Availability     float64     `json:"availability"`
 	DownloadFinished bool        `json:"download_finished"`
 	Tracker          interface{} `json:"tracker"`
 	TotalUploaded    int         `json:"total_uploaded"`
@@ -73,3 +73,5 @@ type torboxInfo struct {
 type InfoResponse APIResponse[torboxInfo]
 
 type DownloadLinksResponse APIResponse[string]
+
+type TorrentsListResponse APIResponse[[]torboxInfo]

pkg/debrid/store/cache.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/sirrobot01/decypharr/pkg/debrid/types"
 	"os"
 	"path"
 	"path/filepath"
@@ -17,13 +16,16 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/sirrobot01/decypharr/pkg/debrid/types"
+
 	"encoding/json"
+	_ "time/tzdata"
+
 	"github.com/go-co-op/gocron/v2"
 	"github.com/rs/zerolog"
 	"github.com/sirrobot01/decypharr/internal/config"
 	"github.com/sirrobot01/decypharr/internal/logger"
 	"github.com/sirrobot01/decypharr/internal/utils"
-	_ "time/tzdata"
 )
 
 type WebDavFolderNaming string
@@ -682,8 +684,13 @@ func (c *Cache) ProcessTorrent(t *types.Torrent) error {
 	}
 
 	if !isComplete(t.Files) {
-		c.logger.Debug().Msgf("Torrent %s is still not complete. Triggering a reinsert(disabled)", t.Id)
+		c.logger.Debug().
+			Str("torrent_id", t.Id).
+			Str("torrent_name", t.Name).
+			Int("total_files", len(t.Files)).
+			Msg("Torrent still not complete after refresh")
 	} else {
+
 		addedOn, err := time.Parse(time.RFC3339, t.Added)
 		if err != nil {
 			addedOn = time.Now()

pkg/debrid/store/download_link.go

@@ -3,6 +3,7 @@ package store
 import (
 	"errors"
 	"fmt"
+
 	"github.com/sirrobot01/decypharr/internal/utils"
 	"github.com/sirrobot01/decypharr/pkg/debrid/types"
 )
@@ -102,9 +103,16 @@ func (c *Cache) fetchDownloadLink(torrentName, filename, fileLink string) (*type
 	}
 
 	c.logger.Trace().Msgf("Getting download link for %s(%s)", filename, file.Link)
+
 	downloadLink, err := c.client.GetDownloadLink(ct.Torrent, &file)
 	if err != nil {
+
 		if errors.Is(err, utils.HosterUnavailableError) {
+			c.logger.Trace().
+				Str("filename", filename).
+				Str("torrent_id", ct.Id).
+				Msg("Hoster unavailable, attempting to reinsert torrent")
+
 			newCt, err := c.reInsertTorrent(ct)
 			if err != nil {
 				return nil, fmt.Errorf("failed to reinsert torrent: %w", err)
@@ -117,12 +125,11 @@ func (c *Cache) fetchDownloadLink(torrentName, filename, fileLink string) (*type
 			// Retry getting the download link
 			downloadLink, err = c.client.GetDownloadLink(ct.Torrent, &file)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("retry failed to get download link: %w", err)
 			}
 			if downloadLink == nil {
-				return nil, fmt.Errorf("download link is empty for")
+				return nil, fmt.Errorf("download link is empty after retry")
 			}
-			return nil, nil
 		} else if errors.Is(err, utils.TrafficExceededError) {
 			// This is likely a fair usage limit error
 			return nil, err

pkg/store/downloader.go

@@ -2,13 +2,14 @@ package store
 
 import (
 	"fmt"
-	"github.com/sirrobot01/decypharr/pkg/debrid/types"
 	"net/http"
 	"os"
 	"path/filepath"
 	"sync"
 	"time"
 
+	"github.com/sirrobot01/decypharr/pkg/debrid/types"
+
 	"github.com/cavaliergopher/grab/v3"
 	"github.com/sirrobot01/decypharr/internal/utils"
 )
@@ -212,7 +213,7 @@ func (s *Store) processSymlink(torrent *Torrent, debridTorrent *types.Torrent) (
 				if _, err := os.Stat(fullFilePath); !os.IsNotExist(err) {
 					fileSymlinkPath := filepath.Join(torrentSymlinkPath, file.Name)
 					if err := os.Symlink(fullFilePath, fileSymlinkPath); err != nil && !os.IsExist(err) {
-						s.logger.Debug().Msgf("Failed to create symlink: %s: %v", fileSymlinkPath, err)
+						s.logger.Warn().Msgf("Failed to create symlink: %s: %v", fileSymlinkPath, err)
 					} else {
 						filePaths = append(filePaths, fileSymlinkPath)
 						delete(pending, path)

pkg/store/queue.go

@@ -96,9 +96,7 @@ func (s *Store) trackAvailableSlots(ctx context.Context) {
 		return
 	}
 
-	for name, slots := range availableSlots {
+	for _, slots := range availableSlots {
-
-		s.logger.Debug().Msgf("Available slots for %s: %d", name, slots)
 		// If slots are available, process the next import request from the queue
 		for slots > 0 {
 			select {

pkg/store/torrent.go

@@ -5,14 +5,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/sirrobot01/decypharr/internal/request"
-	"github.com/sirrobot01/decypharr/internal/utils"
-	debridTypes "github.com/sirrobot01/decypharr/pkg/debrid"
-	"github.com/sirrobot01/decypharr/pkg/debrid/types"
 	"math"
 	"os"
 	"path/filepath"
 	"time"
+
+	"github.com/sirrobot01/decypharr/internal/request"
+	"github.com/sirrobot01/decypharr/internal/utils"
+	debridTypes "github.com/sirrobot01/decypharr/pkg/debrid"
+	"github.com/sirrobot01/decypharr/pkg/debrid/types"
 )
 
 func (s *Store) AddTorrent(ctx context.Context, importReq *ImportRequest) error {
@@ -60,10 +61,16 @@ func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, imp
 	_arr := importReq.Arr
 	backoff := time.NewTimer(s.refreshInterval)
 	defer backoff.Stop()
+
 	for debridTorrent.Status != "downloaded" {
-		s.logger.Debug().Msgf("%s <- (%s) Download Progress: %.2f%%", debridTorrent.Debrid, debridTorrent.Name, debridTorrent.Progress)
+
 		dbT, err := client.CheckStatus(debridTorrent)
 		if err != nil {
+			s.logger.Error().
+				Str("torrent_id", debridTorrent.Id).
+				Str("torrent_name", debridTorrent.Name).
+				Err(err).
+				Msg("Error checking torrent status")
 			if dbT != nil && dbT.Id != "" {
 				// Delete the torrent if it was not downloaded
 				go func() {
@@ -83,15 +90,16 @@ func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, imp
 		torrent = s.partialTorrentUpdate(torrent, debridTorrent)
 
 		// Exit the loop for downloading statuses to prevent memory buildup
-		if debridTorrent.Status == "downloaded" || !utils.Contains(downloadingStatuses, debridTorrent.Status) {
+		exitCondition1 := debridTorrent.Status == "downloaded"
+		exitCondition2 := !utils.Contains(downloadingStatuses, debridTorrent.Status)
+
+		if exitCondition1 || exitCondition2 {
 			break
 		}
-		select {
+		<-backoff.C
-		case <-backoff.C:
+		// Increase interval gradually, cap at max
-			// Increase interval gradually, cap at max
+		nextInterval := min(s.refreshInterval*2, 30*time.Second)
-			nextInterval := min(s.refreshInterval*2, 30*time.Second)
+		backoff.Reset(nextInterval)
-			backoff.Reset(nextInterval)
-		}
 	}
 	var torrentSymlinkPath string
 	var err error
@@ -109,7 +117,6 @@ func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, imp
 		}()
 		s.logger.Error().Err(err).Msgf("Error occured while processing torrent %s", debridTorrent.Name)
 		importReq.markAsFailed(err, torrent, debridTorrent)
-		return
 	}
 
 	onSuccess := func(torrentSymlinkPath string) {