From b901bd5175fa72c2a1860b1588db72d82c9ecf74 Mon Sep 17 00:00:00 2001 From: Sadman Sakib Date: Fri, 11 Jul 2025 18:17:03 +0600 Subject: [PATCH] Feature/torbox provider improvements (#100) - Add Torbox WebDAV implementation - Fix Issues with sample and extension checks --- internal/config/misc.go | 2 +- internal/utils/regex.go | 3 +- pkg/debrid/providers/torbox/torbox.go | 198 +++++++++++++++++++++++--- pkg/debrid/providers/torbox/types.go | 4 +- pkg/debrid/store/cache.go | 13 +- pkg/debrid/store/download_link.go | 13 +- pkg/store/downloader.go | 5 +- pkg/store/queue.go | 4 +- pkg/store/torrent.go | 33 +++-- 9 files changed, 231 insertions(+), 44 deletions(-) diff --git a/internal/config/misc.go b/internal/config/misc.go index 29ea9c5..eb64461 100644 --- a/internal/config/misc.go +++ b/internal/config/misc.go @@ -24,7 +24,7 @@ func (c *Config) IsAllowedFile(filename string) bool { } func getDefaultExtensions() []string { - videoExts := strings.Split("webm,m4v,3gp,nsv,ty,strm,rm,rmvb,m3u,ifo,mov,qt,divx,xvid,bivx,nrg,pva,wmv,asf,asx,ogm,ogv,m2v,avi,bin,dat,dvr-ms,mpg,mpeg,mp4,avc,vp3,svq3,nuv,viv,dv,fli,flv,wpl,img,iso,vob,mkv,mk3d,ts,wtv,m2ts'", ",") + videoExts := strings.Split("webm,m4v,3gp,nsv,ty,strm,rm,rmvb,m3u,ifo,mov,qt,divx,xvid,bivx,nrg,pva,wmv,asf,asx,ogm,ogv,m2v,avi,bin,dat,dvr-ms,mpg,mpeg,mp4,avc,vp3,svq3,nuv,viv,dv,fli,flv,wpl,img,iso,vob,mkv,mk3d,ts,wtv,m2ts", ",") musicExts := strings.Split("MP3,WAV,FLAC,OGG,WMA,AIFF,ALAC,M4A,APE,AC3,DTS,M4P,MID,MIDI,MKA,MP2,MPA,RA,VOC,WV,AMR", ",") // Combine both slices diff --git a/internal/utils/regex.go b/internal/utils/regex.go index 0cd7c03..e71d15f 100644 --- a/internal/utils/regex.go +++ b/internal/utils/regex.go @@ -51,7 +51,8 @@ func IsMediaFile(path string) bool { } func IsSampleFile(path string) bool { - if strings.HasSuffix(strings.ToLower(path), "sample.mkv") { + filename := filepath.Base(path) + if strings.HasSuffix(strings.ToLower(filename), "sample.mkv") { return true } return RegexMatch(sampleRegex, path) diff --git a/pkg/debrid/providers/torbox/torbox.go b/pkg/debrid/providers/torbox/torbox.go index 7d346a1..ba520fd 100644 --- a/pkg/debrid/providers/torbox/torbox.go +++ b/pkg/debrid/providers/torbox/torbox.go @@ -4,13 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/rs/zerolog" - "github.com/sirrobot01/decypharr/internal/config" - "github.com/sirrobot01/decypharr/internal/logger" - "github.com/sirrobot01/decypharr/internal/request" - "github.com/sirrobot01/decypharr/internal/utils" - "github.com/sirrobot01/decypharr/pkg/debrid/types" - "github.com/sirrobot01/decypharr/pkg/version" "mime/multipart" "net/http" gourl "net/url" @@ -21,6 +14,14 @@ import ( "strings" "sync" "time" + + "github.com/rs/zerolog" + "github.com/sirrobot01/decypharr/internal/config" + "github.com/sirrobot01/decypharr/internal/logger" + "github.com/sirrobot01/decypharr/internal/request" + "github.com/sirrobot01/decypharr/internal/utils" + "github.com/sirrobot01/decypharr/pkg/debrid/types" + "github.com/sirrobot01/decypharr/pkg/version" ) type Torbox struct { @@ -168,7 +169,7 @@ func (tb *Torbox) SubmitMagnet(torrent *types.Torrent) (*types.Torrent, error) { return torrent, nil } -func getTorboxStatus(status string, finished bool) string { +func (tb *Torbox) getTorboxStatus(status string, finished bool) string { if finished { return "downloaded" } @@ -176,12 +177,16 @@ func getTorboxStatus(status string, finished bool) string { "checkingResumeData", "metaDL", "pausedUP", "queuedUP", "checkingUP", "forcedUP", "allocating", "downloading", "metaDL", "pausedDL", "queuedDL", "checkingDL", "forcedDL", "checkingResumeData", "moving"} + + var determinedStatus string switch { case utils.Contains(downloading, status): - return "downloading" + determinedStatus = "downloading" default: - return "error" + determinedStatus = "error" } + + return determinedStatus } func (tb *Torbox) GetTorrent(torrentId string) (*types.Torrent, error) { @@ -206,7 +211,7 @@ func (tb *Torbox) GetTorrent(torrentId string) (*types.Torrent, error) { Bytes: data.Size, Folder: data.Name, Progress: data.Progress * 100, - Status: getTorboxStatus(data.DownloadState, data.DownloadFinished), + Status: tb.getTorboxStatus(data.DownloadState, data.DownloadFinished), Speed: data.DownloadSpeed, Seeders: data.Seeds, Filename: data.Name, @@ -217,19 +222,33 @@ func (tb *Torbox) GetTorrent(torrentId string) (*types.Torrent, error) { Added: data.CreatedAt.Format(time.RFC3339), } cfg := config.Get() + + totalFiles := 0 + skippedSamples := 0 + skippedFileType := 0 + skippedSize := 0 + validFiles := 0 + filesWithLinks := 0 + for _, f := range data.Files { + totalFiles++ fileName := filepath.Base(f.Name) + if !tb.addSamples && utils.IsSampleFile(f.AbsolutePath) { - // Skip sample files + skippedSamples++ continue } if !cfg.IsAllowedFile(fileName) { + skippedFileType++ continue } if !cfg.IsSizeAllowed(f.Size) { + skippedSize++ continue } + + validFiles++ file := types.File{ TorrentId: t.Id, Id: strconv.Itoa(f.Id), @@ -237,8 +256,26 @@ func (tb *Torbox) GetTorrent(torrentId string) (*types.Torrent, error) { Size: f.Size, Path: f.Name, } + + // For downloaded torrents, set a placeholder link to indicate file is available + if data.DownloadFinished { + file.Link = fmt.Sprintf("torbox://%s/%d", t.Id, f.Id) + filesWithLinks++ + } + t.Files[fileName] = file } + + // Log summary only if there are issues or for debugging + tb.logger.Debug(). + Str("torrent_id", t.Id). + Str("torrent_name", t.Name). + Bool("download_finished", data.DownloadFinished). + Str("status", t.Status). + Int("total_files", totalFiles). + Int("valid_files", validFiles). + Int("final_file_count", len(t.Files)). + Msg("Torrent file processing completed") var cleanPath string if len(t.Files) > 0 { cleanPath = path.Clean(data.Files[0].Name) @@ -266,24 +303,33 @@ func (tb *Torbox) UpdateTorrent(t *types.Torrent) error { } data := res.Data name := data.Name + t.Name = name t.Bytes = data.Size t.Folder = name t.Progress = data.Progress * 100 - t.Status = getTorboxStatus(data.DownloadState, data.DownloadFinished) + t.Status = tb.getTorboxStatus(data.DownloadState, data.DownloadFinished) t.Speed = data.DownloadSpeed t.Seeders = data.Seeds t.Filename = name t.OriginalFilename = name t.MountPath = tb.MountPath t.Debrid = tb.name + + // Clear existing files map to rebuild it + t.Files = make(map[string]types.File) + cfg := config.Get() + validFiles := 0 + filesWithLinks := 0 + for _, f := range data.Files { fileName := filepath.Base(f.Name) + if !tb.addSamples && utils.IsSampleFile(f.AbsolutePath) { - // Skip sample files continue } + if !cfg.IsAllowedFile(fileName) { continue } @@ -291,6 +337,8 @@ func (tb *Torbox) UpdateTorrent(t *types.Torrent) error { if !cfg.IsSizeAllowed(f.Size) { continue } + + validFiles++ file := types.File{ TorrentId: t.Id, Id: strconv.Itoa(f.Id), @@ -298,8 +346,16 @@ func (tb *Torbox) UpdateTorrent(t *types.Torrent) error { Size: f.Size, Path: fileName, } + + // For downloaded torrents, set a placeholder link to indicate file is available + if data.DownloadFinished { + file.Link = fmt.Sprintf("torbox://%s/%s", t.Id, strconv.Itoa(f.Id)) + filesWithLinks++ + } + t.Files[fileName] = file } + var cleanPath string if len(t.Files) > 0 { cleanPath = path.Clean(data.Files[0].Name) @@ -409,30 +465,58 @@ func (tb *Torbox) GetDownloadLink(t *types.Torrent, file *types.File) (*types.Do query.Add("token", tb.APIKey) query.Add("file_id", file.Id) url += "?" + query.Encode() + req, _ := http.NewRequest(http.MethodGet, url, nil) resp, err := tb.client.MakeRequest(req) if err != nil { + tb.logger.Error(). + Err(err). + Str("torrent_id", t.Id). + Str("file_id", file.Id). + Msg("Failed to make request to Torbox API") return nil, err } + var data DownloadLinksResponse if err = json.Unmarshal(resp, &data); err != nil { + tb.logger.Error(). + Err(err). + Str("torrent_id", t.Id). + Str("file_id", file.Id). + Msg("Failed to unmarshal Torbox API response") return nil, err } + if data.Data == nil { + tb.logger.Error(). + Str("torrent_id", t.Id). + Str("file_id", file.Id). + Bool("success", data.Success). + Interface("error", data.Error). + Str("detail", data.Detail). + Msg("Torbox API returned no data") return nil, fmt.Errorf("error getting download links") } + link := *data.Data if link == "" { + tb.logger.Error(). + Str("torrent_id", t.Id). + Str("file_id", file.Id). + Msg("Torbox API returned empty download link") return nil, fmt.Errorf("error getting download links") } + now := time.Now() - return &types.DownloadLink{ + downloadLink := &types.DownloadLink{ Link: file.Link, DownloadLink: link, Id: file.Id, Generated: now, ExpiresAt: now.Add(tb.autoExpiresLinksAfter), - }, nil + } + + return downloadLink, nil } func (tb *Torbox) GetDownloadingStatus() []string { @@ -440,7 +524,87 @@ func (tb *Torbox) GetDownloadingStatus() []string { } func (tb *Torbox) GetTorrents() ([]*types.Torrent, error) { - return nil, nil + url := fmt.Sprintf("%s/api/torrents/mylist", tb.Host) + req, _ := http.NewRequest(http.MethodGet, url, nil) + resp, err := tb.client.MakeRequest(req) + if err != nil { + return nil, err + } + + var res TorrentsListResponse + err = json.Unmarshal(resp, &res) + if err != nil { + return nil, err + } + + if !res.Success || res.Data == nil { + return nil, fmt.Errorf("torbox API error: %v", res.Error) + } + + torrents := make([]*types.Torrent, 0, len(*res.Data)) + cfg := config.Get() + + for _, data := range *res.Data { + t := &types.Torrent{ + Id: strconv.Itoa(data.Id), + Name: data.Name, + Bytes: data.Size, + Folder: data.Name, + Progress: data.Progress * 100, + Status: tb.getTorboxStatus(data.DownloadState, data.DownloadFinished), + Speed: data.DownloadSpeed, + Seeders: data.Seeds, + Filename: data.Name, + OriginalFilename: data.Name, + MountPath: tb.MountPath, + Debrid: tb.name, + Files: make(map[string]types.File), + Added: data.CreatedAt.Format(time.RFC3339), + InfoHash: data.Hash, + } + + // Process files + for _, f := range data.Files { + fileName := filepath.Base(f.Name) + if !tb.addSamples && utils.IsSampleFile(f.AbsolutePath) { + // Skip sample files + continue + } + if !cfg.IsAllowedFile(fileName) { + continue + } + if !cfg.IsSizeAllowed(f.Size) { + continue + } + file := types.File{ + TorrentId: t.Id, + Id: strconv.Itoa(f.Id), + Name: fileName, + Size: f.Size, + Path: f.Name, + } + + // For downloaded torrents, set a placeholder link to indicate file is available + if data.DownloadFinished { + file.Link = fmt.Sprintf("torbox://%s/%d", t.Id, f.Id) + } + + t.Files[fileName] = file + } + + // Set original filename based on first file or torrent name + var cleanPath string + if len(t.Files) > 0 { + cleanPath = path.Clean(data.Files[0].Name) + } else { + cleanPath = path.Clean(data.Name) + } + t.OriginalFilename = strings.Split(cleanPath, "/")[0] + + torrents = append(torrents, t) + } + + return torrents, nil } func (tb *Torbox) GetDownloadUncached() bool { diff --git a/pkg/debrid/providers/torbox/types.go b/pkg/debrid/providers/torbox/types.go index 0716345..fbce026 100644 --- a/pkg/debrid/providers/torbox/types.go +++ b/pkg/debrid/providers/torbox/types.go @@ -57,7 +57,7 @@ type torboxInfo struct { } `json:"files"` DownloadPath string `json:"download_path"` InactiveCheck int `json:"inactive_check"` - Availability int `json:"availability"` + Availability float64 `json:"availability"` DownloadFinished bool `json:"download_finished"` Tracker interface{} `json:"tracker"` TotalUploaded int `json:"total_uploaded"` @@ -73,3 +73,5 @@ type torboxInfo struct { type InfoResponse APIResponse[torboxInfo] type DownloadLinksResponse APIResponse[string] + +type TorrentsListResponse APIResponse[[]torboxInfo] diff --git a/pkg/debrid/store/cache.go b/pkg/debrid/store/cache.go index 8d2c69b..2968463 100644 --- a/pkg/debrid/store/cache.go +++ b/pkg/debrid/store/cache.go @@ -6,7 +6,6 @@ import ( "context" "errors" "fmt" - "github.com/sirrobot01/decypharr/pkg/debrid/types" "os" "path" "path/filepath" @@ -17,13 +16,16 @@ import ( "sync/atomic" "time" + "github.com/sirrobot01/decypharr/pkg/debrid/types" + "encoding/json" + _ "time/tzdata" + "github.com/go-co-op/gocron/v2" "github.com/rs/zerolog" "github.com/sirrobot01/decypharr/internal/config" "github.com/sirrobot01/decypharr/internal/logger" "github.com/sirrobot01/decypharr/internal/utils" - _ "time/tzdata" ) type WebDavFolderNaming string @@ -682,8 +684,13 @@ func (c *Cache) ProcessTorrent(t *types.Torrent) error { } if !isComplete(t.Files) { - c.logger.Debug().Msgf("Torrent %s is still not complete. Triggering a reinsert(disabled)", t.Id) + c.logger.Debug(). + Str("torrent_id", t.Id). + Str("torrent_name", t.Name). + Int("total_files", len(t.Files)). + Msg("Torrent still not complete after refresh") } else { + addedOn, err := time.Parse(time.RFC3339, t.Added) if err != nil { addedOn = time.Now() diff --git a/pkg/debrid/store/download_link.go b/pkg/debrid/store/download_link.go index 0f73891..3b48767 100644 --- a/pkg/debrid/store/download_link.go +++ b/pkg/debrid/store/download_link.go @@ -3,6 +3,7 @@ package store import ( "errors" "fmt" + "github.com/sirrobot01/decypharr/internal/utils" "github.com/sirrobot01/decypharr/pkg/debrid/types" ) @@ -102,9 +103,16 @@ func (c *Cache) fetchDownloadLink(torrentName, filename, fileLink string) (*type } c.logger.Trace().Msgf("Getting download link for %s(%s)", filename, file.Link) + downloadLink, err := c.client.GetDownloadLink(ct.Torrent, &file) if err != nil { + if errors.Is(err, utils.HosterUnavailableError) { + c.logger.Trace(). + Str("filename", filename). + Str("torrent_id", ct.Id). + Msg("Hoster unavailable, attempting to reinsert torrent") + newCt, err := c.reInsertTorrent(ct) if err != nil { return nil, fmt.Errorf("failed to reinsert torrent: %w", err) @@ -117,12 +125,11 @@ func (c *Cache) fetchDownloadLink(torrentName, filename, fileLink string) (*type // Retry getting the download link downloadLink, err = c.client.GetDownloadLink(ct.Torrent, &file) if err != nil { - return nil, err + return nil, fmt.Errorf("retry failed to get download link: %w", err) } if downloadLink == nil { - return nil, fmt.Errorf("download link is empty for") + return nil, fmt.Errorf("download link is empty after retry") } - return nil, nil } else if errors.Is(err, utils.TrafficExceededError) { // This is likely a fair usage limit error return nil, err diff --git a/pkg/store/downloader.go b/pkg/store/downloader.go index 8510863..fac2159 100644 --- a/pkg/store/downloader.go +++ b/pkg/store/downloader.go @@ -2,13 +2,14 @@ package store import ( "fmt" - "github.com/sirrobot01/decypharr/pkg/debrid/types" "net/http" "os" "path/filepath" "sync" "time" + "github.com/sirrobot01/decypharr/pkg/debrid/types" + "github.com/cavaliergopher/grab/v3" "github.com/sirrobot01/decypharr/internal/utils" ) @@ -212,7 +213,7 @@ func (s *Store) processSymlink(torrent *Torrent, debridTorrent *types.Torrent) ( if _, err := os.Stat(fullFilePath); !os.IsNotExist(err) { fileSymlinkPath := filepath.Join(torrentSymlinkPath, file.Name) if err := os.Symlink(fullFilePath, fileSymlinkPath); err != nil && !os.IsExist(err) { - s.logger.Debug().Msgf("Failed to create symlink: %s: %v", fileSymlinkPath, err) + s.logger.Warn().Msgf("Failed to create symlink: %s: %v", fileSymlinkPath, err) } else { filePaths = append(filePaths, fileSymlinkPath) delete(pending, path) diff --git a/pkg/store/queue.go b/pkg/store/queue.go index f4d573e..3b79153 100644 --- a/pkg/store/queue.go +++ b/pkg/store/queue.go @@ -96,9 +96,7 @@ func (s *Store) trackAvailableSlots(ctx context.Context) { return } - for name, slots := range availableSlots { - - s.logger.Debug().Msgf("Available slots for %s: %d", name, slots) + for _, slots := range availableSlots { // If slots are available, process the next import request from the queue for slots > 0 { select { diff --git a/pkg/store/torrent.go b/pkg/store/torrent.go index 57bac5d..6b67c57 100644 --- a/pkg/store/torrent.go +++ b/pkg/store/torrent.go @@ -5,14 +5,15 @@ import ( "context" "errors" "fmt" - "github.com/sirrobot01/decypharr/internal/request" - "github.com/sirrobot01/decypharr/internal/utils" - debridTypes "github.com/sirrobot01/decypharr/pkg/debrid" - "github.com/sirrobot01/decypharr/pkg/debrid/types" "math" "os" "path/filepath" "time" + + "github.com/sirrobot01/decypharr/internal/request" + "github.com/sirrobot01/decypharr/internal/utils" + debridTypes "github.com/sirrobot01/decypharr/pkg/debrid" + "github.com/sirrobot01/decypharr/pkg/debrid/types" ) func (s *Store) AddTorrent(ctx context.Context, importReq *ImportRequest) error { @@ -60,10 +61,16 @@ func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, imp _arr := importReq.Arr backoff := time.NewTimer(s.refreshInterval) defer backoff.Stop() + for debridTorrent.Status != "downloaded" { - s.logger.Debug().Msgf("%s <- (%s) Download Progress: %.2f%%", debridTorrent.Debrid, debridTorrent.Name, debridTorrent.Progress) + dbT, err := client.CheckStatus(debridTorrent) if err != nil { + s.logger.Error(). + Str("torrent_id", debridTorrent.Id). + Str("torrent_name", debridTorrent.Name). + Err(err). + Msg("Error checking torrent status") if dbT != nil && dbT.Id != "" { // Delete the torrent if it was not downloaded go func() { @@ -83,15 +90,16 @@ func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, imp torrent = s.partialTorrentUpdate(torrent, debridTorrent) // Exit the loop for downloading statuses to prevent memory buildup - if debridTorrent.Status == "downloaded" || !utils.Contains(downloadingStatuses, debridTorrent.Status) { + exitCondition1 := debridTorrent.Status == "downloaded" + exitCondition2 := !utils.Contains(downloadingStatuses, debridTorrent.Status) + + if exitCondition1 || exitCondition2 { break } - select { - case <-backoff.C: - // Increase interval gradually, cap at max - nextInterval := min(s.refreshInterval*2, 30*time.Second) - backoff.Reset(nextInterval) - } + <-backoff.C + // Increase interval gradually, cap at max + nextInterval := min(s.refreshInterval*2, 30*time.Second) + backoff.Reset(nextInterval) } var torrentSymlinkPath string var err error @@ -109,7 +117,6 @@ func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, imp }() s.logger.Error().Err(err).Msgf("Error occured while processing torrent %s", debridTorrent.Name) importReq.markAsFailed(err, torrent, debridTorrent) - return } onSuccess := func(torrentSymlinkPath string) {