From 0c68364a6a95d385a9d0f0d4c670d571edfd121d Mon Sep 17 00:00:00 2001 From: Mukhtar Akere Date: Thu, 20 Mar 2025 10:42:51 +0100 Subject: [PATCH] Improvements: - An improvised caching for stats; using metadata on ls - Integrated into the downloading system - Fix minor bugs noticed - Still experiemental, sike --- internal/config/config.go | 3 +- pkg/debrid/alldebrid/alldebrid.go | 26 +-- pkg/debrid/debrid.go | 105 ---------- pkg/{webdav => debrid/debrid}/cache.go | 255 +++++------------------ pkg/debrid/debrid/debrid.go | 98 +++++++-- pkg/debrid/debrid/engine.go | 51 +++++ pkg/debrid/debrid/misc.go | 10 + pkg/debrid/debrid/refresh.go | 205 ++++++++++++++++++ pkg/debrid/debrid/workers.go | 35 ++++ pkg/debrid/debrid_link/debrid_link.go | 24 +-- pkg/debrid/engine/engine.go | 30 --- pkg/debrid/realdebrid/realdebrid.go | 53 ++--- pkg/debrid/torbox/torbox.go | 22 +- pkg/debrid/types/debrid.go | 23 ++ pkg/debrid/{torrent => types}/torrent.go | 2 +- pkg/qbit/downloader.go | 17 +- pkg/qbit/import.go | 2 +- pkg/qbit/torrent.go | 54 ++++- pkg/qbit/types.go | 10 +- pkg/repair/repair.go | 4 +- pkg/service/service.go | 11 +- pkg/webdav/file.go | 28 ++- pkg/webdav/handler.go | 194 +++++++++-------- pkg/webdav/misc.go | 16 +- pkg/webdav/webdav.go | 4 +- pkg/webdav/workers.go | 69 ------ 26 files changed, 715 insertions(+), 636 deletions(-) delete mode 100644 pkg/debrid/debrid.go rename pkg/{webdav => debrid/debrid}/cache.go (65%) create mode 100644 pkg/debrid/debrid/engine.go create mode 100644 pkg/debrid/debrid/misc.go create mode 100644 pkg/debrid/debrid/refresh.go create mode 100644 pkg/debrid/debrid/workers.go delete mode 100644 pkg/debrid/engine/engine.go create mode 100644 pkg/debrid/types/debrid.go rename pkg/debrid/{torrent => types}/torrent.go (99%) delete mode 100644 pkg/webdav/workers.go diff --git a/internal/config/config.go b/internal/config/config.go index 30a2090..dd6b7bf 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -23,6 +23,7 @@ type Debrid struct { DownloadUncached bool `json:"download_uncached"` CheckCached bool `json:"check_cached"` RateLimit string `json:"rate_limit"` // 200/minute or 10/second + EnableWebDav bool `json:"enable_webdav"` } type Proxy struct { @@ -174,7 +175,7 @@ func validateQbitTorrent(config *QBitTorrent) error { return errors.New("qbittorent download folder is required") } if _, err := os.Stat(config.DownloadFolder); os.IsNotExist(err) { - return errors.New("qbittorent download folder does not exist") + return fmt.Errorf("qbittorent download folder(%s) does not exist", config.DownloadFolder) } return nil } diff --git a/pkg/debrid/alldebrid/alldebrid.go b/pkg/debrid/alldebrid/alldebrid.go index ae82df8..c1f55bb 100644 --- a/pkg/debrid/alldebrid/alldebrid.go +++ b/pkg/debrid/alldebrid/alldebrid.go @@ -8,7 +8,7 @@ import ( "github.com/sirrobot01/debrid-blackhole/internal/logger" "github.com/sirrobot01/debrid-blackhole/internal/request" "github.com/sirrobot01/debrid-blackhole/internal/utils" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" "slices" "time" @@ -47,7 +47,7 @@ func (ad *AllDebrid) IsAvailable(hashes []string) map[string]bool { return result } -func (ad *AllDebrid) SubmitMagnet(torrent *torrent.Torrent) (*torrent.Torrent, error) { +func (ad *AllDebrid) SubmitMagnet(torrent *types.Torrent) (*types.Torrent, error) { url := fmt.Sprintf("%s/magnet/upload", ad.Host) query := gourl.Values{} query.Add("magnets[]", torrent.Magnet.Link) @@ -84,8 +84,8 @@ func getAlldebridStatus(statusCode int) string { } } -func flattenFiles(files []MagnetFile, parentPath string, index *int) map[string]torrent.File { - result := make(map[string]torrent.File) +func flattenFiles(files []MagnetFile, parentPath string, index *int) map[string]types.File { + result := make(map[string]types.File) cfg := config.GetConfig() @@ -123,7 +123,7 @@ func flattenFiles(files []MagnetFile, parentPath string, index *int) map[string] } *index++ - file := torrent.File{ + file := types.File{ Id: strconv.Itoa(*index), Name: fileName, Size: f.Size, @@ -136,7 +136,7 @@ func flattenFiles(files []MagnetFile, parentPath string, index *int) map[string] return result } -func (ad *AllDebrid) UpdateTorrent(t *torrent.Torrent) error { +func (ad *AllDebrid) UpdateTorrent(t *types.Torrent) error { url := fmt.Sprintf("%s/magnet/status?id=%s", ad.Host, t.Id) req, _ := http.NewRequest(http.MethodGet, url, nil) resp, err := ad.client.MakeRequest(req) @@ -172,7 +172,7 @@ func (ad *AllDebrid) UpdateTorrent(t *torrent.Torrent) error { return nil } -func (ad *AllDebrid) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*torrent.Torrent, error) { +func (ad *AllDebrid) CheckStatus(torrent *types.Torrent, isSymlink bool) (*types.Torrent, error) { for { err := ad.UpdateTorrent(torrent) @@ -204,7 +204,7 @@ func (ad *AllDebrid) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*tor return torrent, nil } -func (ad *AllDebrid) DeleteTorrent(torrent *torrent.Torrent) { +func (ad *AllDebrid) DeleteTorrent(torrent *types.Torrent) { url := fmt.Sprintf("%s/magnet/delete?id=%s", ad.Host, torrent.Id) req, _ := http.NewRequest(http.MethodGet, url, nil) _, err := ad.client.MakeRequest(req) @@ -215,7 +215,7 @@ func (ad *AllDebrid) DeleteTorrent(torrent *torrent.Torrent) { } } -func (ad *AllDebrid) GenerateDownloadLinks(t *torrent.Torrent) error { +func (ad *AllDebrid) GenerateDownloadLinks(t *types.Torrent) error { for _, file := range t.Files { url := fmt.Sprintf("%s/link/unlock", ad.Host) query := gourl.Values{} @@ -239,7 +239,7 @@ func (ad *AllDebrid) GenerateDownloadLinks(t *torrent.Torrent) error { return nil } -func (ad *AllDebrid) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.File { +func (ad *AllDebrid) GetDownloadLink(t *types.Torrent, file *types.File) *types.File { url := fmt.Sprintf("%s/link/unlock", ad.Host) query := gourl.Values{} query.Add("link", file.Link) @@ -263,11 +263,11 @@ func (ad *AllDebrid) GetCheckCached() bool { return ad.CheckCached } -func (ad *AllDebrid) GetTorrents() ([]*torrent.Torrent, error) { +func (ad *AllDebrid) GetTorrents() ([]*types.Torrent, error) { return nil, nil } -func (ad *AllDebrid) GetDownloads() (map[string]torrent.DownloadLinks, error) { +func (ad *AllDebrid) GetDownloads() (map[string]types.DownloadLinks, error) { return nil, nil } @@ -279,7 +279,7 @@ func (ad *AllDebrid) GetDownloadUncached() bool { return ad.DownloadUncached } -func (ad *AllDebrid) ConvertLinksToFiles(links []string) []torrent.File { +func (ad *AllDebrid) ConvertLinksToFiles(links []string) []types.File { return nil } diff --git a/pkg/debrid/debrid.go b/pkg/debrid/debrid.go deleted file mode 100644 index 02073f9..0000000 --- a/pkg/debrid/debrid.go +++ /dev/null @@ -1,105 +0,0 @@ -package debrid - -import ( - "fmt" - "github.com/sirrobot01/debrid-blackhole/internal/config" - "github.com/sirrobot01/debrid-blackhole/internal/utils" - "github.com/sirrobot01/debrid-blackhole/pkg/arr" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/alldebrid" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid_link" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/engine" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/realdebrid" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torbox" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" -) - -func New() *engine.Engine { - cfg := config.GetConfig() - debrids := make([]debrid.Client, 0) - - for _, dc := range cfg.Debrids { - client := createDebridClient(dc) - logger := client.GetLogger() - logger.Info().Msg("Debrid Service started") - debrids = append(debrids, client) - } - d := &engine.Engine{ - Debrids: debrids, - LastUsed: 0, - } - return d -} - -func createDebridClient(dc config.Debrid) debrid.Client { - switch dc.Name { - case "realdebrid": - return realdebrid.New(dc) - case "torbox": - return torbox.New(dc) - case "debridlink": - return debrid_link.New(dc) - case "alldebrid": - return alldebrid.New(dc) - default: - return realdebrid.New(dc) - } -} - -func ProcessTorrent(d *engine.Engine, magnet *utils.Magnet, a *arr.Arr, isSymlink, overrideDownloadUncached bool) (*torrent.Torrent, error) { - - debridTorrent := &torrent.Torrent{ - InfoHash: magnet.InfoHash, - Magnet: magnet, - Name: magnet.Name, - Arr: a, - Size: magnet.Size, - Files: make(map[string]torrent.File), - } - - errs := make([]error, 0) - - for index, db := range d.Debrids { - logger := db.GetLogger() - logger.Info().Msgf("Processing debrid: %s", db.GetName()) - - // Override first, arr second, debrid third - - if overrideDownloadUncached { - debridTorrent.DownloadUncached = true - } else if a.DownloadUncached != nil { - // Arr cached is set - debridTorrent.DownloadUncached = *a.DownloadUncached - } else { - debridTorrent.DownloadUncached = db.GetDownloadUncached() - } - - logger.Info().Msgf("Torrent Hash: %s", debridTorrent.InfoHash) - if db.GetCheckCached() { - hash, exists := db.IsAvailable([]string{debridTorrent.InfoHash})[debridTorrent.InfoHash] - if !exists || !hash { - logger.Info().Msgf("Torrent: %s is not cached", debridTorrent.Name) - continue - } else { - logger.Info().Msgf("Torrent: %s is cached(or downloading)", debridTorrent.Name) - } - } - - dbt, err := db.SubmitMagnet(debridTorrent) - if dbt != nil { - dbt.Arr = a - } - if err != nil || dbt == nil || dbt.Id == "" { - errs = append(errs, err) - continue - } - logger.Info().Msgf("Torrent: %s(id=%s) submitted to %s", dbt.Name, dbt.Id, db.GetName()) - d.LastUsed = index - return db.CheckStatus(dbt, isSymlink) - } - err := fmt.Errorf("failed to process torrent") - for _, e := range errs { - err = fmt.Errorf("%w\n%w", err, e) - } - return nil, err -} diff --git a/pkg/webdav/cache.go b/pkg/debrid/debrid/cache.go similarity index 65% rename from pkg/webdav/cache.go rename to pkg/debrid/debrid/cache.go index 035637b..abcc804 100644 --- a/pkg/webdav/cache.go +++ b/pkg/debrid/debrid/cache.go @@ -1,53 +1,50 @@ -package webdav +package debrid import ( "bufio" "context" "fmt" - "github.com/dgraph-io/badger/v4" "github.com/goccy/go-json" "github.com/rs/zerolog" "github.com/sirrobot01/debrid-blackhole/internal/logger" "github.com/sirrobot01/debrid-blackhole/internal/utils" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" "os" "path/filepath" "runtime" - "sort" "sync" "sync/atomic" "time" "github.com/sirrobot01/debrid-blackhole/internal/config" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" ) type DownloadLinkCache struct { Link string `json:"download_link"` } -type propfindResponse struct { - data []byte - ts time.Time +type PropfindResponse struct { + Data []byte + GzippedData []byte + Ts time.Time } type CachedTorrent struct { - *torrent.Torrent + *types.Torrent LastRead time.Time `json:"last_read"` IsComplete bool `json:"is_complete"` } type Cache struct { dir string - client debrid.Client - db *badger.DB + client types.Client logger zerolog.Logger torrents map[string]*CachedTorrent // key: torrent.Id, value: *CachedTorrent torrentsNames map[string]*CachedTorrent // key: torrent.Name, value: torrent listings atomic.Value downloadLinks map[string]string // key: file.Link, value: download link - propfindResp sync.Map + PropfindResp sync.Map workers int @@ -63,13 +60,28 @@ type Cache struct { downloadLinksMutex sync.Mutex // for downloadLinks } +type fileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time + isDir bool +} + +func (fi *fileInfo) Name() string { return fi.name } +func (fi *fileInfo) Size() int64 { return fi.size } +func (fi *fileInfo) Mode() os.FileMode { return fi.mode } +func (fi *fileInfo) ModTime() time.Time { return fi.modTime } +func (fi *fileInfo) IsDir() bool { return fi.isDir } +func (fi *fileInfo) Sys() interface{} { return nil } + func (c *Cache) setTorrent(t *CachedTorrent) { c.torrentsMutex.Lock() c.torrents[t.Id] = t c.torrentsNames[t.Name] = t c.torrentsMutex.Unlock() - go c.refreshListings() // This is concurrent safe + tryLock(&c.listingRefreshMu, c.refreshListings) go func() { if err := c.SaveTorrent(t); err != nil { @@ -78,36 +90,6 @@ func (c *Cache) setTorrent(t *CachedTorrent) { }() } -func (c *Cache) refreshListings() { - // Copy the current torrents to avoid concurrent issues - c.torrentsMutex.RLock() - torrents := make([]string, 0, len(c.torrents)) - for _, t := range c.torrents { - if t != nil && t.Torrent != nil { - torrents = append(torrents, t.Name) - } - } - c.torrentsMutex.RUnlock() - - sort.Slice(torrents, func(i, j int) bool { - return torrents[i] < torrents[j] - }) - - files := make([]os.FileInfo, 0, len(torrents)) - now := time.Now() - for _, t := range torrents { - files = append(files, &FileInfo{ - name: t, - size: 0, - mode: 0755 | os.ModeDir, - modTime: now, - isDir: true, - }) - } - // Atomic store of the complete ready-to-use slice - c.listings.Store(files) -} - func (c *Cache) GetListing() []os.FileInfo { if v, ok := c.listings.Load().([]os.FileInfo); ok { return v @@ -124,7 +106,7 @@ func (c *Cache) setTorrents(torrents map[string]*CachedTorrent) { c.torrentsMutex.Unlock() - go c.refreshListings() // This is concurrent safe + tryLock(&c.listingRefreshMu, c.refreshListings) go func() { if err := c.SaveTorrents(); err != nil { @@ -149,31 +131,7 @@ func (c *Cache) GetTorrentNames() map[string]*CachedTorrent { return c.torrentsNames } -type Manager struct { - caches map[string]*Cache -} - -func NewCacheManager(clients []debrid.Client) *Manager { - m := &Manager{ - caches: make(map[string]*Cache), - } - - for _, client := range clients { - m.caches[client.GetName()] = NewCache(client) - } - - return m -} - -func (m *Manager) GetCaches() map[string]*Cache { - return m.caches -} - -func (m *Manager) GetCache(debridName string) *Cache { - return m.caches[debridName] -} - -func NewCache(client debrid.Client) *Cache { +func NewCache(client types.Client) *Cache { cfg := config.GetConfig() dbPath := filepath.Join(cfg.Path, "cache", client.GetName()) return &Cache{ @@ -202,7 +160,7 @@ func (c *Cache) Start() error { c.downloadLinksRefreshMu.Lock() defer c.downloadLinksRefreshMu.Unlock() // This prevents the download links from being refreshed twice - c.refreshDownloadLinks() + tryLock(&c.downloadLinksRefreshMu, c.refreshDownloadLinks) }() go func() { @@ -216,9 +174,6 @@ func (c *Cache) Start() error { } func (c *Cache) Close() error { - if c.db != nil { - return c.db.Close() - } return nil } @@ -327,7 +282,7 @@ func (c *Cache) Sync() error { c.logger.Info().Msgf("Got %d torrents from %s", len(torrents), c.client.GetName()) - newTorrents := make([]*torrent.Torrent, 0) + newTorrents := make([]*types.Torrent, 0) idStore := make(map[string]bool, len(torrents)) for _, t := range torrents { idStore[t.Id] = true @@ -368,12 +323,12 @@ func (c *Cache) Sync() error { return nil } -func (c *Cache) sync(torrents []*torrent.Torrent) error { +func (c *Cache) sync(torrents []*types.Torrent) error { // Calculate optimal workers - balance between CPU and IO workers := runtime.NumCPU() * 50 // A more balanced multiplier for BadgerDB // Create channels with appropriate buffering - workChan := make(chan *torrent.Torrent, workers*2) + workChan := make(chan *types.Torrent, workers*2) // Use an atomic counter for progress tracking var processed int64 @@ -398,7 +353,7 @@ func (c *Cache) sync(torrents []*torrent.Torrent) error { return // Channel closed, exit goroutine } - if err := c.processTorrent(t); err != nil { + if err := c.ProcessTorrent(t, true); err != nil { c.logger.Error().Err(err).Str("torrent", t.Name).Msg("sync error") atomic.AddInt64(&errorCount, 1) } @@ -435,11 +390,11 @@ func (c *Cache) sync(torrents []*torrent.Torrent) error { return nil } -func (c *Cache) processTorrent(t *torrent.Torrent) error { - var err error - err = c.client.UpdateTorrent(t) - if err != nil { - return fmt.Errorf("failed to get torrent files: %v", err) +func (c *Cache) ProcessTorrent(t *types.Torrent, refreshRclone bool) error { + if len(t.Files) == 0 { + if err := c.client.UpdateTorrent(t); err != nil { + return fmt.Errorf("failed to update torrent: %w", err) + } } ct := &CachedTorrent{ @@ -448,6 +403,9 @@ func (c *Cache) processTorrent(t *torrent.Torrent) error { IsComplete: len(t.Files) > 0, } c.setTorrent(ct) + if err := c.RefreshRclone(); err != nil { + c.logger.Debug().Err(err).Msg("Failed to refresh rclone") + } return nil } @@ -469,7 +427,7 @@ func (c *Cache) GetDownloadLink(torrentId, filename, fileLink string) string { if ct.IsComplete { return "" } - ct = c.refreshTorrent(ct) // Refresh the torrent from the debrid service + ct = c.refreshTorrent(ct) // Refresh the torrent from the debrid if ct == nil { return "" } else { @@ -477,7 +435,7 @@ func (c *Cache) GetDownloadLink(torrentId, filename, fileLink string) string { } } - c.logger.Debug().Msgf("Getting download link for %s", ct.Name) + c.logger.Trace().Msgf("Getting download link for %s", ct.Name) f := c.client.GetDownloadLink(ct.Torrent, &file) if f == nil { return "" @@ -490,7 +448,7 @@ func (c *Cache) GetDownloadLink(torrentId, filename, fileLink string) string { return f.DownloadLink } -func (c *Cache) updateDownloadLink(file *torrent.File) { +func (c *Cache) updateDownloadLink(file *types.File) { c.downloadLinksMutex.Lock() defer c.downloadLinksMutex.Unlock() c.downloadLinks[file.Link] = file.DownloadLink @@ -503,111 +461,10 @@ func (c *Cache) checkDownloadLink(link string) string { return "" } -func (c *Cache) refreshTorrent(t *CachedTorrent) *CachedTorrent { - _torrent := t.Torrent - err := c.client.UpdateTorrent(_torrent) - if err != nil { - c.logger.Debug().Msgf("Failed to get torrent files for %s: %v", t.Id, err) - return nil - } - if len(t.Files) == 0 { - return nil - } - - ct := &CachedTorrent{ - Torrent: _torrent, - LastRead: time.Now(), - IsComplete: len(t.Files) > 0, - } - c.setTorrent(ct) - - return ct -} - -func (c *Cache) refreshDownloadLinks() map[string]string { - c.downloadLinksMutex.Lock() - defer c.downloadLinksMutex.Unlock() - - downloadLinks, err := c.client.GetDownloads() - if err != nil { - c.logger.Debug().Err(err).Msg("Failed to get download links") - return nil - } - for k, v := range downloadLinks { - c.downloadLinks[k] = v.DownloadLink - } - return c.downloadLinks -} - -func (c *Cache) GetClient() debrid.Client { +func (c *Cache) GetClient() types.Client { return c.client } -func (c *Cache) refreshTorrents() { - c.torrentsMutex.RLock() - currentTorrents := c.torrents // - // Create a copy of the current torrents to avoid concurrent issues - torrents := make(map[string]string, len(currentTorrents)) // a mpa of id and name - for _, v := range currentTorrents { - torrents[v.Id] = v.Name - } - c.torrentsMutex.RUnlock() - - // Get new torrents from the debrid service - debTorrents, err := c.client.GetTorrents() - if err != nil { - c.logger.Debug().Err(err).Msg("Failed to get torrents") - return - } - - if len(debTorrents) == 0 { - // Maybe an error occurred - return - } - - // Get the newly added torrents only - newTorrents := make([]*torrent.Torrent, 0) - idStore := make(map[string]bool, len(debTorrents)) - for _, t := range debTorrents { - idStore[t.Id] = true - if _, ok := torrents[t.Id]; !ok { - newTorrents = append(newTorrents, t) - } - } - - // Check for deleted torrents - deletedTorrents := make([]string, 0) - for id, _ := range torrents { - if _, ok := idStore[id]; !ok { - deletedTorrents = append(deletedTorrents, id) - } - } - - if len(deletedTorrents) > 0 { - c.DeleteTorrent(deletedTorrents) - } - - if len(newTorrents) == 0 { - return - } - c.logger.Info().Msgf("Found %d new torrents", len(newTorrents)) - - // No need for a complex sync process, just add the new torrents - wg := sync.WaitGroup{} - wg.Add(len(newTorrents)) - for _, t := range newTorrents { - // processTorrent is concurrent safe - go func() { - defer wg.Done() - if err := c.processTorrent(t); err != nil { - c.logger.Info().Err(err).Msg("Failed to process torrent") - } - - }() - } - wg.Wait() -} - func (c *Cache) DeleteTorrent(ids []string) { c.logger.Info().Msgf("Deleting %d torrents", len(ids)) c.torrentsMutex.Lock() @@ -628,25 +485,7 @@ func (c *Cache) removeFromDB(torrentId string) { } } -func (c *Cache) resetPropfindResponse() { - // Right now, parents are hardcoded - parents := []string{"__all__", "torrents"} - // Reset only the parent directories - // Convert the parents to a keys - // This is a bit hacky, but it works - // Instead of deleting all the keys, we only delete the parent keys, e.g __all__/ or torrents/ - keys := make([]string, 0, len(parents)) - for _, p := range parents { - // Construct the key - // construct url - url := filepath.Join("/webdav/%s/%s", c.client.GetName(), p) - key0 := fmt.Sprintf("propfind:%s:0", url) - key1 := fmt.Sprintf("propfind:%s:1", url) - keys = append(keys, key0, key1) - } - - // Delete the keys - for _, k := range keys { - c.propfindResp.Delete(k) - } +func (c *Cache) OnRemove(torrentId string) { + go c.DeleteTorrent([]string{torrentId}) + go tryLock(&c.listingRefreshMu, c.refreshListings) } diff --git a/pkg/debrid/debrid/debrid.go b/pkg/debrid/debrid/debrid.go index 393abf1..f26bbb0 100644 --- a/pkg/debrid/debrid/debrid.go +++ b/pkg/debrid/debrid/debrid.go @@ -1,24 +1,86 @@ package debrid import ( - "github.com/rs/zerolog" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" + "fmt" + "github.com/sirrobot01/debrid-blackhole/internal/config" + "github.com/sirrobot01/debrid-blackhole/internal/utils" + "github.com/sirrobot01/debrid-blackhole/pkg/arr" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/alldebrid" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid_link" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/realdebrid" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torbox" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" ) -type Client interface { - SubmitMagnet(tr *torrent.Torrent) (*torrent.Torrent, error) - CheckStatus(tr *torrent.Torrent, isSymlink bool) (*torrent.Torrent, error) - GenerateDownloadLinks(tr *torrent.Torrent) error - GetDownloadLink(tr *torrent.Torrent, file *torrent.File) *torrent.File - ConvertLinksToFiles(links []string) []torrent.File - DeleteTorrent(tr *torrent.Torrent) - IsAvailable(infohashes []string) map[string]bool - GetCheckCached() bool - GetDownloadUncached() bool - UpdateTorrent(torrent *torrent.Torrent) error - GetTorrents() ([]*torrent.Torrent, error) - GetName() string - GetLogger() zerolog.Logger - GetDownloadingStatus() []string - GetDownloads() (map[string]torrent.DownloadLinks, error) +func createDebridClient(dc config.Debrid) types.Client { + switch dc.Name { + case "realdebrid": + return realdebrid.New(dc) + case "torbox": + return torbox.New(dc) + case "debridlink": + return debrid_link.New(dc) + case "alldebrid": + return alldebrid.New(dc) + default: + return realdebrid.New(dc) + } +} + +func ProcessTorrent(d *Engine, magnet *utils.Magnet, a *arr.Arr, isSymlink, overrideDownloadUncached bool) (*types.Torrent, error) { + + debridTorrent := &types.Torrent{ + InfoHash: magnet.InfoHash, + Magnet: magnet, + Name: magnet.Name, + Arr: a, + Size: magnet.Size, + Files: make(map[string]types.File), + } + + errs := make([]error, 0) + + for index, db := range d.Clients { + logger := db.GetLogger() + logger.Info().Msgf("Processing debrid: %s", db.GetName()) + + // Override first, arr second, debrid third + + if overrideDownloadUncached { + debridTorrent.DownloadUncached = true + } else if a.DownloadUncached != nil { + // Arr cached is set + debridTorrent.DownloadUncached = *a.DownloadUncached + } else { + debridTorrent.DownloadUncached = db.GetDownloadUncached() + } + + logger.Info().Msgf("Torrent Hash: %s", debridTorrent.InfoHash) + if db.GetCheckCached() { + hash, exists := db.IsAvailable([]string{debridTorrent.InfoHash})[debridTorrent.InfoHash] + if !exists || !hash { + logger.Info().Msgf("Torrent: %s is not cached", debridTorrent.Name) + continue + } else { + logger.Info().Msgf("Torrent: %s is cached(or downloading)", debridTorrent.Name) + } + } + + dbt, err := db.SubmitMagnet(debridTorrent) + if dbt != nil { + dbt.Arr = a + } + if err != nil || dbt == nil || dbt.Id == "" { + errs = append(errs, err) + continue + } + logger.Info().Msgf("Torrent: %s(id=%s) submitted to %s", dbt.Name, dbt.Id, db.GetName()) + d.LastUsed = index + return db.CheckStatus(dbt, isSymlink) + } + err := fmt.Errorf("failed to process torrent") + for _, e := range errs { + err = fmt.Errorf("%w\n%w", err, e) + } + return nil, err } diff --git a/pkg/debrid/debrid/engine.go b/pkg/debrid/debrid/engine.go new file mode 100644 index 0000000..61ded2d --- /dev/null +++ b/pkg/debrid/debrid/engine.go @@ -0,0 +1,51 @@ +package debrid + +import ( + "github.com/sirrobot01/debrid-blackhole/internal/config" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" +) + +type Engine struct { + Clients map[string]types.Client + Caches map[string]*Cache + LastUsed string +} + +func NewEngine() *Engine { + cfg := config.GetConfig() + clients := make(map[string]types.Client) + + caches := make(map[string]*Cache) + + for _, dc := range cfg.Debrids { + client := createDebridClient(dc) + logger := client.GetLogger() + logger.Info().Msg("Debrid Service started") + clients[dc.Name] = client + caches[dc.Name] = NewCache(client) + } + + d := &Engine{ + Clients: clients, + LastUsed: "", + Caches: caches, + } + return d +} + +func (d *Engine) Get() types.Client { + if d.LastUsed == "" { + for _, c := range d.Clients { + return c + } + } + return d.Clients[d.LastUsed] +} + +func (d *Engine) GetByName(name string) types.Client { + return d.Clients[name] +} + +func (d *Engine) GetDebrids() map[string]types.Client { + return d.Clients +} diff --git a/pkg/debrid/debrid/misc.go b/pkg/debrid/debrid/misc.go new file mode 100644 index 0000000..beb6f04 --- /dev/null +++ b/pkg/debrid/debrid/misc.go @@ -0,0 +1,10 @@ +package debrid + +import "sync" + +func tryLock(mu *sync.Mutex, f func()) { + if mu.TryLock() { + defer mu.Unlock() + f() + } +} diff --git a/pkg/debrid/debrid/refresh.go b/pkg/debrid/debrid/refresh.go new file mode 100644 index 0000000..abe0b39 --- /dev/null +++ b/pkg/debrid/debrid/refresh.go @@ -0,0 +1,205 @@ +package debrid + +import ( + "bytes" + "fmt" + "github.com/goccy/go-json" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" + "net/http" + "os" + "path" + "path/filepath" + "sort" + "sync" + "time" +) + +func (c *Cache) refreshListings() { + // Copy the current torrents to avoid concurrent issues + c.torrentsMutex.RLock() + torrents := make([]string, 0, len(c.torrents)) + for _, t := range c.torrents { + if t != nil && t.Torrent != nil { + torrents = append(torrents, t.Name) + } + } + c.torrentsMutex.RUnlock() + + sort.Slice(torrents, func(i, j int) bool { + return torrents[i] < torrents[j] + }) + + files := make([]os.FileInfo, 0, len(torrents)) + now := time.Now() + for _, t := range torrents { + files = append(files, &fileInfo{ + name: t, + size: 0, + mode: 0755 | os.ModeDir, + modTime: now, + isDir: true, + }) + } + // Atomic store of the complete ready-to-use slice + c.listings.Store(files) + c.resetPropfindResponse() + if err := c.RefreshRclone(); err != nil { + c.logger.Debug().Err(err).Msg("Failed to refresh rclone") + } +} + +func (c *Cache) refreshTorrents() { + c.torrentsMutex.RLock() + currentTorrents := c.torrents // + // Create a copy of the current torrents to avoid concurrent issues + torrents := make(map[string]string, len(currentTorrents)) // a mpa of id and name + for _, v := range currentTorrents { + torrents[v.Id] = v.Name + } + c.torrentsMutex.RUnlock() + + // Get new torrents from the debrid service + debTorrents, err := c.client.GetTorrents() + if err != nil { + c.logger.Debug().Err(err).Msg("Failed to get torrents") + return + } + + if len(debTorrents) == 0 { + // Maybe an error occurred + return + } + + // Get the newly added torrents only + newTorrents := make([]*types.Torrent, 0) + idStore := make(map[string]bool, len(debTorrents)) + for _, t := range debTorrents { + idStore[t.Id] = true + if _, ok := torrents[t.Id]; !ok { + newTorrents = append(newTorrents, t) + } + } + + // Check for deleted torrents + deletedTorrents := make([]string, 0) + for id, _ := range torrents { + if _, ok := idStore[id]; !ok { + deletedTorrents = append(deletedTorrents, id) + } + } + + if len(deletedTorrents) > 0 { + c.DeleteTorrent(deletedTorrents) + } + + if len(newTorrents) == 0 { + return + } + c.logger.Info().Msgf("Found %d new torrents", len(newTorrents)) + + // No need for a complex sync process, just add the new torrents + wg := sync.WaitGroup{} + wg.Add(len(newTorrents)) + for _, t := range newTorrents { + // ProcessTorrent is concurrent safe + go func() { + defer wg.Done() + if err := c.ProcessTorrent(t, true); err != nil { + c.logger.Info().Err(err).Msg("Failed to process torrent") + } + + }() + } + wg.Wait() +} + +func (c *Cache) RefreshRclone() error { + params := map[string]interface{}{ + "recursive": "false", + } + + // Convert parameters to JSON + jsonParams, err := json.Marshal(params) + if err != nil { + return err + } + + // Create HTTP request + url := "http://192.168.0.219:9990/vfs/refresh" // Switch to config + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonParams)) + if err != nil { + return err + } + + // Set the appropriate headers + req.Header.Set("Content-Type", "application/json") + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + if resp.StatusCode != 200 { + return fmt.Errorf("failed to refresh rclone: %s", resp.Status) + } + return nil +} + +func (c *Cache) refreshTorrent(t *CachedTorrent) *CachedTorrent { + _torrent := t.Torrent + err := c.client.UpdateTorrent(_torrent) + if err != nil { + c.logger.Debug().Msgf("Failed to get torrent files for %s: %v", t.Id, err) + return nil + } + if len(t.Files) == 0 { + return nil + } + + ct := &CachedTorrent{ + Torrent: _torrent, + LastRead: time.Now(), + IsComplete: len(t.Files) > 0, + } + c.setTorrent(ct) + + return ct +} + +func (c *Cache) refreshDownloadLinks() { + c.downloadLinksMutex.Lock() + defer c.downloadLinksMutex.Unlock() + + downloadLinks, err := c.client.GetDownloads() + if err != nil { + c.logger.Debug().Err(err).Msg("Failed to get download links") + } + for k, v := range downloadLinks { + c.downloadLinks[k] = v.DownloadLink + } +} + +func (c *Cache) resetPropfindResponse() { + // Right now, parents are hardcoded + parents := []string{"__all__", "torrents"} + // Reset only the parent directories + // Convert the parents to a keys + // This is a bit hacky, but it works + // Instead of deleting all the keys, we only delete the parent keys, e.g __all__/ or torrents/ + keys := make([]string, 0, len(parents)) + for _, p := range parents { + // Construct the key + // construct url + url := filepath.Join("/webdav", c.client.GetName(), p) + url = path.Clean(url) + key0 := fmt.Sprintf("propfind:%s:0", url) + key1 := fmt.Sprintf("propfind:%s:1", url) + keys = append(keys, key0, key1) + } + + // Delete the keys + for _, k := range keys { + c.PropfindResp.Delete(k) + } +} diff --git a/pkg/debrid/debrid/workers.go b/pkg/debrid/debrid/workers.go new file mode 100644 index 0000000..a0de397 --- /dev/null +++ b/pkg/debrid/debrid/workers.go @@ -0,0 +1,35 @@ +package debrid + +import "time" + +func (c *Cache) Refresh() error { + // For now, we just want to refresh the listing and download links + c.logger.Info().Msg("Starting cache refresh workers") + go c.refreshDownloadLinksWorker() + go c.refreshTorrentsWorker() + return nil +} + +func (c *Cache) refreshDownloadLinksWorker() { + refreshTicker := time.NewTicker(40 * time.Minute) + defer refreshTicker.Stop() + + for { + select { + case <-refreshTicker.C: + tryLock(&c.downloadLinksRefreshMu, c.refreshDownloadLinks) + } + } +} + +func (c *Cache) refreshTorrentsWorker() { + refreshTicker := time.NewTicker(5 * time.Second) + defer refreshTicker.Stop() + + for { + select { + case <-refreshTicker.C: + tryLock(&c.torrentsRefreshMu, c.refreshTorrents) + } + } +} diff --git a/pkg/debrid/debrid_link/debrid_link.go b/pkg/debrid/debrid_link/debrid_link.go index a65b624..fd19172 100644 --- a/pkg/debrid/debrid_link/debrid_link.go +++ b/pkg/debrid/debrid_link/debrid_link.go @@ -9,7 +9,7 @@ import ( "github.com/sirrobot01/debrid-blackhole/internal/logger" "github.com/sirrobot01/debrid-blackhole/internal/request" "github.com/sirrobot01/debrid-blackhole/internal/utils" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" "slices" "time" @@ -89,7 +89,7 @@ func (dl *DebridLink) IsAvailable(hashes []string) map[string]bool { return result } -func (dl *DebridLink) UpdateTorrent(t *torrent.Torrent) error { +func (dl *DebridLink) UpdateTorrent(t *types.Torrent) error { url := fmt.Sprintf("%s/seedbox/list?ids=%s", dl.Host, t.Id) req, _ := http.NewRequest(http.MethodGet, url, nil) resp, err := dl.client.MakeRequest(req) @@ -133,7 +133,7 @@ func (dl *DebridLink) UpdateTorrent(t *torrent.Torrent) error { if !cfg.IsSizeAllowed(f.Size) { continue } - file := torrent.File{ + file := types.File{ Id: f.ID, Name: f.Name, Size: f.Size, @@ -146,7 +146,7 @@ func (dl *DebridLink) UpdateTorrent(t *torrent.Torrent) error { return nil } -func (dl *DebridLink) SubmitMagnet(t *torrent.Torrent) (*torrent.Torrent, error) { +func (dl *DebridLink) SubmitMagnet(t *types.Torrent) (*types.Torrent, error) { url := fmt.Sprintf("%s/seedbox/add", dl.Host) payload := map[string]string{"url": t.Magnet.Link} jsonPayload, _ := json.Marshal(payload) @@ -179,7 +179,7 @@ func (dl *DebridLink) SubmitMagnet(t *torrent.Torrent) (*torrent.Torrent, error) t.MountPath = dl.MountPath t.Debrid = dl.Name for _, f := range data.Files { - file := torrent.File{ + file := types.File{ Id: f.ID, Name: f.Name, Size: f.Size, @@ -194,7 +194,7 @@ func (dl *DebridLink) SubmitMagnet(t *torrent.Torrent) (*torrent.Torrent, error) return t, nil } -func (dl *DebridLink) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*torrent.Torrent, error) { +func (dl *DebridLink) CheckStatus(torrent *types.Torrent, isSymlink bool) (*types.Torrent, error) { for { err := dl.UpdateTorrent(torrent) if err != nil || torrent == nil { @@ -223,7 +223,7 @@ func (dl *DebridLink) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*to return torrent, nil } -func (dl *DebridLink) DeleteTorrent(torrent *torrent.Torrent) { +func (dl *DebridLink) DeleteTorrent(torrent *types.Torrent) { url := fmt.Sprintf("%s/seedbox/%s/remove", dl.Host, torrent.Id) req, _ := http.NewRequest(http.MethodDelete, url, nil) _, err := dl.client.MakeRequest(req) @@ -234,15 +234,15 @@ func (dl *DebridLink) DeleteTorrent(torrent *torrent.Torrent) { } } -func (dl *DebridLink) GenerateDownloadLinks(t *torrent.Torrent) error { +func (dl *DebridLink) GenerateDownloadLinks(t *types.Torrent) error { return nil } -func (dl *DebridLink) GetDownloads() (map[string]torrent.DownloadLinks, error) { +func (dl *DebridLink) GetDownloads() (map[string]types.DownloadLinks, error) { return nil, nil } -func (dl *DebridLink) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.File { +func (dl *DebridLink) GetDownloadLink(t *types.Torrent, file *types.File) *types.File { return file } @@ -280,10 +280,10 @@ func New(dc config.Debrid) *DebridLink { } } -func (dl *DebridLink) GetTorrents() ([]*torrent.Torrent, error) { +func (dl *DebridLink) GetTorrents() ([]*types.Torrent, error) { return nil, nil } -func (dl *DebridLink) ConvertLinksToFiles(links []string) []torrent.File { +func (dl *DebridLink) ConvertLinksToFiles(links []string) []types.File { return nil } diff --git a/pkg/debrid/engine/engine.go b/pkg/debrid/engine/engine.go deleted file mode 100644 index 6662ff0..0000000 --- a/pkg/debrid/engine/engine.go +++ /dev/null @@ -1,30 +0,0 @@ -package engine - -import ( - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid" -) - -type Engine struct { - Debrids []debrid.Client - LastUsed int -} - -func (d *Engine) Get() debrid.Client { - if d.LastUsed == 0 { - return d.Debrids[0] - } - return d.Debrids[d.LastUsed] -} - -func (d *Engine) GetByName(name string) debrid.Client { - for _, deb := range d.Debrids { - if deb.GetName() == name { - return deb - } - } - return nil -} - -func (d *Engine) GetDebrids() []debrid.Client { - return d.Debrids -} diff --git a/pkg/debrid/realdebrid/realdebrid.go b/pkg/debrid/realdebrid/realdebrid.go index c018ca6..fa94275 100644 --- a/pkg/debrid/realdebrid/realdebrid.go +++ b/pkg/debrid/realdebrid/realdebrid.go @@ -8,7 +8,7 @@ import ( "github.com/sirrobot01/debrid-blackhole/internal/logger" "github.com/sirrobot01/debrid-blackhole/internal/request" "github.com/sirrobot01/debrid-blackhole/internal/utils" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" "io" "net/http" gourl "net/url" @@ -43,8 +43,8 @@ func (r *RealDebrid) GetLogger() zerolog.Logger { // getTorrentFiles returns a list of torrent files from the torrent info // validate is used to determine if the files should be validated // if validate is false, selected files will be returned -func getTorrentFiles(t *torrent.Torrent, data TorrentInfo, validate bool) map[string]torrent.File { - files := make(map[string]torrent.File) +func getTorrentFiles(t *types.Torrent, data TorrentInfo, validate bool) map[string]types.File { + files := make(map[string]types.File) cfg := config.GetConfig() idx := 0 for _, f := range data.Files { @@ -80,7 +80,7 @@ func getTorrentFiles(t *torrent.Torrent, data TorrentInfo, validate bool) map[st continue } - file := torrent.File{ + file := types.File{ Name: name, Path: name, Size: f.Bytes, @@ -141,7 +141,7 @@ func (r *RealDebrid) IsAvailable(hashes []string) map[string]bool { return result } -func (r *RealDebrid) SubmitMagnet(t *torrent.Torrent) (*torrent.Torrent, error) { +func (r *RealDebrid) SubmitMagnet(t *types.Torrent) (*types.Torrent, error) { url := fmt.Sprintf("%s/torrents/addMagnet", r.Host) payload := gourl.Values{ "magnet": {t.Magnet.Link}, @@ -161,7 +161,7 @@ func (r *RealDebrid) SubmitMagnet(t *torrent.Torrent) (*torrent.Torrent, error) return t, nil } -func (r *RealDebrid) UpdateTorrent(t *torrent.Torrent) error { +func (r *RealDebrid) UpdateTorrent(t *types.Torrent) error { url := fmt.Sprintf("%s/torrents/info/%s", r.Host, t.Id) req, _ := http.NewRequest(http.MethodGet, url, nil) resp, err := r.client.MakeRequest(req) @@ -173,7 +173,7 @@ func (r *RealDebrid) UpdateTorrent(t *torrent.Torrent) error { if err != nil { return err } - name := utils.RemoveExtension(data.OriginalFilename) + name := utils.RemoveInvalidChars(data.OriginalFilename) t.Name = name t.Bytes = data.Bytes t.Folder = name @@ -190,7 +190,7 @@ func (r *RealDebrid) UpdateTorrent(t *torrent.Torrent) error { return nil } -func (r *RealDebrid) CheckStatus(t *torrent.Torrent, isSymlink bool) (*torrent.Torrent, error) { +func (r *RealDebrid) CheckStatus(t *types.Torrent, isSymlink bool) (*types.Torrent, error) { url := fmt.Sprintf("%s/torrents/info/%s", r.Host, t.Id) req, _ := http.NewRequest(http.MethodGet, url, nil) for { @@ -204,7 +204,7 @@ func (r *RealDebrid) CheckStatus(t *torrent.Torrent, isSymlink bool) (*torrent.T return t, err } status := data.Status - name := utils.RemoveInvalidChars(data.OriginalFilename) + name := utils.RemoveExtension(data.OriginalFilename) t.Name = name // Important because some magnet changes the name t.Folder = name t.Filename = data.Filename @@ -257,7 +257,7 @@ func (r *RealDebrid) CheckStatus(t *torrent.Torrent, isSymlink bool) (*torrent.T return t, nil } -func (r *RealDebrid) DeleteTorrent(torrent *torrent.Torrent) { +func (r *RealDebrid) DeleteTorrent(torrent *types.Torrent) { url := fmt.Sprintf("%s/torrents/delete/%s", r.Host, torrent.Id) req, _ := http.NewRequest(http.MethodDelete, url, nil) _, err := r.client.MakeRequest(req) @@ -268,7 +268,7 @@ func (r *RealDebrid) DeleteTorrent(torrent *torrent.Torrent) { } } -func (r *RealDebrid) GenerateDownloadLinks(t *torrent.Torrent) error { +func (r *RealDebrid) GenerateDownloadLinks(t *types.Torrent) error { url := fmt.Sprintf("%s/unrestrict/link/", r.Host) for _, f := range t.Files { if f.DownloadLink != "" { @@ -294,8 +294,8 @@ func (r *RealDebrid) GenerateDownloadLinks(t *torrent.Torrent) error { return nil } -func (r *RealDebrid) ConvertLinksToFiles(links []string) []torrent.File { - files := make([]torrent.File, 0) +func (r *RealDebrid) ConvertLinksToFiles(links []string) []types.File { + files := make([]types.File, 0) for _, l := range links { url := fmt.Sprintf("%s/unrestrict/link/", r.Host) payload := gourl.Values{ @@ -310,7 +310,7 @@ func (r *RealDebrid) ConvertLinksToFiles(links []string) []torrent.File { if err = json.Unmarshal(resp, &data); err != nil { continue } - files = append(files, torrent.File{ + files = append(files, types.File{ Name: data.Filename, Size: data.Filesize, Link: l, @@ -321,7 +321,7 @@ func (r *RealDebrid) ConvertLinksToFiles(links []string) []torrent.File { return files } -func (r *RealDebrid) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.File { +func (r *RealDebrid) GetDownloadLink(t *types.Torrent, file *types.File) *types.File { url := fmt.Sprintf("%s/unrestrict/link/", r.Host) payload := gourl.Values{ "link": {file.Link}, @@ -344,9 +344,9 @@ func (r *RealDebrid) GetCheckCached() bool { return r.CheckCached } -func (r *RealDebrid) getTorrents(offset int, limit int) (int, []*torrent.Torrent, error) { +func (r *RealDebrid) getTorrents(offset int, limit int) (int, []*types.Torrent, error) { url := fmt.Sprintf("%s/torrents?limit=%d", r.Host, limit) - torrents := make([]*torrent.Torrent, 0) + torrents := make([]*types.Torrent, 0) if offset > 0 { url = fmt.Sprintf("%s&offset=%d", url, offset) } @@ -374,10 +374,13 @@ func (r *RealDebrid) getTorrents(offset int, limit int) (int, []*torrent.Torrent } filenames := map[string]bool{} for _, t := range data { + if t.Status != "downloaded" { + continue + } if _, exists := filenames[t.Filename]; exists { continue } - torrents = append(torrents, &torrent.Torrent{ + torrents = append(torrents, &types.Torrent{ Id: t.Id, Name: utils.RemoveInvalidChars(t.Filename), Bytes: t.Bytes, @@ -386,7 +389,7 @@ func (r *RealDebrid) getTorrents(offset int, limit int) (int, []*torrent.Torrent Filename: t.Filename, OriginalFilename: t.Filename, Links: t.Links, - Files: make(map[string]torrent.File), + Files: make(map[string]types.File), InfoHash: t.Hash, Debrid: r.Name, MountPath: r.MountPath, @@ -395,7 +398,7 @@ func (r *RealDebrid) getTorrents(offset int, limit int) (int, []*torrent.Torrent return totalItems, torrents, nil } -func (r *RealDebrid) GetTorrents() ([]*torrent.Torrent, error) { +func (r *RealDebrid) GetTorrents() ([]*types.Torrent, error) { limit := 5000 // Get first batch and total count @@ -449,8 +452,8 @@ func (r *RealDebrid) GetTorrents() ([]*torrent.Torrent, error) { return allTorrents, nil } -func (r *RealDebrid) GetDownloads() (map[string]torrent.DownloadLinks, error) { - links := make(map[string]torrent.DownloadLinks) +func (r *RealDebrid) GetDownloads() (map[string]types.DownloadLinks, error) { + links := make(map[string]types.DownloadLinks) offset := 0 limit := 5000 for { @@ -475,7 +478,7 @@ func (r *RealDebrid) GetDownloads() (map[string]torrent.DownloadLinks, error) { return links, nil } -func (r *RealDebrid) _getDownloads(offset int, limit int) ([]torrent.DownloadLinks, error) { +func (r *RealDebrid) _getDownloads(offset int, limit int) ([]types.DownloadLinks, error) { url := fmt.Sprintf("%s/downloads?limit=%d", r.Host, limit) if offset > 0 { url = fmt.Sprintf("%s&offset=%d", url, offset) @@ -489,9 +492,9 @@ func (r *RealDebrid) _getDownloads(offset int, limit int) ([]torrent.DownloadLin if err = json.Unmarshal(resp, &data); err != nil { return nil, err } - links := make([]torrent.DownloadLinks, 0) + links := make([]types.DownloadLinks, 0) for _, d := range data { - links = append(links, torrent.DownloadLinks{ + links = append(links, types.DownloadLinks{ Filename: d.Filename, Size: d.Filesize, Link: d.Link, diff --git a/pkg/debrid/torbox/torbox.go b/pkg/debrid/torbox/torbox.go index 4769c08..ce83f2d 100644 --- a/pkg/debrid/torbox/torbox.go +++ b/pkg/debrid/torbox/torbox.go @@ -9,7 +9,7 @@ import ( "github.com/sirrobot01/debrid-blackhole/internal/logger" "github.com/sirrobot01/debrid-blackhole/internal/request" "github.com/sirrobot01/debrid-blackhole/internal/utils" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" "time" "mime/multipart" @@ -93,7 +93,7 @@ func (tb *Torbox) IsAvailable(hashes []string) map[string]bool { return result } -func (tb *Torbox) SubmitMagnet(torrent *torrent.Torrent) (*torrent.Torrent, error) { +func (tb *Torbox) SubmitMagnet(torrent *types.Torrent) (*types.Torrent, error) { url := fmt.Sprintf("%s/api/torrents/createtorrent", tb.Host) payload := &bytes.Buffer{} writer := multipart.NewWriter(payload) @@ -141,7 +141,7 @@ func getTorboxStatus(status string, finished bool) string { } } -func (tb *Torbox) UpdateTorrent(t *torrent.Torrent) error { +func (tb *Torbox) UpdateTorrent(t *types.Torrent) error { url := fmt.Sprintf("%s/api/torrents/mylist/?id=%s", tb.Host, t.Id) req, _ := http.NewRequest(http.MethodGet, url, nil) resp, err := tb.client.MakeRequest(req) @@ -180,7 +180,7 @@ func (tb *Torbox) UpdateTorrent(t *torrent.Torrent) error { if !cfg.IsSizeAllowed(f.Size) { continue } - file := torrent.File{ + file := types.File{ Id: strconv.Itoa(f.Id), Name: fileName, Size: f.Size, @@ -200,7 +200,7 @@ func (tb *Torbox) UpdateTorrent(t *torrent.Torrent) error { return nil } -func (tb *Torbox) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*torrent.Torrent, error) { +func (tb *Torbox) CheckStatus(torrent *types.Torrent, isSymlink bool) (*types.Torrent, error) { for { err := tb.UpdateTorrent(torrent) @@ -232,7 +232,7 @@ func (tb *Torbox) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*torren return torrent, nil } -func (tb *Torbox) DeleteTorrent(torrent *torrent.Torrent) { +func (tb *Torbox) DeleteTorrent(torrent *types.Torrent) { url := fmt.Sprintf("%s/api/torrents/controltorrent/%s", tb.Host, torrent.Id) payload := map[string]string{"torrent_id": torrent.Id, "action": "Delete"} jsonPayload, _ := json.Marshal(payload) @@ -245,7 +245,7 @@ func (tb *Torbox) DeleteTorrent(torrent *torrent.Torrent) { } } -func (tb *Torbox) GenerateDownloadLinks(t *torrent.Torrent) error { +func (tb *Torbox) GenerateDownloadLinks(t *types.Torrent) error { for _, file := range t.Files { url := fmt.Sprintf("%s/api/torrents/requestdl/", tb.Host) query := gourl.Values{} @@ -273,7 +273,7 @@ func (tb *Torbox) GenerateDownloadLinks(t *torrent.Torrent) error { return nil } -func (tb *Torbox) GetDownloadLink(t *torrent.Torrent, file *torrent.File) *torrent.File { +func (tb *Torbox) GetDownloadLink(t *types.Torrent, file *types.File) *types.File { url := fmt.Sprintf("%s/api/torrents/requestdl/", tb.Host) query := gourl.Values{} query.Add("torrent_id", t.Id) @@ -306,7 +306,7 @@ func (tb *Torbox) GetCheckCached() bool { return tb.CheckCached } -func (tb *Torbox) GetTorrents() ([]*torrent.Torrent, error) { +func (tb *Torbox) GetTorrents() ([]*types.Torrent, error) { return nil, nil } @@ -336,10 +336,10 @@ func New(dc config.Debrid) *Torbox { } } -func (tb *Torbox) ConvertLinksToFiles(links []string) []torrent.File { +func (tb *Torbox) ConvertLinksToFiles(links []string) []types.File { return nil } -func (tb *Torbox) GetDownloads() (map[string]torrent.DownloadLinks, error) { +func (tb *Torbox) GetDownloads() (map[string]types.DownloadLinks, error) { return nil, nil } diff --git a/pkg/debrid/types/debrid.go b/pkg/debrid/types/debrid.go new file mode 100644 index 0000000..58b599a --- /dev/null +++ b/pkg/debrid/types/debrid.go @@ -0,0 +1,23 @@ +package types + +import ( + "github.com/rs/zerolog" +) + +type Client interface { + SubmitMagnet(tr *Torrent) (*Torrent, error) + CheckStatus(tr *Torrent, isSymlink bool) (*Torrent, error) + GenerateDownloadLinks(tr *Torrent) error + GetDownloadLink(tr *Torrent, file *File) *File + ConvertLinksToFiles(links []string) []File + DeleteTorrent(tr *Torrent) + IsAvailable(infohashes []string) map[string]bool + GetCheckCached() bool + GetDownloadUncached() bool + UpdateTorrent(torrent *Torrent) error + GetTorrents() ([]*Torrent, error) + GetName() string + GetLogger() zerolog.Logger + GetDownloadingStatus() []string + GetDownloads() (map[string]DownloadLinks, error) +} diff --git a/pkg/debrid/torrent/torrent.go b/pkg/debrid/types/torrent.go similarity index 99% rename from pkg/debrid/torrent/torrent.go rename to pkg/debrid/types/torrent.go index 2fbbdbb..39bb86f 100644 --- a/pkg/debrid/torrent/torrent.go +++ b/pkg/debrid/types/torrent.go @@ -1,4 +1,4 @@ -package torrent +package types import ( "fmt" diff --git a/pkg/qbit/downloader.go b/pkg/qbit/downloader.go index 2a5ba52..7f752c0 100644 --- a/pkg/qbit/downloader.go +++ b/pkg/qbit/downloader.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/cavaliergopher/grab/v3" "github.com/sirrobot01/debrid-blackhole/internal/utils" - debrid "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" + debrid "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" "io" "net/http" "os" @@ -154,8 +154,13 @@ func (q *QBit) ProcessSymlink(torrent *Torrent) (string, error) { torrentFolder = utils.RemoveExtension(torrentFolder) torrentRclonePath = rCloneBase // /mnt/rclone/magnets/ // Remove the filename since it's in the root folder } - torrentSymlinkPath := filepath.Join(q.DownloadFolder, debridTorrent.Arr.Name, torrentFolder) // /mnt/symlinks/{category}/MyTVShow/ - err = os.MkdirAll(torrentSymlinkPath, os.ModePerm) + return q.createSymlinks(debridTorrent, torrentRclonePath, torrentFolder) // verify cos we're using external webdav +} + +func (q *QBit) createSymlinks(debridTorrent *debrid.Torrent, rclonePath, torrentFolder string) (string, error) { + files := debridTorrent.Files + torrentSymlinkPath := filepath.Join(q.DownloadFolder, debridTorrent.Arr.Name, torrentFolder) + err := os.MkdirAll(torrentSymlinkPath, os.ModePerm) if err != nil { return "", fmt.Errorf("failed to create directory: %s: %v", torrentSymlinkPath, err) } @@ -164,16 +169,16 @@ func (q *QBit) ProcessSymlink(torrent *Torrent) (string, error) { for _, file := range files { pending[file.Path] = file } - ticker := time.NewTicker(200 * time.Millisecond) + ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for len(pending) > 0 { <-ticker.C for path, file := range pending { - fullFilePath := filepath.Join(torrentRclonePath, file.Path) + fullFilePath := filepath.Join(rclonePath, file.Path) if _, err := os.Stat(fullFilePath); !os.IsNotExist(err) { q.logger.Info().Msgf("File is ready: %s", file.Path) - q.createSymLink(torrentSymlinkPath, torrentRclonePath, file) + q.createSymLink(torrentSymlinkPath, rclonePath, file) delete(pending, path) } } diff --git a/pkg/qbit/import.go b/pkg/qbit/import.go index a46f21f..243a99e 100644 --- a/pkg/qbit/import.go +++ b/pkg/qbit/import.go @@ -3,12 +3,12 @@ package qbit import ( "fmt" "github.com/sirrobot01/debrid-blackhole/internal/utils" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid" "github.com/sirrobot01/debrid-blackhole/pkg/service" "time" "github.com/google/uuid" "github.com/sirrobot01/debrid-blackhole/pkg/arr" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid" ) type ImportRequest struct { diff --git a/pkg/qbit/torrent.go b/pkg/qbit/torrent.go index 95165e3..67c855e 100644 --- a/pkg/qbit/torrent.go +++ b/pkg/qbit/torrent.go @@ -7,8 +7,8 @@ import ( "github.com/sirrobot01/debrid-blackhole/internal/request" "github.com/sirrobot01/debrid-blackhole/internal/utils" "github.com/sirrobot01/debrid-blackhole/pkg/arr" - db "github.com/sirrobot01/debrid-blackhole/pkg/debrid" - debrid "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" + db "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid" + debrid "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" "github.com/sirrobot01/debrid-blackhole/pkg/service" "io" "mime/multipart" @@ -74,13 +74,14 @@ func (q *QBit) Process(ctx context.Context, magnet *utils.Magnet, category strin } func (q *QBit) ProcessFiles(torrent *Torrent, debridTorrent *debrid.Torrent, arr *arr.Arr, isSymlink bool) { - debridClient := service.GetDebrid().GetByName(debridTorrent.Debrid) + svc := service.GetService() + client := svc.Debrid.GetByName(debridTorrent.Debrid) for debridTorrent.Status != "downloaded" { q.logger.Debug().Msgf("%s <- (%s) Download Progress: %.2f%%", debridTorrent.Debrid, debridTorrent.Name, debridTorrent.Progress) - dbT, err := debridClient.CheckStatus(debridTorrent, isSymlink) + dbT, err := client.CheckStatus(debridTorrent, isSymlink) if err != nil { q.logger.Error().Msgf("Error checking status: %v", err) - go debridClient.DeleteTorrent(debridTorrent) + go client.DeleteTorrent(debridTorrent) q.MarkAsFailed(torrent) if err := arr.Refresh(); err != nil { q.logger.Error().Msgf("Error refreshing arr: %v", err) @@ -92,7 +93,7 @@ func (q *QBit) ProcessFiles(torrent *Torrent, debridTorrent *debrid.Torrent, arr torrent = q.UpdateTorrentMin(torrent, debridTorrent) // Exit the loop for downloading statuses to prevent memory buildup - if !slices.Contains(debridClient.GetDownloadingStatus(), debridTorrent.Status) { + if !slices.Contains(client.GetDownloadingStatus(), debridTorrent.Status) { break } time.Sleep(time.Duration(q.RefreshInterval) * time.Second) @@ -102,14 +103,51 @@ func (q *QBit) ProcessFiles(torrent *Torrent, debridTorrent *debrid.Torrent, arr err error ) debridTorrent.Arr = arr + + // File is done downloading at this stage + + // Check if debrid supports webdav by checking cache if isSymlink { - torrentSymlinkPath, err = q.ProcessSymlink(torrent) // /mnt/symlinks/{category}/MyTVShow/ + cache, ok := svc.Debrid.Caches[debridTorrent.Debrid] + if ok { + q.logger.Info().Msgf("Using internal webdav for %s", debridTorrent.Debrid) + // Use webdav to download the file + err := cache.ProcessTorrent(debridTorrent, true) + if err != nil { + return + } + rclonePath := filepath.Join(debridTorrent.MountPath, debridTorrent.Name) + + // Check if folder exists here + if _, err := os.Stat(rclonePath); os.IsNotExist(err) { + q.logger.Debug().Msgf("Folder does not exist: %s", rclonePath) + + // Check if torrent is in the listing + listing := cache.GetListing() + for _, t := range listing { + if t.Name() == debridTorrent.Name { + q.logger.Debug().Msgf("Torrent found in listing: %s", debridTorrent.Name) + } + } + + // Check if torrent is in the webdav + if t := cache.GetTorrentByName(debridTorrent.Name); t == nil { + q.logger.Debug().Msgf("Torrent not found in webdav: %s", debridTorrent.Name) + } + } + + torrentSymlinkPath, err = q.createSymlinks(debridTorrent, rclonePath, debridTorrent.Name) + + } else { + // User is using either zurg or debrid webdav + torrentSymlinkPath, err = q.ProcessSymlink(torrent) // /mnt/symlinks/{category}/MyTVShow/ + } } else { torrentSymlinkPath, err = q.ProcessManualFile(torrent) } if err != nil { q.MarkAsFailed(torrent) - go debridClient.DeleteTorrent(debridTorrent) + go client.DeleteTorrent(debridTorrent) q.logger.Info().Msgf("Error: %v", err) return } diff --git a/pkg/qbit/types.go b/pkg/qbit/types.go index eaa159c..5355ff6 100644 --- a/pkg/qbit/types.go +++ b/pkg/qbit/types.go @@ -2,7 +2,7 @@ package qbit import ( "fmt" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" "sync" ) @@ -173,10 +173,10 @@ type TorrentCategory struct { } type Torrent struct { - ID string `json:"id"` - DebridTorrent *torrent.Torrent `json:"-"` - Debrid string `json:"debrid"` - TorrentPath string `json:"-"` + ID string `json:"id"` + DebridTorrent *types.Torrent `json:"-"` + Debrid string `json:"debrid"` + TorrentPath string `json:"-"` AddedOn int64 `json:"added_on,omitempty"` AmountLeft int64 `json:"amount_left"` diff --git a/pkg/repair/repair.go b/pkg/repair/repair.go index 3365030..9e6f0f6 100644 --- a/pkg/repair/repair.go +++ b/pkg/repair/repair.go @@ -10,7 +10,7 @@ import ( "github.com/sirrobot01/debrid-blackhole/internal/logger" "github.com/sirrobot01/debrid-blackhole/internal/request" "github.com/sirrobot01/debrid-blackhole/pkg/arr" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" "golang.org/x/sync/errgroup" "net" "net/http" @@ -29,7 +29,7 @@ import ( type Repair struct { Jobs map[string]*Job arrs *arr.Storage - deb debrid.Client + deb types.Client duration time.Duration runOnStart bool ZurgURL string diff --git a/pkg/service/service.go b/pkg/service/service.go index 104a7c6..a5e0c12 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -2,8 +2,7 @@ package service import ( "github.com/sirrobot01/debrid-blackhole/pkg/arr" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/engine" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid" "github.com/sirrobot01/debrid-blackhole/pkg/repair" "sync" ) @@ -11,7 +10,7 @@ import ( type Service struct { Repair *repair.Repair Arr *arr.Storage - Debrid *engine.Engine + Debrid *debrid.Engine } var ( @@ -22,7 +21,7 @@ var ( func New() *Service { once.Do(func() { arrs := arr.NewStorage() - deb := debrid.New() + deb := debrid.NewEngine() instance = &Service{ Repair: repair.New(arrs), Arr: arrs, @@ -42,7 +41,7 @@ func GetService() *Service { func Update() *Service { arrs := arr.NewStorage() - deb := debrid.New() + deb := debrid.NewEngine() instance = &Service{ Repair: repair.New(arrs), Arr: arrs, @@ -51,6 +50,6 @@ func Update() *Service { return instance } -func GetDebrid() *engine.Engine { +func GetDebrid() *debrid.Engine { return GetService().Debrid } diff --git a/pkg/webdav/file.go b/pkg/webdav/file.go index 6fdc586..6c8b84a 100644 --- a/pkg/webdav/file.go +++ b/pkg/webdav/file.go @@ -2,6 +2,7 @@ package webdav import ( "fmt" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid" "io" "net/http" "os" @@ -20,18 +21,19 @@ var sharedClient = &http.Client{ } type File struct { - cache *Cache + cache *debrid.Cache fileId string torrentId string - size int64 - offset int64 - isDir bool - children []os.FileInfo - reader io.ReadCloser - seekPending bool - content []byte - name string + size int64 + offset int64 + isDir bool + children []os.FileInfo + reader io.ReadCloser + seekPending bool + content []byte + name string + metadataOnly bool downloadLink string link string @@ -49,11 +51,12 @@ func (f *File) Close() error { func (f *File) GetDownloadLink() string { // Check if we already have a final URL cached - if f.downloadLink != "" { + + if f.downloadLink != "" && isValidURL(f.downloadLink) { return f.downloadLink } downloadLink := f.cache.GetDownloadLink(f.torrentId, f.name, f.link) - if downloadLink != "" { + if downloadLink != "" && isValidURL(downloadLink) { f.downloadLink = downloadLink return downloadLink } @@ -65,6 +68,9 @@ func (f *File) Read(p []byte) (n int, err error) { if f.isDir { return 0, os.ErrInvalid } + if f.metadataOnly { + return 0, io.EOF + } // If file content is preloaded, read from memory. if f.content != nil { diff --git a/pkg/webdav/handler.go b/pkg/webdav/handler.go index 94d6656..b17acb2 100644 --- a/pkg/webdav/handler.go +++ b/pkg/webdav/handler.go @@ -2,11 +2,13 @@ package webdav import ( "bytes" + "compress/gzip" "context" "errors" "fmt" "github.com/rs/zerolog" - "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/types" "golang.org/x/net/webdav" "html/template" "io" @@ -14,7 +16,7 @@ import ( "net/http" "net/http/httptest" "os" - "path" + path "path/filepath" "slices" "strings" "sync" @@ -24,7 +26,7 @@ import ( type Handler struct { Name string logger zerolog.Logger - cache *Cache + cache *debrid.Cache lastRefresh time.Time refreshMutex sync.Mutex RootPath string @@ -33,7 +35,7 @@ type Handler struct { ctx context.Context } -func NewHandler(name string, cache *Cache, logger zerolog.Logger) *Handler { +func NewHandler(name string, cache *debrid.Cache, logger zerolog.Logger) *Handler { h := &Handler{ Name: name, cache: cache, @@ -67,9 +69,7 @@ func (h *Handler) RemoveAll(ctx context.Context, name string) error { if filename == "" { h.cache.GetClient().DeleteTorrent(cachedTorrent.Torrent) - go h.cache.refreshListings() - go h.cache.refreshTorrents() - go h.cache.resetPropfindResponse() + h.cache.OnRemove(cachedTorrent.Id) return nil } @@ -117,22 +117,29 @@ func (h *Handler) OpenFile(ctx context.Context, name string, flag int, perm os.F name = path.Clean("/" + name) rootDir := h.getRootPath() + metadataOnly := false + if ctx.Value("metadataOnly") != nil { + metadataOnly = true + } + // Fast path optimization with a map lookup instead of string comparisons switch name { case rootDir: return &File{ - cache: h.cache, - isDir: true, - children: h.getParentFiles(), - name: "/", + cache: h.cache, + isDir: true, + children: h.getParentFiles(), + name: "/", + metadataOnly: metadataOnly, }, nil case path.Join(rootDir, "version.txt"): return &File{ - cache: h.cache, - isDir: false, - content: []byte("v1.0.0"), - name: "version.txt", - size: int64(len("v1.0.0")), + cache: h.cache, + isDir: false, + content: []byte("v1.0.0"), + name: "version.txt", + size: int64(len("v1.0.0")), + metadataOnly: metadataOnly, }, nil } @@ -145,11 +152,12 @@ func (h *Handler) OpenFile(ctx context.Context, name string, flag int, perm os.F children := h.getTorrentsFolders() return &File{ - cache: h.cache, - isDir: true, - children: children, - name: folderName, - size: 0, + cache: h.cache, + isDir: true, + children: children, + name: folderName, + size: 0, + metadataOnly: metadataOnly, }, nil } @@ -168,12 +176,13 @@ func (h *Handler) OpenFile(ctx context.Context, name string, flag int, perm os.F if len(parts) == 2 { // Torrent folder level return &File{ - cache: h.cache, - torrentId: cachedTorrent.Id, - isDir: true, - children: h.getFileInfos(cachedTorrent.Torrent), - name: cachedTorrent.Name, - size: cachedTorrent.Size, + cache: h.cache, + torrentId: cachedTorrent.Id, + isDir: true, + children: h.getFileInfos(cachedTorrent.Torrent), + name: cachedTorrent.Name, + size: cachedTorrent.Size, + metadataOnly: metadataOnly, }, nil } @@ -189,6 +198,7 @@ func (h *Handler) OpenFile(ctx context.Context, name string, flag int, perm os.F size: file.Size, link: file.Link, downloadLink: file.DownloadLink, + metadataOnly: metadataOnly, } return fi, nil } @@ -207,7 +217,7 @@ func (h *Handler) Stat(ctx context.Context, name string) (os.FileInfo, error) { return f.Stat() } -func (h *Handler) getFileInfos(torrent *torrent.Torrent) []os.FileInfo { +func (h *Handler) getFileInfos(torrent *types.Torrent) []os.FileInfo { files := make([]os.FileInfo, 0, len(torrent.Files)) now := time.Now() for _, file := range torrent.Files { @@ -232,34 +242,28 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Cache PROPFIND responses for a short time to reduce load. if r.Method == "PROPFIND" { // Determine the Depth; default to "1" if not provided. + // Set metadata only + ctx := context.WithValue(r.Context(), "metadataOnly", true) + r = r.WithContext(ctx) + cleanPath := path.Clean(r.URL.Path) depth := r.Header.Get("Depth") if depth == "" { depth = "1" } // Use both path and Depth header to form the cache key. - cacheKey := fmt.Sprintf("propfind:%s:%s", r.URL.Path, depth) + cacheKey := fmt.Sprintf("propfind:%s:%s", cleanPath, depth) // Determine TTL based on the requested folder: // - If the path is exactly the parent folder (which changes frequently), // use a short TTL. // - Otherwise, for deeper (torrent folder) paths, use a longer TTL. - var ttl time.Duration + ttl := 30 * time.Minute if h.isParentPath(r.URL.Path) { - ttl = 10 * time.Second - } else { - ttl = 1 * time.Minute + ttl = 20 * time.Second } - // Check if we have a cached response that hasn't expired. - if cached, ok := h.cache.propfindResp.Load(cacheKey); ok { - if respCache, ok := cached.(propfindResponse); ok { - if time.Since(respCache.ts) < ttl { - w.Header().Set("Content-Type", "application/xml; charset=utf-8") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(respCache.data))) - w.Write(respCache.data) - return - } - } + if served := h.serveFromCacheIfValid(w, r, cacheKey, ttl); served { + return } // No valid cache entry; process the PROPFIND request. @@ -276,10 +280,22 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { handler.ServeHTTP(responseRecorder, r) responseData := responseRecorder.Body.Bytes() - // Store the new response in the cache. - h.cache.propfindResp.Store(cacheKey, propfindResponse{ - data: responseData, - ts: time.Now(), + // Create compressed version + var gzippedData []byte + if len(responseData) > 0 { + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + if _, err := gzw.Write(responseData); err == nil { + if err := gzw.Close(); err == nil { + gzippedData = buf.Bytes() + } + } + } + + h.cache.PropfindResp.Store(cacheKey, debrid.PropfindResponse{ + Data: responseData, + GzippedData: gzippedData, + Ts: time.Now(), }) // Forward the captured response to the client. @@ -332,58 +348,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Serve the file with the correct modification time. // http.ServeContent automatically handles Range requests. http.ServeContent(w, r, fileName, fi.ModTime(), rs) - - // Set headers to indicate support for range requests and content type. - //fileName := fi.Name() - //w.Header().Set("Accept-Ranges", "bytes") - //w.Header().Set("Content-Type", getContentType(fileName)) - // - //// If a Range header is provided, parse and handle partial content. - //rangeHeader := r.Header.Get("Range") - //if rangeHeader != "" { - // parts := strings.Split(strings.TrimPrefix(rangeHeader, "bytes="), "-") - // if len(parts) == 2 { - // start, startErr := strconv.ParseInt(parts[0], 10, 64) - // end := fi.Size() - 1 - // if parts[1] != "" { - // var endErr error - // end, endErr = strconv.ParseInt(parts[1], 10, 64) - // if endErr != nil { - // end = fi.Size() - 1 - // } - // } - // - // if startErr == nil && start < fi.Size() { - // if start > end { - // start, end = end, start - // } - // if end >= fi.Size() { - // end = fi.Size() - 1 - // } - // - // contentLength := end - start + 1 - // w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, fi.Size())) - // w.Header().Set("Content-Length", fmt.Sprintf("%d", contentLength)) - // w.WriteHeader(http.StatusPartialContent) - // - // // Attempt to cast to your concrete File type to call Seek. - // if file, ok := f.(*File); ok { - // _, err = file.Seek(start, io.SeekStart) - // if err != nil { - // h.logger.Error().Err(err).Msg("Failed to seek in file") - // http.Error(w, "Server Error", http.StatusInternalServerError) - // return - // } - // - // limitedReader := io.LimitReader(f, contentLength) - // h.ioCopy(limitedReader, w) - // return - // } - // } - // } - //} - //w.Header().Set("Content-Length", fmt.Sprintf("%d", fi.Size())) - //h.ioCopy(f, w) return } @@ -433,13 +397,43 @@ func (h *Handler) isParentPath(_path string) bool { rootPath := h.getRootPath() parents := h.getParentItems() for _, p := range parents { - if _path == path.Join(rootPath, p) { + if path.Clean(_path) == path.Clean(path.Join(rootPath, p)) { return true } } return false } +func (h *Handler) serveFromCacheIfValid(w http.ResponseWriter, r *http.Request, cacheKey string, ttl time.Duration) bool { + cached, ok := h.cache.PropfindResp.Load(cacheKey) + if !ok { + return false + } + + respCache, ok := cached.(debrid.PropfindResponse) + if !ok { + return false + } + + if time.Since(respCache.Ts) >= ttl { + // Remove expired cache entry + h.cache.PropfindResp.Delete(cacheKey) + return false + } + w.Header().Set("Content-Type", "application/xml; charset=utf-8") + + if acceptsGzip(r) && len(respCache.GzippedData) > 0 { + w.Header().Set("Content-Encoding", "gzip") + w.Header().Set("Vary", "Accept-Encoding") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(respCache.GzippedData))) + w.Write(respCache.GzippedData) + } else { + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(respCache.Data))) + w.Write(respCache.Data) + } + return true +} + func (h *Handler) serveDirectory(w http.ResponseWriter, r *http.Request, file webdav.File) { var children []os.FileInfo if f, ok := file.(*File); ok { diff --git a/pkg/webdav/misc.go b/pkg/webdav/misc.go index ebbaa7f..2de354d 100644 --- a/pkg/webdav/misc.go +++ b/pkg/webdav/misc.go @@ -1,6 +1,10 @@ package webdav -import "strings" +import ( + "net/http" + "net/url" + "strings" +) // getName: Returns the torrent name and filename from the path // /webdav/alldebrid/__all__/TorrentName @@ -12,3 +16,13 @@ func getName(rootDir, path string) (string, string) { } return parts[0], strings.Join(parts[1:], "/") } + +func acceptsGzip(r *http.Request) bool { + return strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") +} + +func isValidURL(str string) bool { + u, err := url.Parse(str) + // A valid URL should parse without error, and have a non-empty scheme and host. + return err == nil && u.Scheme != "" && u.Host != "" +} diff --git a/pkg/webdav/webdav.go b/pkg/webdav/webdav.go index 1d108ae..3617943 100644 --- a/pkg/webdav/webdav.go +++ b/pkg/webdav/webdav.go @@ -20,9 +20,7 @@ func New() *WebDav { w := &WebDav{ Handlers: make([]*Handler, 0), } - debrids := svc.Debrid.GetDebrids() - cacheManager := NewCacheManager(debrids) - for name, c := range cacheManager.GetCaches() { + for name, c := range svc.Debrid.Caches { h := NewHandler(name, c, logger.NewLogger(fmt.Sprintf("%s-webdav", name))) w.Handlers = append(w.Handlers, h) } diff --git a/pkg/webdav/workers.go b/pkg/webdav/workers.go deleted file mode 100644 index 376e1e9..0000000 --- a/pkg/webdav/workers.go +++ /dev/null @@ -1,69 +0,0 @@ -package webdav - -import "time" - -func (c *Cache) Refresh() error { - // For now, we just want to refresh the listing and download links - c.logger.Info().Msg("Starting cache refresh workers") - go c.refreshListingWorker() - go c.refreshDownloadLinksWorker() - go c.refreshTorrentsWorker() - return nil -} - -func (c *Cache) refreshListingWorker() { - refreshTicker := time.NewTicker(10 * time.Second) - defer refreshTicker.Stop() - - for { - select { - case <-refreshTicker.C: - if c.listingRefreshMu.TryLock() { - func() { - defer c.listingRefreshMu.Unlock() - c.refreshListings() - }() - } else { - c.logger.Debug().Msg("Refresh already in progress") - } - } - } -} - -func (c *Cache) refreshDownloadLinksWorker() { - refreshTicker := time.NewTicker(40 * time.Minute) - defer refreshTicker.Stop() - - for { - select { - case <-refreshTicker.C: - if c.downloadLinksRefreshMu.TryLock() { - func() { - defer c.downloadLinksRefreshMu.Unlock() - c.refreshDownloadLinks() - }() - } else { - c.logger.Debug().Msg("Refresh already in progress") - } - } - } -} - -func (c *Cache) refreshTorrentsWorker() { - refreshTicker := time.NewTicker(5 * time.Second) - defer refreshTicker.Stop() - - for { - select { - case <-refreshTicker.C: - if c.listingRefreshMu.TryLock() { - func() { - defer c.listingRefreshMu.Unlock() - c.refreshTorrents() - }() - } else { - c.logger.Debug().Msg("Refresh already in progress") - } - } - } -}