johno/decypharr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Add PROPFIND for root path

Browse Source 
- Reduce signifcantly memoery footprint
- Fix minor bugs
This commit is contained in:
Mukhtar Akere
2025-05-20 12:57:27 +01:00
parent 53748ea297
commit 5aa1c67544
36 changed files with 632 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/debrid/alldebrid/alldebrid.go
View File

@@ -1,8 +1,8 @@
package alldebrid
import (
"encoding/json"
"fmt"
"github.com/goccy/go-json"
"github.com/rs/zerolog"
"github.com/sirrobot01/decypharr/internal/config"
"github.com/sirrobot01/decypharr/internal/logger"

120
pkg/debrid/debrid/cache.go
View File

@@ -16,8 +16,8 @@ import (
"sync/atomic"
"time"
"encoding/json"
"github.com/go-co-op/gocron/v2"
"github.com/goccy/go-json"
"github.com/rs/zerolog"
"github.com/sirrobot01/decypharr/internal/config"
"github.com/sirrobot01/decypharr/internal/logger"
@@ -85,6 +85,9 @@ type Cache struct {
// repair
repairChan chan RepairRequest
// readiness
ready chan struct{}
// config
workers int
torrentRefreshInterval string
@@ -95,10 +98,10 @@ type Cache struct {
downloadLinksRefreshMu sync.RWMutex // for refreshing download links
torrentsRefreshMu sync.RWMutex // for refreshing torrents
scheduler gocron.Scheduler
scheduler gocron.Scheduler
cetScheduler gocron.Scheduler
saveSemaphore chan struct{}
ctx context.Context
config config.Debrid
customFolders []string
@@ -107,7 +110,8 @@ type Cache struct {
func New(dc config.Debrid, client types.Client) *Cache {
cfg := config.Get()
cet, _ := time.LoadLocation("CET")
s, _ := gocron.NewScheduler(gocron.WithLocation(cet))
cetSc, _ := gocron.NewScheduler(gocron.WithLocation(cet))
scheduler, _ := gocron.NewScheduler(gocron.WithLocation(time.Local))
autoExpiresLinksAfter, err := time.ParseDuration(dc.AutoExpireLinksAfter)
if autoExpiresLinksAfter == 0 || err != nil {
@@ -145,54 +149,111 @@ func New(dc config.Debrid, client types.Client) *Cache {
folderNaming: WebDavFolderNaming(dc.FolderNaming),
autoExpiresLinksAfterDuration: autoExpiresLinksAfter,
saveSemaphore: make(chan struct{}, 50),
ctx: context.Background(),
scheduler: s,
cetScheduler: cetSc,
scheduler: scheduler,
config: dc,
customFolders: customFolders,
ready: make(chan struct{}),
}
c.listingDebouncer = utils.NewDebouncer[bool](100*time.Millisecond, func(refreshRclone bool) {
c.RefreshListings(refreshRclone)
})
return c
}
func (c *Cache) IsReady() chan struct{} {
return c.ready
}
// Reset clears all internal state so the Cache can be reused without leaks.
// Call this after stopping the old Cache (so no goroutines are holding references),
// and before you discard the instance on a restart.
func (c *Cache) Reset() {
if err := c.scheduler.StopJobs(); err != nil {
c.logger.Error().Err(err).Msg("Failed to stop scheduler jobs")
}
if err := c.scheduler.Shutdown(); err != nil {
c.logger.Error().Err(err).Msg("Failed to stop scheduler")
}
// Stop the listing debouncer
c.listingDebouncer.Stop()
// Close the repair channel
close(c.repairChan)
// 1. Reset torrent storage
c.torrents.reset()
// 2. Reset download-link cache
c.downloadLinks.reset()
// 3. Clear any sync.Maps
c.invalidDownloadLinks = sync.Map{}
c.repairRequest = sync.Map{}
c.failedToReinsert = sync.Map{}
c.downloadLinkRequests = sync.Map{}
// 5. Rebuild the listing debouncer
c.listingDebouncer = utils.NewDebouncer[bool](
100*time.Millisecond,
func(refreshRclone bool) {
c.RefreshListings(refreshRclone)
},
)
// 6. Reset repair channel so the next Start() can spin it up
c.repairChan = make(chan RepairRequest, 100)
}
func (c *Cache) Start(ctx context.Context) error {
if err := os.MkdirAll(c.dir, 0755); err != nil {
return fmt.Errorf("failed to create cache directory: %w", err)
}
c.ctx = ctx
if err := c.Sync(); err != nil {
if err := c.Sync(ctx); err != nil {
return fmt.Errorf("failed to sync cache: %w", err)
}
// initial download links
go func() {
c.refreshDownloadLinks()
}()
go c.refreshDownloadLinks(ctx)
if err := c.StartSchedule(); err != nil {
if err := c.StartSchedule(ctx); err != nil {
c.logger.Error().Err(err).Msg("Failed to start cache worker")
}
c.repairChan = make(chan RepairRequest, 100)
go c.repairWorker()
go c.repairWorker(ctx)
// Fire the ready channel
close(c.ready)
cfg := config.Get()
name := c.client.GetName()
addr := cfg.BindAddress + ":" + cfg.Port + cfg.URLBase + "webdav/" + name + "/"
c.logger.Info().Msgf("%s WebDav server running at %s", name, addr)
<-ctx.Done()
c.logger.Info().Msgf("Stopping %s WebDav server", name)
c.Reset()
return nil
}
func (c *Cache) load() (map[string]CachedTorrent, error) {
torrents := make(map[string]CachedTorrent)
func (c *Cache) load(ctx context.Context) (map[string]CachedTorrent, error) {
mu := sync.Mutex{}
if err := os.MkdirAll(c.dir, 0755); err != nil {
return torrents, fmt.Errorf("failed to create cache directory: %w", err)
return nil, fmt.Errorf("failed to create cache directory: %w", err)
}
files, err := os.ReadDir(c.dir)
if err != nil {
return torrents, fmt.Errorf("failed to read cache directory: %w", err)
return nil, fmt.Errorf("failed to read cache directory: %w", err)
}
// Get only json files
@@ -204,7 +265,7 @@ func (c *Cache) load() (map[string]CachedTorrent, error) {
}
if len(jsonFiles) == 0 {
return torrents, nil
return nil, nil
}
// Create channels with appropriate buffering
@@ -213,6 +274,8 @@ func (c *Cache) load() (map[string]CachedTorrent, error) {
// Create a wait group for workers
var wg sync.WaitGroup
torrents := make(map[string]CachedTorrent, len(jsonFiles))
// Start workers
for i := 0; i < c.workers; i++ {
wg.Add(1)
@@ -272,7 +335,7 @@ func (c *Cache) load() (map[string]CachedTorrent, error) {
// Feed work to workers
for _, file := range jsonFiles {
select {
case <-c.ctx.Done():
case <-ctx.Done():
break // Context cancelled
default:
workChan <- file
@@ -288,13 +351,8 @@ func (c *Cache) load() (map[string]CachedTorrent, error) {
return torrents, nil
}
func (c *Cache) Sync() error {
cfg := config.Get()
name := c.client.GetName()
addr := cfg.BindAddress + ":" + cfg.Port + cfg.URLBase + "webdav/" + name + "/"
defer c.logger.Info().Msgf("%s WebDav server running at %s", name, addr)
cachedTorrents, err := c.load()
func (c *Cache) Sync(ctx context.Context) error {
cachedTorrents, err := c.load(ctx)
if err != nil {
c.logger.Error().Err(err).Msg("Failed to load cache")
}
@@ -342,7 +400,7 @@ func (c *Cache) Sync() error {
if len(newTorrents) > 0 {
c.logger.Info().Msgf("Found %d new torrents", len(newTorrents))
if err := c.sync(newTorrents); err != nil {
if err := c.sync(ctx, newTorrents); err != nil {
return fmt.Errorf("failed to sync torrents: %v", err)
}
}
@@ -350,7 +408,7 @@ func (c *Cache) Sync() error {
return nil
}
func (c *Cache) sync(torrents []*types.Torrent) error {
func (c *Cache) sync(ctx context.Context, torrents []*types.Torrent) error {
// Create channels with appropriate buffering
workChan := make(chan *types.Torrent, min(c.workers, len(torrents)))
@@ -384,7 +442,7 @@ func (c *Cache) sync(torrents []*types.Torrent) error {
c.logger.Info().Msgf("Progress: %d/%d torrents processed", count, len(torrents))
}
case <-c.ctx.Done():
case <-ctx.Done():
return // Context cancelled, exit goroutine
}
}
@@ -396,7 +454,7 @@ func (c *Cache) sync(torrents []*types.Torrent) error {
select {
case workChan <- t:
// Work sent successfully
case <-c.ctx.Done():
case <-ctx.Done():
break // Context cancelled
}
}
@@ -443,7 +501,7 @@ func (c *Cache) setTorrent(t CachedTorrent, callback func(torrent CachedTorrent)
updatedTorrent.Files = mergedFiles
}
c.torrents.set(torrentName, t, updatedTorrent)
c.SaveTorrent(updatedTorrent)
c.SaveTorrent(t)
if callback != nil {
callback(updatedTorrent)
}

10
pkg/debrid/debrid/download_link.go
View File

@@ -26,6 +26,13 @@ func newDownloadLinkCache() *downloadLinkCache {
data: make(map[string]linkCache),
}
}
func (c *downloadLinkCache) reset() {
c.mu.Lock()
c.data = make(map[string]linkCache)
c.mu.Unlock()
}
func (c *downloadLinkCache) Load(key string) (linkCache, bool) {
c.mu.Lock()
defer c.mu.Unlock()
@@ -155,14 +162,13 @@ func (c *Cache) fetchDownloadLink(torrentName, filename, fileLink string) (strin
func (c *Cache) GenerateDownloadLinks(t CachedTorrent) {
if err := c.client.GenerateDownloadLinks(t.Torrent); err != nil {
c.logger.Error().Err(err).Msg("Failed to generate download links")
c.logger.Error().Err(err).Str("torrent", t.Name).Msg("Failed to generate download links")
return
}
for _, file := range t.Files {
if file.DownloadLink != nil {
c.updateDownloadLink(file.DownloadLink)
}
}
c.setTorrent(t, nil)
}

18
pkg/debrid/debrid/engine.go
View File

@@ -7,11 +7,11 @@ import (
)
type Engine struct {
Clients map[string]types.Client
Clients map[string]types.Client
clientsMu sync.Mutex
Caches map[string]*Cache
CacheMu sync.Mutex
LastUsed string
Caches map[string]*Cache
CacheMu sync.Mutex
LastUsed string
}
func NewEngine() *Engine {
@@ -46,6 +46,16 @@ func (d *Engine) GetClient(name string) types.Client {
return d.Clients[name]
}
func (d *Engine) Reset() {
d.clientsMu.Lock()
d.Clients = make(map[string]types.Client)
d.clientsMu.Unlock()
d.CacheMu.Lock()
d.Caches = make(map[string]*Cache)
d.CacheMu.Unlock()
}
func (d *Engine) GetDebrids() map[string]types.Client {
return d.Clients
}

51
pkg/debrid/debrid/refresh.go
View File

@@ -1,6 +1,7 @@
package debrid
import (
"context"
"fmt"
"github.com/sirrobot01/decypharr/pkg/debrid/types"
"io"
@@ -34,25 +35,23 @@ func (c *Cache) RefreshListings(refreshRclone bool) {
if refreshRclone {
if err := c.refreshRclone(); err != nil {
c.logger.Trace().Err(err).Msg("Failed to refresh rclone") // silent error
c.logger.Error().Err(err).Msg("Failed to refresh rclone") // silent error
}
}
}
func (c *Cache) refreshTorrents() {
// Use a mutex to prevent concurrent refreshes
if c.torrentsRefreshMu.TryLock() {
defer c.torrentsRefreshMu.Unlock()
} else {
return
}
func (c *Cache) refreshTorrents(ctx context.Context) {
select {
case <-c.ctx.Done():
case <-ctx.Done():
return
default:
}
if !c.torrentsRefreshMu.TryLock() {
return
}
defer c.torrentsRefreshMu.Unlock()
// Get all torrents from the debrid service
debTorrents, err := c.client.GetTorrents()
if err != nil {
@@ -72,7 +71,8 @@ func (c *Cache) refreshTorrents() {
// Let's implement deleting torrents removed from debrid
deletedTorrents := make([]string, 0)
for _, id := range c.torrents.getAllIDs() {
cachedTorrents := c.torrents.getIdMaps()
for id := range cachedTorrents {
if _, exists := currentTorrentIds[id]; !exists {
deletedTorrents = append(deletedTorrents, id)
}
@@ -83,9 +83,8 @@ func (c *Cache) refreshTorrents() {
}
newTorrents := make([]*types.Torrent, 0)
cachedIdsMaps := c.torrents.getIdMaps()
for _, t := range debTorrents {
if _, exists := cachedIdsMaps[t.Id]; !exists {
if _, exists := cachedTorrents[t.Id]; !exists {
newTorrents = append(newTorrents, t)
}
}
@@ -129,12 +128,6 @@ func (c *Cache) refreshTorrents() {
func (c *Cache) refreshRclone() error {
cfg := c.config
select {
case <-c.ctx.Done():
return nil
default:
}
if cfg.RcUrl == "" {
return nil
}
@@ -214,12 +207,6 @@ func (c *Cache) refreshTorrent(torrentId string) *CachedTorrent {
return nil
}
select {
case <-c.ctx.Done():
return nil
default:
}
torrent, err := c.client.GetTorrent(torrentId)
if err != nil {
c.logger.Error().Err(err).Msgf("Failed to get torrent %s", torrentId)
@@ -241,19 +228,19 @@ func (c *Cache) refreshTorrent(torrentId string) *CachedTorrent {
return &ct
}
func (c *Cache) refreshDownloadLinks() {
if c.downloadLinksRefreshMu.TryLock() {
defer c.downloadLinksRefreshMu.Unlock()
} else {
return
}
func (c *Cache) refreshDownloadLinks(ctx context.Context) {
select {
case <-c.ctx.Done():
case <-ctx.Done():
return
default:
}
if !c.downloadLinksRefreshMu.TryLock() {
return
}
defer c.downloadLinksRefreshMu.Unlock()
downloadLinks, err := c.client.GetDownloads()
if err != nil {

6
pkg/debrid/debrid/repair.go
View File

@@ -1,6 +1,7 @@
package debrid
import (
"context"
"errors"
"fmt"
"github.com/sirrobot01/decypharr/internal/request"
@@ -122,12 +123,11 @@ func (c *Cache) IsTorrentBroken(t *CachedTorrent, filenames []string) bool {
return isBroken
}
func (c *Cache) repairWorker() {
func (c *Cache) repairWorker(ctx context.Context) {
// This watches a channel for torrents to repair and can be cancelled via context
for {
select {
case <-c.ctx.Done():
// Context was cancelled, exit the goroutine
case <-ctx.Done():
return
case req, ok := <-c.repairChan:

28
pkg/debrid/debrid/torrent.go
View File

@@ -73,6 +73,22 @@ func newTorrentCache(dirFilters map[string][]directoryFilter) *torrentCache {
return tc
}
func (tc *torrentCache) reset() {
tc.mu.Lock()
tc.byID = make(map[string]CachedTorrent)
tc.byName = make(map[string]CachedTorrent)
tc.mu.Unlock()
// reset the sorted listing
tc.sortNeeded.Store(false)
tc.listing.Store(make([]os.FileInfo, 0))
// reset any per-folder views
tc.folderListingMu.Lock()
tc.folderListing = make(map[string][]os.FileInfo)
tc.folderListingMu.Unlock()
}
func (tc *torrentCache) getByID(id string) (CachedTorrent, bool) {
tc.mu.Lock()
defer tc.mu.Unlock()
@@ -251,22 +267,12 @@ func (tc *torrentCache) getAll() map[string]CachedTorrent {
tc.mu.Lock()
defer tc.mu.Unlock()
result := make(map[string]CachedTorrent)
for name, torrent := range tc.byName {
for name, torrent := range tc.byID {
result[name] = torrent
}
return result
}
func (tc *torrentCache) getAllIDs() []string {
tc.mu.Lock()
defer tc.mu.Unlock()
ids := make([]string, 0, len(tc.byID))
for id := range tc.byID {
ids = append(ids, id)
}
return ids
}
func (tc *torrentCache) getIdMaps() map[string]struct{} {
tc.mu.Lock()
defer tc.mu.Unlock()

50
pkg/debrid/debrid/worker.go
View File

@@ -1,32 +1,60 @@
package debrid
import (
"context"
"github.com/go-co-op/gocron/v2"
"github.com/sirrobot01/decypharr/internal/utils"
"time"
)
func (c *Cache) StartSchedule() error {
func (c *Cache) StartSchedule(ctx context.Context) error {
// For now, we just want to refresh the listing and download links
if _, err := utils.ScheduleJob(c.ctx, c.downloadLinksRefreshInterval, nil, c.refreshDownloadLinks); err != nil {
c.logger.Error().Err(err).Msg("Failed to add download link refresh job")
// Schedule download link refresh job
if jd, err := utils.ConvertToJobDef(c.downloadLinksRefreshInterval); err != nil {
c.logger.Error().Err(err).Msg("Failed to convert download link refresh interval to job definition")
} else {
c.logger.Debug().Msgf("Download link refresh job scheduled for every %s", c.downloadLinksRefreshInterval)
// Schedule the job
if _, err := c.scheduler.NewJob(jd, gocron.NewTask(func() {
c.refreshDownloadLinks(ctx)
}), gocron.WithContext(ctx)); err != nil {
c.logger.Error().Err(err).Msg("Failed to create download link refresh job")
} else {
c.logger.Debug().Msgf("Download link refresh job scheduled for every %s", c.downloadLinksRefreshInterval)
}
}
if _, err := utils.ScheduleJob(c.ctx, c.torrentRefreshInterval, nil, c.refreshTorrents); err != nil {
c.logger.Error().Err(err).Msg("Failed to add torrent refresh job")
// Schedule torrent refresh job
if jd, err := utils.ConvertToJobDef(c.torrentRefreshInterval); err != nil {
c.logger.Error().Err(err).Msg("Failed to convert torrent refresh interval to job definition")
} else {
c.logger.Debug().Msgf("Torrent refresh job scheduled for every %s", c.torrentRefreshInterval)
// Schedule the job
if _, err := c.scheduler.NewJob(jd, gocron.NewTask(func() {
c.refreshTorrents(ctx)
}), gocron.WithContext(ctx)); err != nil {
c.logger.Error().Err(err).Msg("Failed to create torrent refresh job")
} else {
c.logger.Debug().Msgf("Torrent refresh job scheduled for every %s", c.torrentRefreshInterval)
}
}
// Schedule the reset invalid links job
// This job will run every at 00:00 CET
// and reset the invalid links in the cache
cet, _ := time.LoadLocation("CET")
if _, err := utils.ScheduleJob(c.ctx, "00:00", cet, c.resetInvalidLinks); err != nil {
c.logger.Error().Err(err).Msg("Failed to add reset invalid links job")
if jd, err := utils.ConvertToJobDef("00:00"); err != nil {
c.logger.Error().Err(err).Msg("Failed to convert link reset interval to job definition")
} else {
// Schedule the job
if _, err := c.cetScheduler.NewJob(jd, gocron.NewTask(func() {
c.resetInvalidLinks()
}), gocron.WithContext(ctx)); err != nil {
c.logger.Error().Err(err).Msg("Failed to create link reset job")
} else {
c.logger.Debug().Msgf("Link reset job scheduled for every midnight, CET")
}
}
// Start the scheduler
c.scheduler.Start()
c.cetScheduler.Start()
return nil
}

2
pkg/debrid/debrid_link/debrid_link.go
View File

@@ -2,8 +2,8 @@ package debrid_link
import (
"bytes"
"encoding/json"
"fmt"
"github.com/goccy/go-json"
"github.com/rs/zerolog"
"github.com/sirrobot01/decypharr/internal/config"
"github.com/sirrobot01/decypharr/internal/logger"

2
pkg/debrid/realdebrid/realdebrid.go
View File

@@ -2,9 +2,9 @@ package realdebrid
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"github.com/goccy/go-json"
"github.com/rs/zerolog"
"github.com/sirrobot01/decypharr/internal/config"
"github.com/sirrobot01/decypharr/internal/logger"

2
pkg/debrid/realdebrid/types.go
View File

@@ -1,8 +1,8 @@
package realdebrid
import (
"encoding/json"
"fmt"
"github.com/goccy/go-json"
"time"
)

2
pkg/debrid/torbox/torbox.go
View File

@@ -2,8 +2,8 @@ package torbox
import (
"bytes"
"encoding/json"
"fmt"
"github.com/goccy/go-json"
"github.com/rs/zerolog"
"github.com/sirrobot01/decypharr/internal/config"
"github.com/sirrobot01/decypharr/internal/logger"