Improve webdav; add workers for refreshes

internal/logger/logger.go

@@ -29,7 +29,9 @@ func GetLogPath() string {
 	return filepath.Join(logsDir, "decypharr.log")
 }
 
-func NewLogger(prefix string, level string) zerolog.Logger {
+func NewLogger(prefix string) zerolog.Logger {
+
+	level := config.GetConfig().LogLevel
 
 	rotatingLogFile := &lumberjack.Logger{
 		Filename: GetLogPath(),
@@ -86,8 +88,7 @@ func NewLogger(prefix string, level string) zerolog.Logger {
 
 func GetDefaultLogger() zerolog.Logger {
 	once.Do(func() {
-		cfg := config.GetConfig()
+		logger = NewLogger("decypharr")
-		logger = NewLogger("decypharr", cfg.LogLevel)
 	})
 	return logger
 }

internal/request/request.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"github.com/goccy/go-json"
 	"github.com/rs/zerolog"
-	"github.com/sirrobot01/debrid-blackhole/internal/config"
 	"github.com/sirrobot01/debrid-blackhole/internal/logger"
 	"golang.org/x/time/rate"
 	"io"
@@ -227,7 +226,7 @@ func New(options ...ClientOption) *Client {
 			http.StatusServiceUnavailable:  true,
 			http.StatusGatewayTimeout:      true,
 		},
-		logger: logger.NewLogger("request", config.GetConfig().LogLevel),
+		logger: logger.NewLogger("request"),
 	}
 
 	// Apply options

main.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"github.com/sirrobot01/debrid-blackhole/cmd/decypharr"
 	"github.com/sirrobot01/debrid-blackhole/internal/config"
+	"github.com/sirrobot01/debrid-blackhole/pkg/version"
 	"log"
 	"net/http"
 	_ "net/http/pprof" // registers pprof handlers
@@ -19,11 +20,14 @@ func main() {
 		}
 	}()
 
-	go func() {
+	if version.GetInfo().Channel == "dev" {
-		if err := http.ListenAndServe(":6060", nil); err != nil {
+		log.Println("Running in dev mode")
-			log.Fatalf("pprof server failed: %v", err)
+		go func() {
-		}
+			if err := http.ListenAndServe(":6060", nil); err != nil {
-	}()
+				log.Fatalf("pprof server failed: %v", err)
+			}
+		}()
+	}
 	var configPath string
 	flag.StringVar(&configPath, "config", "/data", "path to the data folder")
 	flag.Parse()

pkg/debrid/alldebrid/alldebrid.go

@@ -288,7 +288,7 @@ func New(dc config.Debrid) *AllDebrid {
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 	}
-	_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
+	_log := logger.NewLogger(dc.Name)
 	client := request.New().
 		WithHeaders(headers).
 		WithRateLimiter(rl).WithLogger(_log)
@@ -299,7 +299,7 @@ func New(dc config.Debrid) *AllDebrid {
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
 		MountPath:        dc.Folder,
-		logger:           logger.NewLogger(dc.Name, config.GetConfig().LogLevel),
+		logger:           logger.NewLogger(dc.Name),
 		CheckCached:      dc.CheckCached,
 	}
 }

pkg/debrid/debrid_link/debrid_link.go

@@ -264,7 +264,7 @@ func New(dc config.Debrid) *DebridLink {
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 		"Content-Type":  "application/json",
 	}
-	_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
+	_log := logger.NewLogger(dc.Name)
 	client := request.New().
 		WithHeaders(headers).
 		WithRateLimiter(rl).WithLogger(_log)
@@ -275,7 +275,7 @@ func New(dc config.Debrid) *DebridLink {
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
 		MountPath:        dc.Folder,
-		logger:           logger.NewLogger(dc.Name, config.GetConfig().LogLevel),
+		logger:           logger.NewLogger(dc.Name),
 		CheckCached:      dc.CheckCached,
 	}
 }

pkg/debrid/realdebrid/realdebrid.go

@@ -517,7 +517,7 @@ func New(dc config.Debrid) *RealDebrid {
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 	}
-	_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
+	_log := logger.NewLogger(dc.Name)
 	client := request.New().
 		WithHeaders(headers).
 		WithRateLimiter(rl).WithLogger(_log)
@@ -528,7 +528,7 @@ func New(dc config.Debrid) *RealDebrid {
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
 		MountPath:        dc.Folder,
-		logger:           logger.NewLogger(dc.Name, config.GetConfig().LogLevel),
+		logger:           logger.NewLogger(dc.Name),
 		CheckCached:      dc.CheckCached,
 	}
 }

pkg/debrid/torbox/torbox.go

@@ -319,7 +319,7 @@ func New(dc config.Debrid) *Torbox {
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
 	}
-	_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
+	_log := logger.NewLogger(dc.Name)
 	client := request.New().
 		WithHeaders(headers).
 		WithRateLimiter(rl).WithLogger(_log)

pkg/proxy/proxy.go

@@ -88,7 +88,7 @@ func NewProxy() *Proxy {
 		username:   cfg.Username,
 		password:   cfg.Password,
 		cachedOnly: cfg.CachedOnly,
-		logger:     logger.NewLogger("proxy", cfg.LogLevel),
+		logger:     logger.NewLogger("proxy"),
 	}
 }
 

pkg/qbit/qbit.go

@@ -34,7 +34,7 @@ func New() *QBit {
 		DownloadFolder:  cfg.DownloadFolder,
 		Categories:      cfg.Categories,
 		Storage:         NewTorrentStorage(filepath.Join(_cfg.Path, "torrents.json")),
-		logger:          logger.NewLogger("qbit", _cfg.LogLevel),
+		logger:          logger.NewLogger("qbit"),
 		RefreshInterval: refreshInterval,
 		SkipPreCache:    cfg.SkipPreCache,
 	}

pkg/repair/repair.go

@@ -47,7 +47,7 @@ func New(arrs *arr.Storage) *Repair {
 	}
 	r := &Repair{
 		arrs:        arrs,
-		logger:      logger.NewLogger("repair", cfg.LogLevel),
+		logger:      logger.NewLogger("repair"),
 		duration:    duration,
 		runOnStart:  cfg.Repair.RunOnStart,
 		ZurgURL:     cfg.Repair.ZurgURL,

pkg/server/server.go

@@ -22,8 +22,7 @@ type Server struct {
 }
 
 func New() *Server {
-	cfg := config.GetConfig()
+	l := logger.NewLogger("http")
-	l := logger.NewLogger("http", cfg.LogLevel)
 	r := chi.NewRouter()
 	r.Use(middleware.Recoverer)
 	r.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

pkg/web/server.go

@@ -60,10 +60,9 @@ type Handler struct {
 }
 
 func New(qbit *qbit.QBit) *Handler {
-	cfg := config.GetConfig()
 	return &Handler{
 		qbit:   qbit,
-		logger: logger.NewLogger("ui", cfg.LogLevel),
+		logger: logger.NewLogger("ui"),
 	}
 }
 

pkg/webdav/cache.go

@@ -7,10 +7,12 @@ import (
 	"github.com/dgraph-io/badger/v4"
 	"github.com/goccy/go-json"
 	"github.com/rs/zerolog"
+	"github.com/sirrobot01/debrid-blackhole/internal/logger"
 	"github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid"
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -23,6 +25,11 @@ type DownloadLinkCache struct {
 	Link string `json:"download_link"`
 }
 
+type propfindResponse struct {
+	data []byte
+	ts   time.Time
+}
+
 type CachedTorrent struct {
 	*torrent.Torrent
 	LastRead   time.Time `json:"last_read"`
@@ -39,27 +46,29 @@ type Cache struct {
 	torrentsNames map[string]*CachedTorrent // key: torrent.Name, value: torrent
 	listings      atomic.Value
 	downloadLinks map[string]string // key: file.Link, value: download link
+	propfindResp  sync.Map
 
 	workers int
 
 	LastUpdated time.Time `json:"last_updated"`
 
 	// refresh mutex
-	torrentsRefreshMutex      sync.Mutex // for refreshing torrents
+	listingRefreshMu       sync.Mutex // for refreshing torrents
-	downloadLinksRefreshMutex sync.Mutex // for refreshing download links
+	downloadLinksRefreshMu sync.Mutex // for refreshing download links
+	torrentsRefreshMu      sync.Mutex // for refreshing torrents
 
-	// Mutexes
+	// Data Mutexes
 	torrentsMutex      sync.RWMutex // for torrents and torrentsNames
-	downloadLinksMutex sync.Mutex
+	downloadLinksMutex sync.Mutex   // for downloadLinks
 }
 
 func (c *Cache) setTorrent(t *CachedTorrent) {
 	c.torrentsMutex.Lock()
-	defer c.torrentsMutex.Unlock()
 	c.torrents[t.Id] = t
 	c.torrentsNames[t.Name] = t
+	c.torrentsMutex.Unlock()
 
-	c.refreshListings()
+	go c.refreshListings() // This is concurrent safe
 
 	go func() {
 		if err := c.SaveTorrent(t); err != nil {
@@ -69,19 +78,31 @@ func (c *Cache) setTorrent(t *CachedTorrent) {
 }
 
 func (c *Cache) refreshListings() {
-	files := make([]os.FileInfo, 0, len(c.torrents))
+	// Copy the current torrents to avoid concurrent issues
-	now := time.Now()
+	c.torrentsMutex.RLock()
+	torrents := make([]string, 0, len(c.torrents))
 	for _, t := range c.torrents {
 		if t != nil && t.Torrent != nil {
-			files = append(files, &FileInfo{
+			torrents = append(torrents, t.Name)
-				name:    t.Name,
-				size:    0,
-				mode:    0755 | os.ModeDir,
-				modTime: now,
-				isDir:   true,
-			})
 		}
 	}
+	c.torrentsMutex.RUnlock()
+
+	sort.Slice(torrents, func(i, j int) bool {
+		return torrents[i] < torrents[j]
+	})
+
+	files := make([]os.FileInfo, 0, len(torrents))
+	now := time.Now()
+	for _, t := range torrents {
+		files = append(files, &FileInfo{
+			name:    t,
+			size:    0,
+			mode:    0755 | os.ModeDir,
+			modTime: now,
+			isDir:   true,
+		})
+	}
 	// Atomic store of the complete ready-to-use slice
 	c.listings.Store(files)
 }
@@ -90,15 +111,16 @@ func (c *Cache) GetListing() []os.FileInfo {
 	return c.listings.Load().([]os.FileInfo)
 }
 
-func (c *Cache) setTorrents(torrents []*CachedTorrent) {
+func (c *Cache) setTorrents(torrents map[string]*CachedTorrent) {
 	c.torrentsMutex.Lock()
-	defer c.torrentsMutex.Unlock()
 	for _, t := range torrents {
 		c.torrents[t.Id] = t
 		c.torrentsNames[t.Name] = t
 	}
 
-	go c.refreshListings()
+	c.torrentsMutex.Unlock()
+
+	go c.refreshListings() // This is concurrent safe
 
 	go func() {
 		if err := c.SaveTorrents(); err != nil {
@@ -148,13 +170,14 @@ func (m *Manager) GetCache(debridName string) *Cache {
 }
 
 func NewCache(client debrid.Client) *Cache {
-	dbPath := filepath.Join(config.GetConfig().Path, "cache", client.GetName())
+	cfg := config.GetConfig()
+	dbPath := filepath.Join(cfg.Path, "cache", client.GetName())
 	return &Cache{
 		dir:           dbPath,
 		torrents:      make(map[string]*CachedTorrent),
 		torrentsNames: make(map[string]*CachedTorrent),
 		client:        client,
-		logger:        client.GetLogger(),
+		logger:        logger.NewLogger(fmt.Sprintf("%s-cache", client.GetName())),
 		workers:       200,
 		downloadLinks: make(map[string]string),
 	}
@@ -172,8 +195,8 @@ func (c *Cache) Start() error {
 	// initial download links
 	go func() {
 		// lock download refresh mutex
-		c.downloadLinksRefreshMutex.Lock()
+		c.downloadLinksRefreshMu.Lock()
-		defer c.downloadLinksRefreshMutex.Unlock()
+		defer c.downloadLinksRefreshMu.Unlock()
 		// This prevents the download links from being refreshed twice
 		c.refreshDownloadLinks()
 	}()
@@ -195,8 +218,8 @@ func (c *Cache) Close() error {
 	return nil
 }
 
-func (c *Cache) load() ([]*CachedTorrent, error) {
+func (c *Cache) load() (map[string]*CachedTorrent, error) {
-	torrents := make([]*CachedTorrent, 0)
+	torrents := make(map[string]*CachedTorrent)
 	if err := os.MkdirAll(c.dir, 0755); err != nil {
 		return torrents, fmt.Errorf("failed to create cache directory: %w", err)
 	}
@@ -225,7 +248,8 @@ func (c *Cache) load() ([]*CachedTorrent, error) {
 		}
 		if len(ct.Files) != 0 {
 			// We can assume the torrent is complete
-			torrents = append(torrents, &ct)
+			ct.IsComplete = true
+			torrents[ct.Id] = &ct
 		}
 	}
 
@@ -290,27 +314,48 @@ func (c *Cache) Sync() error {
 	if err != nil {
 		c.logger.Debug().Err(err).Msg("Failed to load cache")
 	}
-	// Write these torrents to the cache
-	c.setTorrents(cachedTorrents)
-	c.logger.Info().Msgf("Loaded %d torrents from cache", len(cachedTorrents))
 
 	torrents, err := c.client.GetTorrents()
-
-	c.logger.Info().Msgf("Got %d torrents from %s", len(torrents), c.client.GetName())
 	if err != nil {
 		return fmt.Errorf("failed to sync torrents: %v", err)
 	}
 
-	mewTorrents := make([]*torrent.Torrent, 0)
+	c.logger.Info().Msgf("Got %d torrents from %s", len(torrents), c.client.GetName())
+
+	newTorrents := make([]*torrent.Torrent, 0)
+	idStore := make(map[string]bool, len(torrents))
 	for _, t := range torrents {
-		if _, ok := c.torrents[t.Id]; !ok {
+		idStore[t.Id] = true
-			mewTorrents = append(mewTorrents, t)
+		if _, ok := cachedTorrents[t.Id]; !ok {
+			newTorrents = append(newTorrents, t)
 		}
 	}
-	c.logger.Info().Msgf("Found %d new torrents", len(mewTorrents))
 
-	if len(mewTorrents) > 0 {
+	// Check for deleted torrents
-		if err := c.sync(mewTorrents); err != nil {
+	deletedTorrents := make([]string, 0)
+	for _, t := range cachedTorrents {
+		if _, ok := idStore[t.Id]; !ok {
+			deletedTorrents = append(deletedTorrents, t.Id)
+		}
+	}
+
+	if len(deletedTorrents) > 0 {
+		c.logger.Info().Msgf("Found %d deleted torrents", len(deletedTorrents))
+		for _, id := range deletedTorrents {
+			if _, ok := cachedTorrents[id]; ok {
+				delete(cachedTorrents, id)
+				c.removeFromDB(id)
+			}
+		}
+	}
+
+	// Write these torrents to the cache
+	c.setTorrents(cachedTorrents)
+	c.logger.Info().Msgf("Loaded %d torrents from cache", len(cachedTorrents))
+
+	if len(newTorrents) > 0 {
+		c.logger.Info().Msgf("Found %d new torrents", len(newTorrents))
+		if err := c.sync(newTorrents); err != nil {
 			return fmt.Errorf("failed to sync torrents: %v", err)
 		}
 	}
@@ -474,46 +519,6 @@ func (c *Cache) refreshTorrent(t *CachedTorrent) *CachedTorrent {
 	return ct
 }
 
-func (c *Cache) refreshListingWorker() {
-	c.logger.Info().Msg("WebDAV Background Refresh Worker started")
-	refreshTicker := time.NewTicker(10 * time.Second)
-	defer refreshTicker.Stop()
-
-	for {
-		select {
-		case <-refreshTicker.C:
-			if c.torrentsRefreshMutex.TryLock() {
-				func() {
-					defer c.torrentsRefreshMutex.Unlock()
-					c.refreshListings()
-				}()
-			} else {
-				c.logger.Debug().Msg("Refresh already in progress")
-			}
-		}
-	}
-}
-
-func (c *Cache) refreshDownloadLinksWorker() {
-	c.logger.Info().Msg("WebDAV Background Refresh Download Worker started")
-	refreshTicker := time.NewTicker(40 * time.Minute)
-	defer refreshTicker.Stop()
-
-	for {
-		select {
-		case <-refreshTicker.C:
-			if c.downloadLinksRefreshMutex.TryLock() {
-				func() {
-					defer c.downloadLinksRefreshMutex.Unlock()
-					c.refreshDownloadLinks()
-				}()
-			} else {
-				c.logger.Debug().Msg("Refresh already in progress")
-			}
-		}
-	}
-}
-
 func (c *Cache) refreshDownloadLinks() map[string]string {
 	c.downloadLinksMutex.Lock()
 	defer c.downloadLinksMutex.Unlock()
@@ -526,7 +531,6 @@ func (c *Cache) refreshDownloadLinks() map[string]string {
 	for k, v := range downloadLinks {
 		c.downloadLinks[k] = v.DownloadLink
 	}
-	c.logger.Info().Msgf("Refreshed %d download links", len(downloadLinks))
 	return c.downloadLinks
 }
 
@@ -534,9 +538,110 @@ func (c *Cache) GetClient() debrid.Client {
 	return c.client
 }
 
-func (c *Cache) Refresh() error {
+func (c *Cache) refreshTorrents() {
-	// For now, we just want to refresh the listing
+	c.torrentsMutex.RLock()
-	go c.refreshListingWorker()
+	currentTorrents := c.torrents //
-	go c.refreshDownloadLinksWorker()
+	// Create a copy of the current torrents to avoid concurrent issues
-	return nil
+	torrents := make(map[string]string, len(currentTorrents)) // a mpa of id and name
+	for _, v := range currentTorrents {
+		torrents[v.Id] = v.Name
+	}
+	c.torrentsMutex.RUnlock()
+
+	// Get new torrents from the debrid service
+	debTorrents, err := c.client.GetTorrents()
+	if err != nil {
+		c.logger.Debug().Err(err).Msg("Failed to get torrents")
+		return
+	}
+
+	if len(debTorrents) == 0 {
+		// Maybe an error occurred
+		return
+	}
+
+	// Get the newly added torrents only
+	newTorrents := make([]*torrent.Torrent, 0)
+	idStore := make(map[string]bool, len(debTorrents))
+	for _, t := range debTorrents {
+		idStore[t.Id] = true
+		if _, ok := torrents[t.Id]; !ok {
+			newTorrents = append(newTorrents, t)
+		}
+	}
+
+	// Check for deleted torrents
+	deletedTorrents := make([]string, 0)
+	for id, _ := range torrents {
+		if _, ok := idStore[id]; !ok {
+			deletedTorrents = append(deletedTorrents, id)
+		}
+	}
+
+	if len(deletedTorrents) > 0 {
+		c.DeleteTorrent(deletedTorrents)
+	}
+
+	if len(newTorrents) == 0 {
+		return
+	}
+	c.logger.Info().Msgf("Found %d new torrents", len(newTorrents))
+
+	// No need for a complex sync process, just add the new torrents
+	wg := sync.WaitGroup{}
+	wg.Add(len(newTorrents))
+	for _, t := range newTorrents {
+		// processTorrent is concurrent safe
+		go func() {
+			defer wg.Done()
+			if err := c.processTorrent(t); err != nil {
+				c.logger.Info().Err(err).Msg("Failed to process torrent")
+			}
+
+		}()
+	}
+	wg.Wait()
+}
+
+func (c *Cache) DeleteTorrent(ids []string) {
+	c.logger.Info().Msgf("Deleting %d torrents", len(ids))
+	c.torrentsMutex.Lock()
+	defer c.torrentsMutex.Unlock()
+	for _, id := range ids {
+		if t, ok := c.torrents[id]; ok {
+			delete(c.torrents, id)
+			delete(c.torrentsNames, t.Name)
+			c.removeFromDB(id)
+		}
+	}
+}
+
+func (c *Cache) removeFromDB(torrentId string) {
+	filePath := filepath.Join(c.dir, torrentId+".json")
+	if err := os.Remove(filePath); err != nil {
+		c.logger.Debug().Err(err).Msgf("Failed to remove file: %s", filePath)
+	}
+}
+
+func (c *Cache) resetPropfindResponse() {
+	// Right now, parents are hardcoded
+	parents := []string{"__all__", "torrents"}
+	// Reset only the parent directories
+	// Convert the parents to a keys
+	// This is a bit hacky, but it works
+	// Instead of deleting all the keys, we only delete the parent keys, e.g __all__/ or torrents/
+	keys := make([]string, 0, len(parents))
+	for _, p := range parents {
+		// Construct the key
+		// construct url
+		url := filepath.Join("/webdav/%s/%s", c.client.GetName(), p)
+		key0 := fmt.Sprintf("propfind:%s:0", url)
+		key1 := fmt.Sprintf("propfind:%s:1", url)
+		keys = append(keys, key0, key1)
+	}
+
+	// Delete the keys
+	for _, k := range keys {
+		c.propfindResp.Delete(k)
+	}
 }

pkg/webdav/file.go

@@ -138,7 +138,7 @@ func (f *File) Seek(offset int64, whence int) (int64, error) {
 	case io.SeekCurrent:
 		newOffset = f.offset + offset
 	case io.SeekEnd:
-		newOffset = f.size - offset
+		newOffset = f.size + offset
 	default:
 		return 0, os.ErrInvalid
 	}

pkg/webdav/handler.go

@@ -15,6 +15,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"path"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -52,7 +53,7 @@ func (h *Handler) Mkdir(ctx context.Context, name string, perm os.FileMode) erro
 func (h *Handler) RemoveAll(ctx context.Context, name string) error {
 	name = path.Clean("/" + name)
 
-	rootDir := h.getParentRootPath()
+	rootDir := h.getRootPath()
 
 	if name == rootDir {
 		return os.ErrPermission
@@ -67,6 +68,8 @@ func (h *Handler) RemoveAll(ctx context.Context, name string) error {
 	if filename == "" {
 		h.cache.GetClient().DeleteTorrent(cachedTorrent.Torrent)
 		go h.cache.refreshListings()
+		go h.cache.refreshTorrents()
+		go h.cache.resetPropfindResponse()
 		return nil
 	}
 
@@ -78,7 +81,7 @@ func (h *Handler) Rename(ctx context.Context, oldName, newName string) error {
 	return os.ErrPermission // Read-only filesystem
 }
 
-func (h *Handler) getParentRootPath() string {
+func (h *Handler) getRootPath() string {
 	return fmt.Sprintf("/webdav/%s", h.Name)
 }
 
@@ -86,37 +89,33 @@ func (h *Handler) getTorrentsFolders() []os.FileInfo {
 	return h.cache.GetListing()
 }
 
+func (h *Handler) getParentItems() []string {
+	return []string{"__all__", "torrents", "version.txt"}
+}
+
 func (h *Handler) getParentFiles() []os.FileInfo {
 	now := time.Now()
-	rootFiles := []os.FileInfo{
+	rootFiles := make([]os.FileInfo, 0, len(h.getParentItems()))
-		&FileInfo{
+	for _, item := range h.getParentItems() {
-			name:    "__all__",
+		f := &FileInfo{
+			name:    item,
 			size:    0,
 			mode:    0755 | os.ModeDir,
 			modTime: now,
 			isDir:   true,
-		},
+		}
-		&FileInfo{
+		if item == "version.txt" {
-			name:    "torrents",
+			f.isDir = false
-			size:    0,
+			f.size = int64(len("v1.0.0"))
-			mode:    0755 | os.ModeDir,
+		}
-			modTime: now,
+		rootFiles = append(rootFiles, f)
-			isDir:   true,
-		},
-		&FileInfo{
-			name:    "version.txt",
-			size:    int64(len("v1.0.0")),
-			mode:    0644,
-			modTime: now,
-			isDir:   false,
-		},
 	}
 	return rootFiles
 }
 
 func (h *Handler) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
 	name = path.Clean("/" + name)
-	rootDir := h.getParentRootPath()
+	rootDir := h.getRootPath()
 
 	// Fast path optimization with a map lookup instead of string comparisons
 	switch name {
@@ -138,7 +137,7 @@ func (h *Handler) OpenFile(ctx context.Context, name string, flag int, perm os.F
 	}
 
 	// Single check for top-level folders
-	if name == path.Join(rootDir, "__all__") || name == path.Join(rootDir, "torrents") {
+	if h.isParentPath(name) {
 		folderName := strings.TrimPrefix(name, rootDir)
 		folderName = strings.TrimPrefix(folderName, "/")
 
@@ -157,7 +156,7 @@ func (h *Handler) OpenFile(ctx context.Context, name string, flag int, perm os.F
 	_path := strings.TrimPrefix(name, rootDir)
 	parts := strings.Split(strings.TrimPrefix(_path, "/"), "/")
 
-	if len(parts) >= 2 && (parts[0] == "__all__" || parts[0] == "torrents") {
+	if len(parts) >= 2 && (slices.Contains(h.getParentItems(), parts[0])) {
 
 		torrentName := parts[1]
 		cachedTorrent := h.cache.GetTorrentByName(torrentName)
@@ -224,71 +223,76 @@ func (h *Handler) getFileInfos(torrent *torrent.Torrent) []os.FileInfo {
 }
 
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-
 	// Handle OPTIONS
 	if r.Method == "OPTIONS" {
 		w.WriteHeader(http.StatusOK)
 		return
 	}
 
-	//Add specific PROPFIND optimization
+	// Cache PROPFIND responses for a short time to reduce load.
 	if r.Method == "PROPFIND" {
-		propfindStart := time.Now()
+		// Determine the Depth; default to "1" if not provided.
+		depth := r.Header.Get("Depth")
+		if depth == "" {
+			depth = "1"
+		}
+		// Use both path and Depth header to form the cache key.
+		cacheKey := fmt.Sprintf("propfind:%s:%s", r.URL.Path, depth)
 
-		// Check if this is the slow path we identified
+		// Determine TTL based on the requested folder:
-		if strings.Contains(r.URL.Path, "__all__") {
+		// - If the path is exactly the parent folder (which changes frequently),
-			// Fast path for this specific directory
+		//   use a short TTL.
-			depth := r.Header.Get("Depth")
+		// - Otherwise, for deeper (torrent folder) paths, use a longer TTL.
-			if depth == "1" || depth == "" {
+		var ttl time.Duration
-				// This is a listing request
+		if h.isParentPath(r.URL.Path) {
+			ttl = 10 * time.Second
+		} else {
+			ttl = 1 * time.Minute
+		}
 
-				// Use a cached response if available
+		// Check if we have a cached response that hasn't expired.
-				cachedKey := "propfind_" + r.URL.Path
+		if cached, ok := h.cache.propfindResp.Load(cacheKey); ok {
-				if cachedResponse, ok := h.responseCache.Load(cachedKey); ok {
+			if respCache, ok := cached.(propfindResponse); ok {
-					responseData := cachedResponse.([]byte)
+				if time.Since(respCache.ts) < ttl {
 					w.Header().Set("Content-Type", "application/xml; charset=utf-8")
-					w.Header().Set("Content-Length", fmt.Sprintf("%d", len(responseData)))
+					w.Header().Set("Content-Length", fmt.Sprintf("%d", len(respCache.data)))
-					w.Write(responseData)
+					w.Write(respCache.data)
 					return
 				}
-
-				// Otherwise process normally but cache the result
-				responseRecorder := httptest.NewRecorder()
-
-				// Process the request with the standard handler
-				handler := &webdav.Handler{
-					FileSystem: h,
-					LockSystem: webdav.NewMemLS(),
-					Logger: func(r *http.Request, err error) {
-						if err != nil {
-							h.logger.Error().Err(err).Msg("WebDAV error")
-						}
-					},
-				}
-				handler.ServeHTTP(responseRecorder, r)
-
-				// Cache the response for future requests
-				responseData := responseRecorder.Body.Bytes()
-				h.responseCache.Store(cachedKey, responseData)
-
-				// Send to the real client
-				for k, v := range responseRecorder.Header() {
-					w.Header()[k] = v
-				}
-				w.WriteHeader(responseRecorder.Code)
-				w.Write(responseData)
-				return
 			}
 		}
 
-		h.logger.Debug().
+		// No valid cache entry; process the PROPFIND request.
-			Dur("propfind_prepare", time.Since(propfindStart)).
+		responseRecorder := httptest.NewRecorder()
-			Msg("Proceeding with standard PROPFIND")
+		handler := &webdav.Handler{
+			FileSystem: h,
+			LockSystem: webdav.NewMemLS(),
+			Logger: func(r *http.Request, err error) {
+				if err != nil {
+					h.logger.Error().Err(err).Msg("WebDAV error")
+				}
+			},
+		}
+		handler.ServeHTTP(responseRecorder, r)
+		responseData := responseRecorder.Body.Bytes()
+
+		// Store the new response in the cache.
+		h.cache.propfindResp.Store(cacheKey, propfindResponse{
+			data: responseData,
+			ts:   time.Now(),
+		})
+
+		// Forward the captured response to the client.
+		for k, v := range responseRecorder.Header() {
+			w.Header()[k] = v
+		}
+		w.WriteHeader(responseRecorder.Code)
+		w.Write(responseData)
+		return
 	}
 
-	// Check if this is a GET request for a file
+	// Handle GET requests for file/directory content
 	if r.Method == "GET" {
-		openStart := time.Now()
 		f, err := h.OpenFile(r.Context(), r.URL.Path, os.O_RDONLY, 0)
 		if err != nil {
 			h.logger.Debug().Err(err).Str("path", r.URL.Path).Msg("Failed to open file")
@@ -304,17 +308,12 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
+		// If the target is a directory, use your directory listing logic.
 		if fi.IsDir() {
-			dirStart := time.Now()
 			h.serveDirectory(w, r, f)
-			h.logger.Info().
-				Dur("directory_time", time.Since(dirStart)).
-				Msg("Directory served")
 			return
 		}
 
-		// For file requests, use http.ServeContent.
-		// Ensure f implements io.ReadSeeker.
 		rs, ok := f.(io.ReadSeeker)
 		if !ok {
 			// If not, read the entire file into memory as a fallback.
@@ -326,8 +325,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			}
 			rs = bytes.NewReader(buf)
 		}
-
-		// Set Content-Type based on file name.
 		fileName := fi.Name()
 		contentType := getContentType(fileName)
 		w.Header().Set("Content-Type", contentType)
@@ -335,13 +332,62 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		// Serve the file with the correct modification time.
 		// http.ServeContent automatically handles Range requests.
 		http.ServeContent(w, r, fileName, fi.ModTime(), rs)
-		h.logger.Info().
+
-			Dur("open_attempt_time", time.Since(openStart)).
+		// Set headers to indicate support for range requests and content type.
-			Msg("Served file using ServeContent")
+		//fileName := fi.Name()
+		//w.Header().Set("Accept-Ranges", "bytes")
+		//w.Header().Set("Content-Type", getContentType(fileName))
+		//
+		//// If a Range header is provided, parse and handle partial content.
+		//rangeHeader := r.Header.Get("Range")
+		//if rangeHeader != "" {
+		//	parts := strings.Split(strings.TrimPrefix(rangeHeader, "bytes="), "-")
+		//	if len(parts) == 2 {
+		//		start, startErr := strconv.ParseInt(parts[0], 10, 64)
+		//		end := fi.Size() - 1
+		//		if parts[1] != "" {
+		//			var endErr error
+		//			end, endErr = strconv.ParseInt(parts[1], 10, 64)
+		//			if endErr != nil {
+		//				end = fi.Size() - 1
+		//			}
+		//		}
+		//
+		//		if startErr == nil && start < fi.Size() {
+		//			if start > end {
+		//				start, end = end, start
+		//			}
+		//			if end >= fi.Size() {
+		//				end = fi.Size() - 1
+		//			}
+		//
+		//			contentLength := end - start + 1
+		//			w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, fi.Size()))
+		//			w.Header().Set("Content-Length", fmt.Sprintf("%d", contentLength))
+		//			w.WriteHeader(http.StatusPartialContent)
+		//
+		//			// Attempt to cast to your concrete File type to call Seek.
+		//			if file, ok := f.(*File); ok {
+		//				_, err = file.Seek(start, io.SeekStart)
+		//				if err != nil {
+		//					h.logger.Error().Err(err).Msg("Failed to seek in file")
+		//					http.Error(w, "Server Error", http.StatusInternalServerError)
+		//					return
+		//				}
+		//
+		//				limitedReader := io.LimitReader(f, contentLength)
+		//				h.ioCopy(limitedReader, w)
+		//				return
+		//			}
+		//		}
+		//	}
+		//}
+		//w.Header().Set("Content-Length", fmt.Sprintf("%d", fi.Size()))
+		//h.ioCopy(f, w)
 		return
 	}
 
-	// Default to standard WebDAV handler for other requests
+	// Fallback: for other methods, use the standard WebDAV handler.
 	handler := &webdav.Handler{
 		FileSystem: h,
 		LockSystem: webdav.NewMemLS(),
@@ -355,7 +401,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			}
 		},
 	}
-
 	handler.ServeHTTP(w, r)
 }
 
@@ -384,6 +429,17 @@ func getContentType(fileName string) string {
 	return contentType
 }
 
+func (h *Handler) isParentPath(_path string) bool {
+	rootPath := h.getRootPath()
+	parents := h.getParentItems()
+	for _, p := range parents {
+		if _path == path.Join(rootPath, p) {
+			return true
+		}
+	}
+	return false
+}
+
 func (h *Handler) serveDirectory(w http.ResponseWriter, r *http.Request, file webdav.File) {
 	var children []os.FileInfo
 	if f, ok := file.(*File); ok {
@@ -432,36 +488,35 @@ func (h *Handler) serveDirectory(w http.ResponseWriter, r *http.Request, file we
 }
 
 func (h *Handler) ioCopy(reader io.Reader, w io.Writer) (int64, error) {
-	// Start with a smaller initial buffer for faster first byte time
+	// Start with a smaller buffer for faster first byte delivery.
-	buffer := make([]byte, 8*1024) // 8KB initial buffer
+	buf := make([]byte, 4*1024) // 8KB initial buffer
-	written := int64(0)
+	totalWritten := int64(0)
-
-	// First chunk needs to be delivered ASAP
 	firstChunk := true
 
 	for {
-		n, err := reader.Read(buffer)
+		n, err := reader.Read(buf)
 		if n > 0 {
-			nw, ew := w.Write(buffer[:n])
+			nw, ew := w.Write(buf[:n])
 			if ew != nil {
 				var opErr *net.OpError
 				if errors.As(ew, &opErr) && opErr.Err.Error() == "write: broken pipe" {
 					h.logger.Debug().Msg("Client closed connection (normal for streaming)")
+					return totalWritten, ew
 				}
-				break
+				return totalWritten, ew
 			}
-			written += int64(nw)
+			totalWritten += int64(nw)
 
-			// Flush immediately after first chunk, then less frequently
+			// Flush immediately after the first chunk.
 			if firstChunk {
 				if flusher, ok := w.(http.Flusher); ok {
 					flusher.Flush()
 				}
 				firstChunk = false
-
+				// Increase buffer size for subsequent reads.
-				// Increase buffer size after first chunk
+				buf = make([]byte, 512*1024) // 64KB buffer after first chunk
-				buffer = make([]byte, 64*1024) // 512KB for subsequent reads
+			} else if totalWritten%(2*1024*1024) < int64(n) {
-			} else if written%(2*1024*1024) < int64(n) { // Flush every 2MB
+				// Flush roughly every 2MB of data transferred.
 				if flusher, ok := w.(http.Flusher); ok {
 					flusher.Flush()
 				}
@@ -476,5 +531,5 @@ func (h *Handler) ioCopy(reader io.Reader, w io.Writer) (int64, error) {
 		}
 	}
 
-	return written, nil
+	return totalWritten, nil
 }

pkg/webdav/webdav.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"github.com/go-chi/chi/v5"
-	"github.com/sirrobot01/debrid-blackhole/internal/config"
 	"github.com/sirrobot01/debrid-blackhole/internal/logger"
 	"github.com/sirrobot01/debrid-blackhole/pkg/service"
 	"html/template"
@@ -18,14 +17,13 @@ type WebDav struct {
 
 func New() *WebDav {
 	svc := service.GetService()
-	cfg := config.GetConfig()
 	w := &WebDav{
 		Handlers: make([]*Handler, 0),
 	}
 	debrids := svc.Debrid.GetDebrids()
 	cacheManager := NewCacheManager(debrids)
 	for name, c := range cacheManager.GetCaches() {
-		h := NewHandler(name, c, logger.NewLogger(fmt.Sprintf("%s-webdav", name), cfg.LogLevel))
+		h := NewHandler(name, c, logger.NewLogger(fmt.Sprintf("%s-webdav", name)))
 		w.Handlers = append(w.Handlers, h)
 	}
 	return w

pkg/webdav/workers.go

@@ -0,0 +1,69 @@
+package webdav
+
+import "time"
+
+func (c *Cache) Refresh() error {
+	// For now, we just want to refresh the listing and download links
+	c.logger.Info().Msg("Starting cache refresh workers")
+	go c.refreshListingWorker()
+	go c.refreshDownloadLinksWorker()
+	go c.refreshTorrentsWorker()
+	return nil
+}
+
+func (c *Cache) refreshListingWorker() {
+	refreshTicker := time.NewTicker(10 * time.Second)
+	defer refreshTicker.Stop()
+
+	for {
+		select {
+		case <-refreshTicker.C:
+			if c.listingRefreshMu.TryLock() {
+				func() {
+					defer c.listingRefreshMu.Unlock()
+					c.refreshListings()
+				}()
+			} else {
+				c.logger.Debug().Msg("Refresh already in progress")
+			}
+		}
+	}
+}
+
+func (c *Cache) refreshDownloadLinksWorker() {
+	refreshTicker := time.NewTicker(40 * time.Minute)
+	defer refreshTicker.Stop()
+
+	for {
+		select {
+		case <-refreshTicker.C:
+			if c.downloadLinksRefreshMu.TryLock() {
+				func() {
+					defer c.downloadLinksRefreshMu.Unlock()
+					c.refreshDownloadLinks()
+				}()
+			} else {
+				c.logger.Debug().Msg("Refresh already in progress")
+			}
+		}
+	}
+}
+
+func (c *Cache) refreshTorrentsWorker() {
+	refreshTicker := time.NewTicker(5 * time.Second)
+	defer refreshTicker.Stop()
+
+	for {
+		select {
+		case <-refreshTicker.C:
+			if c.listingRefreshMu.TryLock() {
+				func() {
+					defer c.listingRefreshMu.Unlock()
+					c.refreshTorrents()
+				}()
+			} else {
+				c.logger.Debug().Msg("Refresh already in progress")
+			}
+		}
+	}
+}

pkg/worker/worker.go

@@ -18,8 +18,7 @@ var (
 func getLogger() zerolog.Logger {
 
 	once.Do(func() {
-		cfg := config.GetConfig()
+		_logInstance = logger.NewLogger("worker")
-		_logInstance = logger.NewLogger("worker", cfg.LogLevel)
 	})
 	return _logInstance
 }