johno/decypharr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve webdav; add workers for refreshes

Browse Source
This commit is contained in:
Mukhtar Akere
2025-03-19 03:08:22 +01:00
parent 5d2fabe20b
commit 0d178992ef
18 changed files with 438 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/logger/logger.go
View File

@@ -29,7 +29,9 @@ func GetLogPath() string {
return filepath.Join(logsDir, "decypharr.log")
}
func NewLogger(prefix string, level string) zerolog.Logger {
func NewLogger(prefix string) zerolog.Logger {
level := config.GetConfig().LogLevel
rotatingLogFile := &lumberjack.Logger{
Filename: GetLogPath(),
@@ -86,8 +88,7 @@ func NewLogger(prefix string, level string) zerolog.Logger {
func GetDefaultLogger() zerolog.Logger {
once.Do(func() {
cfg := config.GetConfig()
logger = NewLogger("decypharr", cfg.LogLevel)
logger = NewLogger("decypharr")
})
return logger
}

3
internal/request/request.go
View File

@@ -7,7 +7,6 @@ import (
"fmt"
"github.com/goccy/go-json"
"github.com/rs/zerolog"
"github.com/sirrobot01/debrid-blackhole/internal/config"
"github.com/sirrobot01/debrid-blackhole/internal/logger"
"golang.org/x/time/rate"
"io"
@@ -227,7 +226,7 @@ func New(options ...ClientOption) *Client {
http.StatusServiceUnavailable: true,
http.StatusGatewayTimeout: true,
},
logger: logger.NewLogger("request", config.GetConfig().LogLevel),
logger: logger.NewLogger("request"),
}
// Apply options

4
main.go
View File

@@ -5,6 +5,7 @@ import (
"flag"
"github.com/sirrobot01/debrid-blackhole/cmd/decypharr"
"github.com/sirrobot01/debrid-blackhole/internal/config"
"github.com/sirrobot01/debrid-blackhole/pkg/version"
"log"
"net/http"
_ "net/http/pprof" // registers pprof handlers
@@ -19,11 +20,14 @@ func main() {
}
}()
if version.GetInfo().Channel == "dev" {
log.Println("Running in dev mode")
go func() {
if err := http.ListenAndServe(":6060", nil); err != nil {
log.Fatalf("pprof server failed: %v", err)
}
}()
}
var configPath string
flag.StringVar(&configPath, "config", "/data", "path to the data folder")
flag.Parse()

4
pkg/debrid/alldebrid/alldebrid.go
View File

@@ -288,7 +288,7 @@ func New(dc config.Debrid) *AllDebrid {
headers := map[string]string{
"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
}
_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
_log := logger.NewLogger(dc.Name)
client := request.New().
WithHeaders(headers).
WithRateLimiter(rl).WithLogger(_log)
@@ -299,7 +299,7 @@ func New(dc config.Debrid) *AllDebrid {
DownloadUncached: dc.DownloadUncached,
client: client,
MountPath: dc.Folder,
logger: logger.NewLogger(dc.Name, config.GetConfig().LogLevel),
logger: logger.NewLogger(dc.Name),
CheckCached: dc.CheckCached,
}
}

4
pkg/debrid/debrid_link/debrid_link.go
View File

@@ -264,7 +264,7 @@ func New(dc config.Debrid) *DebridLink {
"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
"Content-Type": "application/json",
}
_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
_log := logger.NewLogger(dc.Name)
client := request.New().
WithHeaders(headers).
WithRateLimiter(rl).WithLogger(_log)
@@ -275,7 +275,7 @@ func New(dc config.Debrid) *DebridLink {
DownloadUncached: dc.DownloadUncached,
client: client,
MountPath: dc.Folder,
logger: logger.NewLogger(dc.Name, config.GetConfig().LogLevel),
logger: logger.NewLogger(dc.Name),
CheckCached: dc.CheckCached,
}
}

4
pkg/debrid/realdebrid/realdebrid.go
View File

@@ -517,7 +517,7 @@ func New(dc config.Debrid) *RealDebrid {
headers := map[string]string{
"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
}
_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
_log := logger.NewLogger(dc.Name)
client := request.New().
WithHeaders(headers).
WithRateLimiter(rl).WithLogger(_log)
@@ -528,7 +528,7 @@ func New(dc config.Debrid) *RealDebrid {
DownloadUncached: dc.DownloadUncached,
client: client,
MountPath: dc.Folder,
logger: logger.NewLogger(dc.Name, config.GetConfig().LogLevel),
logger: logger.NewLogger(dc.Name),
CheckCached: dc.CheckCached,
}
}

2
pkg/debrid/torbox/torbox.go
View File

@@ -319,7 +319,7 @@ func New(dc config.Debrid) *Torbox {
headers := map[string]string{
"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
}
_log := logger.NewLogger(dc.Name, config.GetConfig().LogLevel)
_log := logger.NewLogger(dc.Name)
client := request.New().
WithHeaders(headers).
WithRateLimiter(rl).WithLogger(_log)

2
pkg/proxy/proxy.go
View File

@@ -88,7 +88,7 @@ func NewProxy() *Proxy {
username: cfg.Username,
password: cfg.Password,
cachedOnly: cfg.CachedOnly,
logger: logger.NewLogger("proxy", cfg.LogLevel),
logger: logger.NewLogger("proxy"),
}
}

2
pkg/qbit/qbit.go
View File

@@ -34,7 +34,7 @@ func New() *QBit {
DownloadFolder: cfg.DownloadFolder,
Categories: cfg.Categories,
Storage: NewTorrentStorage(filepath.Join(_cfg.Path, "torrents.json")),
logger: logger.NewLogger("qbit", _cfg.LogLevel),
logger: logger.NewLogger("qbit"),
RefreshInterval: refreshInterval,
SkipPreCache: cfg.SkipPreCache,
}

2
pkg/repair/repair.go
View File

@@ -47,7 +47,7 @@ func New(arrs *arr.Storage) *Repair {
}
r := &Repair{
arrs: arrs,
logger: logger.NewLogger("repair", cfg.LogLevel),
logger: logger.NewLogger("repair"),
duration: duration,
runOnStart: cfg.Repair.RunOnStart,
ZurgURL: cfg.Repair.ZurgURL,

3
pkg/server/server.go
View File

@@ -22,8 +22,7 @@ type Server struct {
}
func New() *Server {
cfg := config.GetConfig()
l := logger.NewLogger("http", cfg.LogLevel)
l := logger.NewLogger("http")
r := chi.NewRouter()
r.Use(middleware.Recoverer)
r.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

3
pkg/web/server.go
View File

@@ -60,10 +60,9 @@ type Handler struct {
}
func New(qbit *qbit.QBit) *Handler {
cfg := config.GetConfig()
return &Handler{
qbit: qbit,
logger: logger.NewLogger("ui", cfg.LogLevel),
logger: logger.NewLogger("ui"),
}
}

265
pkg/webdav/cache.go
View File

@@ -7,10 +7,12 @@ import (
"github.com/dgraph-io/badger/v4"
"github.com/goccy/go-json"
"github.com/rs/zerolog"
"github.com/sirrobot01/debrid-blackhole/internal/logger"
"github.com/sirrobot01/debrid-blackhole/pkg/debrid/debrid"
"os"
"path/filepath"
"runtime"
"sort"
"sync"
"sync/atomic"
"time"
@@ -23,6 +25,11 @@ type DownloadLinkCache struct {
Link string `json:"download_link"`
}
type propfindResponse struct {
data []byte
ts time.Time
}
type CachedTorrent struct {
*torrent.Torrent
LastRead time.Time `json:"last_read"`
@@ -39,27 +46,29 @@ type Cache struct {
torrentsNames map[string]*CachedTorrent // key: torrent.Name, value: torrent
listings atomic.Value
downloadLinks map[string]string // key: file.Link, value: download link
propfindResp sync.Map
workers int
LastUpdated time.Time `json:"last_updated"`
// refresh mutex
torrentsRefreshMutex sync.Mutex // for refreshing torrents
downloadLinksRefreshMutex sync.Mutex // for refreshing download links
listingRefreshMu sync.Mutex // for refreshing torrents
downloadLinksRefreshMu sync.Mutex // for refreshing download links
torrentsRefreshMu sync.Mutex // for refreshing torrents
// Mutexes
// Data Mutexes
torrentsMutex sync.RWMutex // for torrents and torrentsNames
downloadLinksMutex sync.Mutex
downloadLinksMutex sync.Mutex // for downloadLinks
}
func (c *Cache) setTorrent(t *CachedTorrent) {
c.torrentsMutex.Lock()
defer c.torrentsMutex.Unlock()
c.torrents[t.Id] = t
c.torrentsNames[t.Name] = t
c.torrentsMutex.Unlock()
c.refreshListings()
go c.refreshListings() // This is concurrent safe
go func() {
if err := c.SaveTorrent(t); err != nil {
@@ -69,19 +78,31 @@ func (c *Cache) setTorrent(t *CachedTorrent) {
}
func (c *Cache) refreshListings() {
files := make([]os.FileInfo, 0, len(c.torrents))
now := time.Now()
// Copy the current torrents to avoid concurrent issues
c.torrentsMutex.RLock()
torrents := make([]string, 0, len(c.torrents))
for _, t := range c.torrents {
if t != nil && t.Torrent != nil {
torrents = append(torrents, t.Name)
}
}
c.torrentsMutex.RUnlock()
sort.Slice(torrents, func(i, j int) bool {
return torrents[i] < torrents[j]
})
files := make([]os.FileInfo, 0, len(torrents))
now := time.Now()
for _, t := range torrents {
files = append(files, &FileInfo{
name: t.Name,
name: t,
size: 0,
mode: 0755 | os.ModeDir,
modTime: now,
isDir: true,
})
}
}
// Atomic store of the complete ready-to-use slice
c.listings.Store(files)
}
@@ -90,15 +111,16 @@ func (c *Cache) GetListing() []os.FileInfo {
return c.listings.Load().([]os.FileInfo)
}
func (c *Cache) setTorrents(torrents []*CachedTorrent) {
func (c *Cache) setTorrents(torrents map[string]*CachedTorrent) {
c.torrentsMutex.Lock()
defer c.torrentsMutex.Unlock()
for _, t := range torrents {
c.torrents[t.Id] = t
c.torrentsNames[t.Name] = t
}
go c.refreshListings()
c.torrentsMutex.Unlock()
go c.refreshListings() // This is concurrent safe
go func() {
if err := c.SaveTorrents(); err != nil {
@@ -148,13 +170,14 @@ func (m *Manager) GetCache(debridName string) *Cache {
}
func NewCache(client debrid.Client) *Cache {
dbPath := filepath.Join(config.GetConfig().Path, "cache", client.GetName())
cfg := config.GetConfig()
dbPath := filepath.Join(cfg.Path, "cache", client.GetName())
return &Cache{
dir: dbPath,
torrents: make(map[string]*CachedTorrent),
torrentsNames: make(map[string]*CachedTorrent),
client: client,
logger: client.GetLogger(),
logger: logger.NewLogger(fmt.Sprintf("%s-cache", client.GetName())),
workers: 200,
downloadLinks: make(map[string]string),
}
@@ -172,8 +195,8 @@ func (c *Cache) Start() error {
// initial download links
go func() {
// lock download refresh mutex
c.downloadLinksRefreshMutex.Lock()
defer c.downloadLinksRefreshMutex.Unlock()
c.downloadLinksRefreshMu.Lock()
defer c.downloadLinksRefreshMu.Unlock()
// This prevents the download links from being refreshed twice
c.refreshDownloadLinks()
}()
@@ -195,8 +218,8 @@ func (c *Cache) Close() error {
return nil
}
func (c *Cache) load() ([]*CachedTorrent, error) {
torrents := make([]*CachedTorrent, 0)
func (c *Cache) load() (map[string]*CachedTorrent, error) {
torrents := make(map[string]*CachedTorrent)
if err := os.MkdirAll(c.dir, 0755); err != nil {
return torrents, fmt.Errorf("failed to create cache directory: %w", err)
}
@@ -225,7 +248,8 @@ func (c *Cache) load() ([]*CachedTorrent, error) {
}
if len(ct.Files) != 0 {
// We can assume the torrent is complete
torrents = append(torrents, &ct)
ct.IsComplete = true
torrents[ct.Id] = &ct
}
}
@@ -290,27 +314,48 @@ func (c *Cache) Sync() error {
if err != nil {
c.logger.Debug().Err(err).Msg("Failed to load cache")
}
// Write these torrents to the cache
c.setTorrents(cachedTorrents)
c.logger.Info().Msgf("Loaded %d torrents from cache", len(cachedTorrents))
torrents, err := c.client.GetTorrents()
c.logger.Info().Msgf("Got %d torrents from %s", len(torrents), c.client.GetName())
if err != nil {
return fmt.Errorf("failed to sync torrents: %v", err)
}
mewTorrents := make([]*torrent.Torrent, 0)
for _, t := range torrents {
if _, ok := c.torrents[t.Id]; !ok {
mewTorrents = append(mewTorrents, t)
}
}
c.logger.Info().Msgf("Found %d new torrents", len(mewTorrents))
c.logger.Info().Msgf("Got %d torrents from %s", len(torrents), c.client.GetName())
if len(mewTorrents) > 0 {
if err := c.sync(mewTorrents); err != nil {
newTorrents := make([]*torrent.Torrent, 0)
idStore := make(map[string]bool, len(torrents))
for _, t := range torrents {
idStore[t.Id] = true
if _, ok := cachedTorrents[t.Id]; !ok {
newTorrents = append(newTorrents, t)
}
}
// Check for deleted torrents
deletedTorrents := make([]string, 0)
for _, t := range cachedTorrents {
if _, ok := idStore[t.Id]; !ok {
deletedTorrents = append(deletedTorrents, t.Id)
}
}
if len(deletedTorrents) > 0 {
c.logger.Info().Msgf("Found %d deleted torrents", len(deletedTorrents))
for _, id := range deletedTorrents {
if _, ok := cachedTorrents[id]; ok {
delete(cachedTorrents, id)
c.removeFromDB(id)
}
}
}
// Write these torrents to the cache
c.setTorrents(cachedTorrents)
c.logger.Info().Msgf("Loaded %d torrents from cache", len(cachedTorrents))
if len(newTorrents) > 0 {
c.logger.Info().Msgf("Found %d new torrents", len(newTorrents))
if err := c.sync(newTorrents); err != nil {
return fmt.Errorf("failed to sync torrents: %v", err)
}
}
@@ -474,46 +519,6 @@ func (c *Cache) refreshTorrent(t *CachedTorrent) *CachedTorrent {
return ct
}
func (c *Cache) refreshListingWorker() {
c.logger.Info().Msg("WebDAV Background Refresh Worker started")
refreshTicker := time.NewTicker(10 * time.Second)
defer refreshTicker.Stop()
for {
select {
case <-refreshTicker.C:
if c.torrentsRefreshMutex.TryLock() {
func() {
defer c.torrentsRefreshMutex.Unlock()
c.refreshListings()
}()
} else {
c.logger.Debug().Msg("Refresh already in progress")
}
}
}
}
func (c *Cache) refreshDownloadLinksWorker() {
c.logger.Info().Msg("WebDAV Background Refresh Download Worker started")
refreshTicker := time.NewTicker(40 * time.Minute)
defer refreshTicker.Stop()
for {
select {
case <-refreshTicker.C:
if c.downloadLinksRefreshMutex.TryLock() {
func() {
defer c.downloadLinksRefreshMutex.Unlock()
c.refreshDownloadLinks()
}()
} else {
c.logger.Debug().Msg("Refresh already in progress")
}
}
}
}
func (c *Cache) refreshDownloadLinks() map[string]string {
c.downloadLinksMutex.Lock()
defer c.downloadLinksMutex.Unlock()
@@ -526,7 +531,6 @@ func (c *Cache) refreshDownloadLinks() map[string]string {
for k, v := range downloadLinks {
c.downloadLinks[k] = v.DownloadLink
}
c.logger.Info().Msgf("Refreshed %d download links", len(downloadLinks))
return c.downloadLinks
}
@@ -534,9 +538,110 @@ func (c *Cache) GetClient() debrid.Client {
return c.client
}
func (c *Cache) Refresh() error {
// For now, we just want to refresh the listing
go c.refreshListingWorker()
go c.refreshDownloadLinksWorker()
return nil
func (c *Cache) refreshTorrents() {
c.torrentsMutex.RLock()
currentTorrents := c.torrents //
// Create a copy of the current torrents to avoid concurrent issues
torrents := make(map[string]string, len(currentTorrents)) // a mpa of id and name
for _, v := range currentTorrents {
torrents[v.Id] = v.Name
}
c.torrentsMutex.RUnlock()
// Get new torrents from the debrid service
debTorrents, err := c.client.GetTorrents()
if err != nil {
c.logger.Debug().Err(err).Msg("Failed to get torrents")
return
}
if len(debTorrents) == 0 {
// Maybe an error occurred
return
}
// Get the newly added torrents only
newTorrents := make([]*torrent.Torrent, 0)
idStore := make(map[string]bool, len(debTorrents))
for _, t := range debTorrents {
idStore[t.Id] = true
if _, ok := torrents[t.Id]; !ok {
newTorrents = append(newTorrents, t)
}
}
// Check for deleted torrents
deletedTorrents := make([]string, 0)
for id, _ := range torrents {
if _, ok := idStore[id]; !ok {
deletedTorrents = append(deletedTorrents, id)
}
}
if len(deletedTorrents) > 0 {
c.DeleteTorrent(deletedTorrents)
}
if len(newTorrents) == 0 {
return
}
c.logger.Info().Msgf("Found %d new torrents", len(newTorrents))
// No need for a complex sync process, just add the new torrents
wg := sync.WaitGroup{}
wg.Add(len(newTorrents))
for _, t := range newTorrents {
// processTorrent is concurrent safe
go func() {
defer wg.Done()
if err := c.processTorrent(t); err != nil {
c.logger.Info().Err(err).Msg("Failed to process torrent")
}
}()
}
wg.Wait()
}
func (c *Cache) DeleteTorrent(ids []string) {
c.logger.Info().Msgf("Deleting %d torrents", len(ids))
c.torrentsMutex.Lock()
defer c.torrentsMutex.Unlock()
for _, id := range ids {
if t, ok := c.torrents[id]; ok {
delete(c.torrents, id)
delete(c.torrentsNames, t.Name)
c.removeFromDB(id)
}
}
}
func (c *Cache) removeFromDB(torrentId string) {
filePath := filepath.Join(c.dir, torrentId+".json")
if err := os.Remove(filePath); err != nil {
c.logger.Debug().Err(err).Msgf("Failed to remove file: %s", filePath)
}
}
func (c *Cache) resetPropfindResponse() {
// Right now, parents are hardcoded
parents := []string{"__all__", "torrents"}
// Reset only the parent directories
// Convert the parents to a keys
// This is a bit hacky, but it works
// Instead of deleting all the keys, we only delete the parent keys, e.g __all__/ or torrents/
keys := make([]string, 0, len(parents))
for _, p := range parents {
// Construct the key
// construct url
url := filepath.Join("/webdav/%s/%s", c.client.GetName(), p)
key0 := fmt.Sprintf("propfind:%s:0", url)
key1 := fmt.Sprintf("propfind:%s:1", url)
keys = append(keys, key0, key1)
}
// Delete the keys
for _, k := range keys {
c.propfindResp.Delete(k)
}
}

2
pkg/webdav/file.go
View File

@@ -138,7 +138,7 @@ func (f *File) Seek(offset int64, whence int) (int64, error) {
case io.SeekCurrent:
newOffset = f.offset + offset
case io.SeekEnd:
newOffset = f.size - offset
newOffset = f.size + offset
default:
return 0, os.ErrInvalid
}

221
pkg/webdav/handler.go
View File

@@ -15,6 +15,7 @@ import (
"net/http/httptest"
"os"
"path"
"slices"
"strings"
"sync"
"time"
@@ -52,7 +53,7 @@ func (h *Handler) Mkdir(ctx context.Context, name string, perm os.FileMode) erro
func (h *Handler) RemoveAll(ctx context.Context, name string) error {
name = path.Clean("/" + name)
rootDir := h.getParentRootPath()
rootDir := h.getRootPath()
if name == rootDir {
return os.ErrPermission
@@ -67,6 +68,8 @@ func (h *Handler) RemoveAll(ctx context.Context, name string) error {
if filename == "" {
h.cache.GetClient().DeleteTorrent(cachedTorrent.Torrent)
go h.cache.refreshListings()
go h.cache.refreshTorrents()
go h.cache.resetPropfindResponse()
return nil
}
@@ -78,7 +81,7 @@ func (h *Handler) Rename(ctx context.Context, oldName, newName string) error {
return os.ErrPermission // Read-only filesystem
}
func (h *Handler) getParentRootPath() string {
func (h *Handler) getRootPath() string {
return fmt.Sprintf("/webdav/%s", h.Name)
}
@@ -86,37 +89,33 @@ func (h *Handler) getTorrentsFolders() []os.FileInfo {
return h.cache.GetListing()
}
func (h *Handler) getParentItems() []string {
return []string{"__all__", "torrents", "version.txt"}
}
func (h *Handler) getParentFiles() []os.FileInfo {
now := time.Now()
rootFiles := []os.FileInfo{
&FileInfo{
name: "__all__",
rootFiles := make([]os.FileInfo, 0, len(h.getParentItems()))
for _, item := range h.getParentItems() {
f := &FileInfo{
name: item,
size: 0,
mode: 0755 | os.ModeDir,
modTime: now,
isDir: true,
},
&FileInfo{
name: "torrents",
size: 0,
mode: 0755 | os.ModeDir,
modTime: now,
isDir: true,
},
&FileInfo{
name: "version.txt",
size: int64(len("v1.0.0")),
mode: 0644,
modTime: now,
isDir: false,
},
}
if item == "version.txt" {
f.isDir = false
f.size = int64(len("v1.0.0"))
}
rootFiles = append(rootFiles, f)
}
return rootFiles
}
func (h *Handler) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
name = path.Clean("/" + name)
rootDir := h.getParentRootPath()
rootDir := h.getRootPath()
// Fast path optimization with a map lookup instead of string comparisons
switch name {
@@ -138,7 +137,7 @@ func (h *Handler) OpenFile(ctx context.Context, name string, flag int, perm os.F
}
// Single check for top-level folders
if name == path.Join(rootDir, "__all__") || name == path.Join(rootDir, "torrents") {
if h.isParentPath(name) {
folderName := strings.TrimPrefix(name, rootDir)
folderName = strings.TrimPrefix(folderName, "/")
@@ -157,7 +156,7 @@ func (h *Handler) OpenFile(ctx context.Context, name string, flag int, perm os.F
_path := strings.TrimPrefix(name, rootDir)
parts := strings.Split(strings.TrimPrefix(_path, "/"), "/")
if len(parts) >= 2 && (parts[0] == "__all__" || parts[0] == "torrents") {
if len(parts) >= 2 && (slices.Contains(h.getParentItems(), parts[0])) {
torrentName := parts[1]
cachedTorrent := h.cache.GetTorrentByName(torrentName)
@@ -224,38 +223,47 @@ func (h *Handler) getFileInfos(torrent *torrent.Torrent) []os.FileInfo {
}
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Handle OPTIONS
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
return
}
//Add specific PROPFIND optimization
// Cache PROPFIND responses for a short time to reduce load.
if r.Method == "PROPFIND" {
propfindStart := time.Now()
// Check if this is the slow path we identified
if strings.Contains(r.URL.Path, "__all__") {
// Fast path for this specific directory
// Determine the Depth; default to "1" if not provided.
depth := r.Header.Get("Depth")
if depth == "1" || depth == "" {
// This is a listing request
if depth == "" {
depth = "1"
}
// Use both path and Depth header to form the cache key.
cacheKey := fmt.Sprintf("propfind:%s:%s", r.URL.Path, depth)
// Use a cached response if available
cachedKey := "propfind_" + r.URL.Path
if cachedResponse, ok := h.responseCache.Load(cachedKey); ok {
responseData := cachedResponse.([]byte)
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(responseData)))
w.Write(responseData)
return
// Determine TTL based on the requested folder:
// - If the path is exactly the parent folder (which changes frequently),
// use a short TTL.
// - Otherwise, for deeper (torrent folder) paths, use a longer TTL.
var ttl time.Duration
if h.isParentPath(r.URL.Path) {
ttl = 10 * time.Second
} else {
ttl = 1 * time.Minute
}
// Otherwise process normally but cache the result
responseRecorder := httptest.NewRecorder()
// Check if we have a cached response that hasn't expired.
if cached, ok := h.cache.propfindResp.Load(cacheKey); ok {
if respCache, ok := cached.(propfindResponse); ok {
if time.Since(respCache.ts) < ttl {
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(respCache.data)))
w.Write(respCache.data)
return
}
}
}
// Process the request with the standard handler
// No valid cache entry; process the PROPFIND request.
responseRecorder := httptest.NewRecorder()
handler := &webdav.Handler{
FileSystem: h,
LockSystem: webdav.NewMemLS(),
@@ -266,12 +274,15 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
},
}
handler.ServeHTTP(responseRecorder, r)
// Cache the response for future requests
responseData := responseRecorder.Body.Bytes()
h.responseCache.Store(cachedKey, responseData)
// Send to the real client
// Store the new response in the cache.
h.cache.propfindResp.Store(cacheKey, propfindResponse{
data: responseData,
ts: time.Now(),
})
// Forward the captured response to the client.
for k, v := range responseRecorder.Header() {
w.Header()[k] = v
}
@@ -279,16 +290,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Write(responseData)
return
}
}
h.logger.Debug().
Dur("propfind_prepare", time.Since(propfindStart)).
Msg("Proceeding with standard PROPFIND")
}
// Check if this is a GET request for a file
// Handle GET requests for file/directory content
if r.Method == "GET" {
openStart := time.Now()
f, err := h.OpenFile(r.Context(), r.URL.Path, os.O_RDONLY, 0)
if err != nil {
h.logger.Debug().Err(err).Str("path", r.URL.Path).Msg("Failed to open file")
@@ -304,17 +308,12 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
// If the target is a directory, use your directory listing logic.
if fi.IsDir() {
dirStart := time.Now()
h.serveDirectory(w, r, f)
h.logger.Info().
Dur("directory_time", time.Since(dirStart)).
Msg("Directory served")
return
}
// For file requests, use http.ServeContent.
// Ensure f implements io.ReadSeeker.
rs, ok := f.(io.ReadSeeker)
if !ok {
// If not, read the entire file into memory as a fallback.
@@ -326,8 +325,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
rs = bytes.NewReader(buf)
}
// Set Content-Type based on file name.
fileName := fi.Name()
contentType := getContentType(fileName)
w.Header().Set("Content-Type", contentType)
@@ -335,13 +332,62 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Serve the file with the correct modification time.
// http.ServeContent automatically handles Range requests.
http.ServeContent(w, r, fileName, fi.ModTime(), rs)
h.logger.Info().
Dur("open_attempt_time", time.Since(openStart)).
Msg("Served file using ServeContent")
// Set headers to indicate support for range requests and content type.
//fileName := fi.Name()
//w.Header().Set("Accept-Ranges", "bytes")
//w.Header().Set("Content-Type", getContentType(fileName))
//
//// If a Range header is provided, parse and handle partial content.
//rangeHeader := r.Header.Get("Range")
//if rangeHeader != "" {
// parts := strings.Split(strings.TrimPrefix(rangeHeader, "bytes="), "-")
// if len(parts) == 2 {
// start, startErr := strconv.ParseInt(parts[0], 10, 64)
// end := fi.Size() - 1
// if parts[1] != "" {
// var endErr error
// end, endErr = strconv.ParseInt(parts[1], 10, 64)
// if endErr != nil {
// end = fi.Size() - 1
// }
// }
//
// if startErr == nil && start < fi.Size() {
// if start > end {
// start, end = end, start
// }
// if end >= fi.Size() {
// end = fi.Size() - 1
// }
//
// contentLength := end - start + 1
// w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, fi.Size()))
// w.Header().Set("Content-Length", fmt.Sprintf("%d", contentLength))
// w.WriteHeader(http.StatusPartialContent)
//
// // Attempt to cast to your concrete File type to call Seek.
// if file, ok := f.(*File); ok {
// _, err = file.Seek(start, io.SeekStart)
// if err != nil {
// h.logger.Error().Err(err).Msg("Failed to seek in file")
// http.Error(w, "Server Error", http.StatusInternalServerError)
// return
// }
//
// limitedReader := io.LimitReader(f, contentLength)
// h.ioCopy(limitedReader, w)
// return
// }
// }
// }
//}
//w.Header().Set("Content-Length", fmt.Sprintf("%d", fi.Size()))
//h.ioCopy(f, w)
return
}
// Default to standard WebDAV handler for other requests
// Fallback: for other methods, use the standard WebDAV handler.
handler := &webdav.Handler{
FileSystem: h,
LockSystem: webdav.NewMemLS(),
@@ -355,7 +401,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
},
}
handler.ServeHTTP(w, r)
}
@@ -384,6 +429,17 @@ func getContentType(fileName string) string {
return contentType
}
func (h *Handler) isParentPath(_path string) bool {
rootPath := h.getRootPath()
parents := h.getParentItems()
for _, p := range parents {
if _path == path.Join(rootPath, p) {
return true
}
}
return false
}
func (h *Handler) serveDirectory(w http.ResponseWriter, r *http.Request, file webdav.File) {
var children []os.FileInfo
if f, ok := file.(*File); ok {
@@ -432,36 +488,35 @@ func (h *Handler) serveDirectory(w http.ResponseWriter, r *http.Request, file we
}
func (h *Handler) ioCopy(reader io.Reader, w io.Writer) (int64, error) {
// Start with a smaller initial buffer for faster first byte time
buffer := make([]byte, 8*1024) // 8KB initial buffer
written := int64(0)
// First chunk needs to be delivered ASAP
// Start with a smaller buffer for faster first byte delivery.
buf := make([]byte, 4*1024) // 8KB initial buffer
totalWritten := int64(0)
firstChunk := true
for {
n, err := reader.Read(buffer)
n, err := reader.Read(buf)
if n > 0 {
nw, ew := w.Write(buffer[:n])
nw, ew := w.Write(buf[:n])
if ew != nil {
var opErr *net.OpError
if errors.As(ew, &opErr) && opErr.Err.Error() == "write: broken pipe" {
h.logger.Debug().Msg("Client closed connection (normal for streaming)")
return totalWritten, ew
}
break
return totalWritten, ew
}
written += int64(nw)
totalWritten += int64(nw)
// Flush immediately after first chunk, then less frequently
// Flush immediately after the first chunk.
if firstChunk {
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
}
firstChunk = false
// Increase buffer size after first chunk
buffer = make([]byte, 64*1024) // 512KB for subsequent reads
} else if written%(2*1024*1024) < int64(n) { // Flush every 2MB
// Increase buffer size for subsequent reads.
buf = make([]byte, 512*1024) // 64KB buffer after first chunk
} else if totalWritten%(2*1024*1024) < int64(n) {
// Flush roughly every 2MB of data transferred.
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
}
@@ -476,5 +531,5 @@ func (h *Handler) ioCopy(reader io.Reader, w io.Writer) (int64, error) {
}
}
return written, nil
return totalWritten, nil
}

4
pkg/webdav/webdav.go
View File

@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"github.com/go-chi/chi/v5"
"github.com/sirrobot01/debrid-blackhole/internal/config"
"github.com/sirrobot01/debrid-blackhole/internal/logger"
"github.com/sirrobot01/debrid-blackhole/pkg/service"
"html/template"
@@ -18,14 +17,13 @@ type WebDav struct {
func New() *WebDav {
svc := service.GetService()
cfg := config.GetConfig()
w := &WebDav{
Handlers: make([]*Handler, 0),
}
debrids := svc.Debrid.GetDebrids()
cacheManager := NewCacheManager(debrids)
for name, c := range cacheManager.GetCaches() {
h := NewHandler(name, c, logger.NewLogger(fmt.Sprintf("%s-webdav", name), cfg.LogLevel))
h := NewHandler(name, c, logger.NewLogger(fmt.Sprintf("%s-webdav", name)))
w.Handlers = append(w.Handlers, h)
}
return w

69
pkg/webdav/workers.go Normal file
View File

@@ -0,0 +1,69 @@
package webdav
import "time"
func (c *Cache) Refresh() error {
// For now, we just want to refresh the listing and download links
c.logger.Info().Msg("Starting cache refresh workers")
go c.refreshListingWorker()
go c.refreshDownloadLinksWorker()
go c.refreshTorrentsWorker()
return nil
}
func (c *Cache) refreshListingWorker() {
refreshTicker := time.NewTicker(10 * time.Second)
defer refreshTicker.Stop()
for {
select {
case <-refreshTicker.C:
if c.listingRefreshMu.TryLock() {
func() {
defer c.listingRefreshMu.Unlock()
c.refreshListings()
}()
} else {
c.logger.Debug().Msg("Refresh already in progress")
}
}
}
}
func (c *Cache) refreshDownloadLinksWorker() {
refreshTicker := time.NewTicker(40 * time.Minute)
defer refreshTicker.Stop()
for {
select {
case <-refreshTicker.C:
if c.downloadLinksRefreshMu.TryLock() {
func() {
defer c.downloadLinksRefreshMu.Unlock()
c.refreshDownloadLinks()
}()
} else {
c.logger.Debug().Msg("Refresh already in progress")
}
}
}
}
func (c *Cache) refreshTorrentsWorker() {
refreshTicker := time.NewTicker(5 * time.Second)
defer refreshTicker.Stop()
for {
select {
case <-refreshTicker.C:
if c.listingRefreshMu.TryLock() {
func() {
defer c.listingRefreshMu.Unlock()
c.refreshTorrents()
}()
} else {
c.logger.Debug().Msg("Refresh already in progress")
}
}
}
}

3
pkg/worker/worker.go
View File

@@ -18,8 +18,7 @@ var (
func getLogger() zerolog.Logger {
once.Do(func() {
cfg := config.GetConfig()
_logInstance = logger.NewLogger("worker", cfg.LogLevel)
_logInstance = logger.NewLogger("worker")
})
return _logInstance
}