diff --git a/.air.toml b/.air.toml index 37821e3..ee825ab 100644 --- a/.air.toml +++ b/.air.toml @@ -5,7 +5,7 @@ tmp_dir = "tmp" [build] args_bin = ["--config", "data/"] bin = "./tmp/main" - cmd = "bash -c 'go build -ldflags \"-X github.com/sirrobot01/debrid-blackhole/pkg/version.Version=0.0.0 -X github.com/sirrobot01/debrid-blackhole/pkg/version.Channel=nightly\" -o ./tmp/main .'" + cmd = "bash -c 'go build -ldflags \"-X github.com/sirrobot01/debrid-blackhole/pkg/version.Version=0.0.0 -X github.com/sirrobot01/debrid-blackhole/pkg/version.Channel=dev\" -o ./tmp/main .'" delay = 1000 exclude_dir = ["assets", "tmp", "vendor", "testdata", "data"] exclude_file = [] diff --git a/cmd/decypharr/main.go b/cmd/decypharr/main.go index ac42c6a..ed7c9bc 100644 --- a/cmd/decypharr/main.go +++ b/cmd/decypharr/main.go @@ -13,11 +13,24 @@ import ( "github.com/sirrobot01/debrid-blackhole/pkg/web" "github.com/sirrobot01/debrid-blackhole/pkg/webdav" "github.com/sirrobot01/debrid-blackhole/pkg/worker" + "os" "runtime/debug" + "strconv" "sync" + "syscall" ) func Start(ctx context.Context) error { + + if umaskStr := os.Getenv("UMASK"); umaskStr != "" { + umask, err := strconv.ParseInt(umaskStr, 8, 32) + if err != nil { + return fmt.Errorf("invalid UMASK value: %s", umaskStr) + } + // Set umask + syscall.Umask(int(umask)) + } + cfg := config.GetConfig() var wg sync.WaitGroup errChan := make(chan error) diff --git a/doc/config.full.json b/doc/config.full.json index b3107e5..dda1890 100644 --- a/doc/config.full.json +++ b/doc/config.full.json @@ -51,6 +51,7 @@ "download_folder": "/mnt/symlinks/", "categories": ["sonarr", "radarr"], "refresh_interval": 5, + "skip_pre_cache": false }, "arrs": [ { diff --git a/internal/config/config.go b/internal/config/config.go index 70f4372..7ceac43 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -41,6 +41,7 @@ type QBitTorrent struct { DownloadFolder string `json:"download_folder"` Categories []string `json:"categories"` RefreshInterval int `json:"refresh_interval"` + SkipPreCache bool `json:"skip_pre_cache"` } type Arr struct { @@ -49,7 +50,7 @@ type Arr struct { Token string `json:"token"` Cleanup bool `json:"cleanup"` SkipRepair bool `json:"skip_repair"` - DownloadUncached bool `json:"download_uncached"` + DownloadUncached *bool `json:"download_uncached"` } type Repair struct { diff --git a/pkg/arr/arr.go b/pkg/arr/arr.go index 808b8b7..45ca1ec 100644 --- a/pkg/arr/arr.go +++ b/pkg/arr/arr.go @@ -31,11 +31,11 @@ type Arr struct { Type Type `json:"type"` Cleanup bool `json:"cleanup"` SkipRepair bool `json:"skip_repair"` - DownloadUncached bool `json:"download_uncached"` + DownloadUncached *bool `json:"download_uncached"` client *http.Client } -func New(name, host, token string, cleanup, skipRepair, downloadUncached bool) *Arr { +func New(name, host, token string, cleanup, skipRepair bool, downloadUncached *bool) *Arr { return &Arr{ Name: name, Host: host, diff --git a/pkg/debrid/alldebrid/alldebrid.go b/pkg/debrid/alldebrid/alldebrid.go index fd509c2..a9deea3 100644 --- a/pkg/debrid/alldebrid/alldebrid.go +++ b/pkg/debrid/alldebrid/alldebrid.go @@ -192,7 +192,7 @@ func (ad *AllDebrid) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*tor } break } else if slices.Contains(ad.GetDownloadingStatus(), status) { - if !ad.DownloadUncached && !torrent.DownloadUncached { + if !torrent.DownloadUncached { return torrent, fmt.Errorf("torrent: %s not cached", torrent.Name) } // Break out of the loop if the torrent is downloading. @@ -280,6 +280,10 @@ func (ad *AllDebrid) GetDownloadingStatus() []string { return []string{"downloading"} } +func (ad *AllDebrid) GetDownloadUncached() bool { + return ad.DownloadUncached +} + func New(dc config.Debrid, cache *cache.Cache) *AllDebrid { rl := request.ParseRateLimit(dc.RateLimit) headers := map[string]string{ diff --git a/pkg/debrid/debrid.go b/pkg/debrid/debrid.go index cabda74..e3dc0df 100644 --- a/pkg/debrid/debrid.go +++ b/pkg/debrid/debrid.go @@ -47,14 +47,14 @@ func createDebrid(dc config.Debrid, cache *cache.Cache) engine.Service { } } -func ProcessTorrent(d *engine.Engine, magnet *utils.Magnet, a *arr.Arr, isSymlink, downloadUncached bool) (*torrent.Torrent, error) { +func ProcessTorrent(d *engine.Engine, magnet *utils.Magnet, a *arr.Arr, isSymlink, overrideDownloadUncached bool) (*torrent.Torrent, error) { + debridTorrent := &torrent.Torrent{ - InfoHash: magnet.InfoHash, - Magnet: magnet, - Name: magnet.Name, - Arr: a, - Size: magnet.Size, - DownloadUncached: cmp.Or(downloadUncached, a.DownloadUncached), + InfoHash: magnet.InfoHash, + Magnet: magnet, + Name: magnet.Name, + Arr: a, + Size: magnet.Size, } errs := make([]error, 0) @@ -63,6 +63,17 @@ func ProcessTorrent(d *engine.Engine, magnet *utils.Magnet, a *arr.Arr, isSymlin logger := db.GetLogger() logger.Info().Msgf("Processing debrid: %s", db.GetName()) + // Override first, arr second, debrid third + + if overrideDownloadUncached { + debridTorrent.DownloadUncached = true + } else if a.DownloadUncached != nil { + // Arr cached is set + debridTorrent.DownloadUncached = *a.DownloadUncached + } else { + debridTorrent.DownloadUncached = db.GetDownloadUncached() + } + logger.Info().Msgf("Torrent Hash: %s", debridTorrent.InfoHash) if db.GetCheckCached() { hash, exists := db.IsAvailable([]string{debridTorrent.InfoHash})[debridTorrent.InfoHash] diff --git a/pkg/debrid/debrid_link/debrid_link.go b/pkg/debrid/debrid_link/debrid_link.go index 5cbd447..eac1e09 100644 --- a/pkg/debrid/debrid_link/debrid_link.go +++ b/pkg/debrid/debrid_link/debrid_link.go @@ -217,7 +217,7 @@ func (dl *DebridLink) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*to } break } else if slices.Contains(dl.GetDownloadingStatus(), status) { - if !dl.DownloadUncached && !torrent.DownloadUncached { + if !torrent.DownloadUncached { return torrent, fmt.Errorf("torrent: %s not cached", torrent.Name) } // Break out of the loop if the torrent is downloading. @@ -271,6 +271,10 @@ func (dl *DebridLink) GetCheckCached() bool { return dl.CheckCached } +func (dl *DebridLink) GetDownloadUncached() bool { + return dl.DownloadUncached +} + func New(dc config.Debrid, cache *cache.Cache) *DebridLink { rl := request.ParseRateLimit(dc.RateLimit) headers := map[string]string{ diff --git a/pkg/debrid/engine/service.go b/pkg/debrid/engine/service.go index a670b79..30dcf25 100644 --- a/pkg/debrid/engine/service.go +++ b/pkg/debrid/engine/service.go @@ -13,6 +13,7 @@ type Service interface { DeleteTorrent(tr *torrent.Torrent) IsAvailable(infohashes []string) map[string]bool GetCheckCached() bool + GetDownloadUncached() bool GetTorrent(torrent *torrent.Torrent) (*torrent.Torrent, error) GetTorrents() ([]*torrent.Torrent, error) GetName() string diff --git a/pkg/debrid/realdebrid/realdebrid.go b/pkg/debrid/realdebrid/realdebrid.go index 8d25bf8..74d327b 100644 --- a/pkg/debrid/realdebrid/realdebrid.go +++ b/pkg/debrid/realdebrid/realdebrid.go @@ -249,12 +249,9 @@ func (r *RealDebrid) CheckStatus(t *torrent.Torrent, isSymlink bool) (*torrent.T } break } else if slices.Contains(r.GetDownloadingStatus(), status) { - if !r.DownloadUncached && !t.DownloadUncached { + if !t.DownloadUncached { return t, fmt.Errorf("torrent: %s not cached", t.Name) } - // Break out of the loop if the torrent is downloading. - // This is necessary to prevent infinite loop since we moved to sync downloading and async processing - break } else { return t, fmt.Errorf("torrent: %s has error: %s", t.Name, status) } @@ -379,6 +376,10 @@ func (r *RealDebrid) GetDownloadingStatus() []string { return []string{"downloading", "magnet_conversion", "queued", "compressing", "uploading"} } +func (r *RealDebrid) GetDownloadUncached() bool { + return r.DownloadUncached +} + func New(dc config.Debrid, cache *cache.Cache) *RealDebrid { rl := request.ParseRateLimit(dc.RateLimit) headers := map[string]string{ diff --git a/pkg/debrid/torbox/torbox.go b/pkg/debrid/torbox/torbox.go index b9346ed..f6f9be3 100644 --- a/pkg/debrid/torbox/torbox.go +++ b/pkg/debrid/torbox/torbox.go @@ -231,7 +231,7 @@ func (tb *Torbox) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*torren } break } else if slices.Contains(tb.GetDownloadingStatus(), status) { - if !tb.DownloadUncached && !torrent.DownloadUncached { + if !torrent.DownloadUncached { return torrent, fmt.Errorf("torrent: %s not cached", torrent.Name) } // Break out of the loop if the torrent is downloading. @@ -332,6 +332,10 @@ func (tb *Torbox) GetTorrents() ([]*torrent.Torrent, error) { return nil, nil } +func (tb *Torbox) GetDownloadUncached() bool { + return tb.DownloadUncached +} + func New(dc config.Debrid, cache *cache.Cache) *Torbox { rl := request.ParseRateLimit(dc.RateLimit) headers := map[string]string{ diff --git a/pkg/qbit/downloader.go b/pkg/qbit/downloader.go index c9ae894..d2ecb52 100644 --- a/pkg/qbit/downloader.go +++ b/pkg/qbit/downloader.go @@ -6,6 +6,7 @@ import ( "github.com/cavaliergopher/grab/v3" "github.com/sirrobot01/debrid-blackhole/internal/utils" debrid "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent" + "io" "net/http" "os" "path/filepath" @@ -202,4 +203,56 @@ func (q *QBit) createSymLink(path string, torrentMountPath string, file debrid.F // It's okay if the symlink already exists q.logger.Debug().Msgf("Failed to create symlink: %s: %v", fullPath, err) } + if q.SkipPreCache { + return + } + go func() { + err := q.preCacheFile(torrentFilePath) + if err != nil { + q.logger.Debug().Msgf("Failed to pre-cache file: %s: %v", torrentFilePath, err) + } + }() +} + +func (q *QBit) preCacheFile(filePath string) error { + q.logger.Trace().Msgf("Pre-caching file: %s", filePath) + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("error opening file: %v", err) + } + defer file.Close() + + // Pre-cache the file header (first 256KB) using 16KB chunks. + q.readSmallChunks(file, 0, 256*1024, 16*1024) + q.readSmallChunks(file, 1024*1024, 64*1024, 16*1024) + + return nil +} + +func (q *QBit) readSmallChunks(file *os.File, startPos int64, totalToRead int, chunkSize int) { + _, err := file.Seek(startPos, 0) + if err != nil { + return + } + + buf := make([]byte, chunkSize) + bytesRemaining := totalToRead + + for bytesRemaining > 0 { + toRead := chunkSize + if bytesRemaining < chunkSize { + toRead = bytesRemaining + } + + n, err := file.Read(buf[:toRead]) + if err != nil { + if err == io.EOF { + break + } + return + } + + bytesRemaining -= n + } + return } diff --git a/pkg/qbit/http.go b/pkg/qbit/http.go index d40769a..1764eda 100644 --- a/pkg/qbit/http.go +++ b/pkg/qbit/http.go @@ -59,7 +59,8 @@ func (q *QBit) authContext(next http.Handler) http.Handler { // Check if arr exists a := svc.Arr.Get(category) if a == nil { - a = arr.New(category, "", "", false, false, false) + downloadUncached := false + a = arr.New(category, "", "", false, false, &downloadUncached) } if err == nil { host = strings.TrimSpace(host) diff --git a/pkg/qbit/qbit.go b/pkg/qbit/qbit.go index c7102dd..84726ea 100644 --- a/pkg/qbit/qbit.go +++ b/pkg/qbit/qbit.go @@ -19,6 +19,7 @@ type QBit struct { logger zerolog.Logger Tags []string RefreshInterval int + SkipPreCache bool } func New() *QBit { @@ -35,5 +36,6 @@ func New() *QBit { Storage: NewTorrentStorage(filepath.Join(_cfg.Path, "torrents.json")), logger: logger.NewLogger("qbit", _cfg.LogLevel, os.Stdout), RefreshInterval: refreshInterval, + SkipPreCache: cfg.SkipPreCache, } } diff --git a/pkg/web/server.go b/pkg/web/server.go index 433d481..16dacdc 100644 --- a/pkg/web/server.go +++ b/pkg/web/server.go @@ -311,7 +311,7 @@ func (ui *Handler) handleAddContent(w http.ResponseWriter, r *http.Request) { _arr := svc.Arr.Get(arrName) if _arr == nil { - _arr = arr.New(arrName, "", "", false, false, false) + _arr = arr.New(arrName, "", "", false, false, &downloadUncached) } // Handle URLs