Merge branch 'beta' of github.com:sirrobot01/debrid-blackhole into beta

Signed-off-by: David Young <davidy@funkypenguin.co.nz>

.air.toml

@@ -5,7 +5,7 @@ tmp_dir = "tmp"
 [build]
   args_bin = ["--config", "data/"]
   bin = "./tmp/main"
-  cmd = "bash -c 'go build -ldflags \"-X github.com/sirrobot01/debrid-blackhole/pkg/version.Version=0.0.0 -X github.com/sirrobot01/debrid-blackhole/pkg/version.Channel=nightly\" -o ./tmp/main .'"
+  cmd = "bash -c 'go build -ldflags \"-X github.com/sirrobot01/debrid-blackhole/pkg/version.Version=0.0.0 -X github.com/sirrobot01/debrid-blackhole/pkg/version.Channel=dev\" -o ./tmp/main .'"
   delay = 1000
   exclude_dir = ["assets", "tmp", "vendor", "testdata", "data"]
   exclude_file = []

CHANGELOG.md

@@ -163,4 +163,10 @@
   - Add an option to skip the repair worker for a specific arr
   - Arr specific uncached downloading option
   - Option to download uncached torrents from UI
-- Remove QbitTorrent Log level(Use the global log level)
+- Remove QbitTorrent Log level(Use the global log level)
+
+#### 0.5.1
+
+- Faster import by prefetching newly downloaded torrents
+- Fix UMASK issue due to the docker container
+- Arr-Selective uncached downloading

README.md

@@ -6,23 +6,26 @@ This is an implementation of QbitTorrent with a **Multiple Debrid service suppor
 
 ### Table of Contents
 
+- [DecyphArr(Qbittorent, but with Debrid Support)](#decypharrqbittorent-but-with-debrid-support)
+- [Table of Contents](#table-of-contents)
 - [Features](#features)
 - [Supported Debrid Providers](#supported-debrid-providers)
 - [Installation](#installation)
-  - [Docker Compose](#docker-compose)
-  - [Binary](#binary)
+    - [Docker](#docker)
+      - [Registry](#registry)
+      - [Tags](#tags)
+    - [Binary](#binary)
 - [Usage](#usage)
-- [Connecting to Sonarr/Radarr](#connecting-to-sonarrradarr)
-- [Sample Config](#sample-config)
-- [Config Notes](#config-notes)
-  - [Log Level](#log-level)
-  - [Max Cache Size](#max-cache-size)
-  - [Debrid Config](#debrid-config)
-  - [Proxy Config](#proxy-config)
-  - [Qbittorrent Config](#qbittorrent-config)
-  - [Arrs Config](#arrs-config)
-- [Proxy](#proxy)
+  - [Connecting to Sonarr/Radarr](#connecting-to-sonarrradarr)
+  - [Basic Sample Config](#basic-sample-config)
+    - [Debrid Config](#debrid-config)
+    - [Repair Config (**BETA**)](#repair-config-beta)
+    - [Proxy Config](#proxy-config)
+    - [Qbittorrent Config](#qbittorrent-config)
+    - [Arrs Config](#arrs-config)
 - [Repair Worker](#repair-worker)
+- [Proxy](#proxy)
+  - [**Note**: Proxy has stopped working for Real Debrid, Debrid Link, and All Debrid. It still works for Torbox. This is due to the changes in the API of the Debrid Providers.](#note-proxy-has-stopped-working-for-real-debrid-debrid-link-and-all-debrid-it-still-works-for-torbox-this-is-due-to-the-changes-in-the-api-of-the-debrid-providers)
 - [Changelog](#changelog)
 - [TODO](#todo)
 
@@ -139,7 +142,7 @@ This is the default config file. You can create a `config.json` file in the root
   "qbittorrent": {
     "port": "8282",
     "download_folder": "/mnt/symlinks/",
-    "categories": ["sonarr", "radarr"],
+    "categories": ["sonarr", "radarr"]
   },
   "repair": {
     "enabled": false,

cmd/decypharr/main.go

@@ -12,11 +12,22 @@ import (
 	"github.com/sirrobot01/debrid-blackhole/pkg/version"
 	"github.com/sirrobot01/debrid-blackhole/pkg/web"
 	"github.com/sirrobot01/debrid-blackhole/pkg/worker"
+	"os"
 	"runtime/debug"
+	"strconv"
 	"sync"
 )
 
 func Start(ctx context.Context) error {
+
+	if umaskStr := os.Getenv("UMASK"); umaskStr != "" {
+		umask, err := strconv.ParseInt(umaskStr, 8, 32)
+		if err != nil {
+			return fmt.Errorf("invalid UMASK value: %s", umaskStr)
+		}
+		SetUmask(int(umask))
+	}
+
 	cfg := config.GetConfig()
 	var wg sync.WaitGroup
 	errChan := make(chan error)

cmd/decypharr/umask_unix.go

@@ -0,0 +1,9 @@
+//go:build !windows
+
+package decypharr
+
+import "syscall"
+
+func SetUmask(umask int) {
+	syscall.Umask(umask)
+}

cmd/decypharr/umask_win.go

@@ -0,0 +1,8 @@
+//go:build windows
+// +build windows
+
+package decypharr
+
+func SetUmask(umask int) {
+	// No-op on Windows
+}

doc/config.full.json

@@ -51,11 +51,12 @@
     "download_folder": "/mnt/symlinks/",
     "categories": ["sonarr", "radarr"],
     "refresh_interval": 5,
+    "skip_pre_cache": false
   },
   "arrs": [
     {
       "name": "sonarr",
-      "host": "http://radarr:8989",
+      "host": "http://sonarr:8989",
       "token": "arr_key",
       "cleanup": true,
       "skip_repair": true,
@@ -70,7 +71,7 @@
     },
     {
       "name": "lidarr",
-      "host": "http://lidarr:7878",
+      "host": "http://lidarr:8686",
       "token": "arr_key",
       "cleanup": false,
       "skip_repair": true,

internal/config/config.go

@@ -41,6 +41,7 @@ type QBitTorrent struct {
 	DownloadFolder  string   `json:"download_folder"`
 	Categories      []string `json:"categories"`
 	RefreshInterval int      `json:"refresh_interval"`
+	SkipPreCache    bool     `json:"skip_pre_cache"`
 }
 
 type Arr struct {
@@ -49,7 +50,7 @@ type Arr struct {
 	Token            string `json:"token"`
 	Cleanup          bool   `json:"cleanup"`
 	SkipRepair       bool   `json:"skip_repair"`
-	DownloadUncached bool   `json:"download_uncached"`
+	DownloadUncached *bool  `json:"download_uncached"`
 }
 
 type Repair struct {

pkg/arr/arr.go

@@ -31,11 +31,11 @@ type Arr struct {
 	Type             Type   `json:"type"`
 	Cleanup          bool   `json:"cleanup"`
 	SkipRepair       bool   `json:"skip_repair"`
-	DownloadUncached bool   `json:"download_uncached"`
+	DownloadUncached *bool  `json:"download_uncached"`
 	client           *http.Client
 }
 
-func New(name, host, token string, cleanup, skipRepair, downloadUncached bool) *Arr {
+func New(name, host, token string, cleanup, skipRepair bool, downloadUncached *bool) *Arr {
 	return &Arr{
 		Name:             name,
 		Host:             host,

pkg/debrid/alldebrid/alldebrid.go

@@ -192,7 +192,7 @@ func (ad *AllDebrid) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*tor
 			}
 			break
 		} else if slices.Contains(ad.GetDownloadingStatus(), status) {
-			if !ad.DownloadUncached && !torrent.DownloadUncached {
+			if !torrent.DownloadUncached {
 				return torrent, fmt.Errorf("torrent: %s not cached", torrent.Name)
 			}
 			// Break out of the loop if the torrent is downloading.
@@ -280,6 +280,10 @@ func (ad *AllDebrid) GetDownloadingStatus() []string {
 	return []string{"downloading"}
 }
 
+func (ad *AllDebrid) GetDownloadUncached() bool {
+	return ad.DownloadUncached
+}
+
 func New(dc config.Debrid, cache *cache.Cache) *AllDebrid {
 	rl := request.ParseRateLimit(dc.RateLimit)
 	headers := map[string]string{

pkg/debrid/debrid.go

@@ -47,14 +47,14 @@ func createDebrid(dc config.Debrid, cache *cache.Cache) engine.Service {
 	}
 }
 
-func ProcessTorrent(d *engine.Engine, magnet *utils.Magnet, a *arr.Arr, isSymlink, downloadUncached bool) (*torrent.Torrent, error) {
+func ProcessTorrent(d *engine.Engine, magnet *utils.Magnet, a *arr.Arr, isSymlink, overrideDownloadUncached bool) (*torrent.Torrent, error) {
+
 	debridTorrent := &torrent.Torrent{
-		InfoHash:         magnet.InfoHash,
-		Magnet:           magnet,
-		Name:             magnet.Name,
-		Arr:              a,
-		Size:             magnet.Size,
-		DownloadUncached: cmp.Or(downloadUncached, a.DownloadUncached),
+		InfoHash: magnet.InfoHash,
+		Magnet:   magnet,
+		Name:     magnet.Name,
+		Arr:      a,
+		Size:     magnet.Size,
 	}
 
 	errs := make([]error, 0)
@@ -63,6 +63,17 @@ func ProcessTorrent(d *engine.Engine, magnet *utils.Magnet, a *arr.Arr, isSymlin
 		logger := db.GetLogger()
 		logger.Info().Msgf("Processing debrid: %s", db.GetName())
 
+		// Override first, arr second, debrid third
+
+		if overrideDownloadUncached {
+			debridTorrent.DownloadUncached = true
+		} else if a.DownloadUncached != nil {
+			// Arr cached is set
+			debridTorrent.DownloadUncached = *a.DownloadUncached
+		} else {
+			debridTorrent.DownloadUncached = db.GetDownloadUncached()
+		}
+
 		logger.Info().Msgf("Torrent Hash: %s", debridTorrent.InfoHash)
 		if db.GetCheckCached() {
 			hash, exists := db.IsAvailable([]string{debridTorrent.InfoHash})[debridTorrent.InfoHash]

pkg/debrid/debrid_link/debrid_link.go

@@ -217,7 +217,7 @@ func (dl *DebridLink) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*to
 			}
 			break
 		} else if slices.Contains(dl.GetDownloadingStatus(), status) {
-			if !dl.DownloadUncached && !torrent.DownloadUncached {
+			if !torrent.DownloadUncached {
 				return torrent, fmt.Errorf("torrent: %s not cached", torrent.Name)
 			}
 			// Break out of the loop if the torrent is downloading.
@@ -271,6 +271,10 @@ func (dl *DebridLink) GetCheckCached() bool {
 	return dl.CheckCached
 }
 
+func (dl *DebridLink) GetDownloadUncached() bool {
+	return dl.DownloadUncached
+}
+
 func New(dc config.Debrid, cache *cache.Cache) *DebridLink {
 	rl := request.ParseRateLimit(dc.RateLimit)
 	headers := map[string]string{

pkg/debrid/engine/service.go

@@ -13,6 +13,7 @@ type Service interface {
 	DeleteTorrent(tr *torrent.Torrent)
 	IsAvailable(infohashes []string) map[string]bool
 	GetCheckCached() bool
+	GetDownloadUncached() bool
 	GetTorrent(torrent *torrent.Torrent) (*torrent.Torrent, error)
 	GetTorrents() ([]*torrent.Torrent, error)
 	GetName() string

pkg/debrid/realdebrid/realdebrid.go

@@ -249,11 +249,9 @@ func (r *RealDebrid) CheckStatus(t *torrent.Torrent, isSymlink bool) (*torrent.T
 			}
 			break
 		} else if slices.Contains(r.GetDownloadingStatus(), status) {
-			if !r.DownloadUncached && !t.DownloadUncached {
+			if !t.DownloadUncached {
 				return t, fmt.Errorf("torrent: %s not cached", t.Name)
 			}
-			// Break out of the loop if the torrent is downloading.
-			// This is necessary to prevent infinite loop since we moved to sync downloading and async processing
 			break
 		} else {
 			return t, fmt.Errorf("torrent: %s has error: %s", t.Name, status)
@@ -384,6 +382,10 @@ func (r *RealDebrid) GetDownloadingStatus() []string {
 	return []string{"downloading", "magnet_conversion", "queued", "compressing", "uploading"}
 }
 
+func (r *RealDebrid) GetDownloadUncached() bool {
+	return r.DownloadUncached
+}
+
 func New(dc config.Debrid, cache *cache.Cache) *RealDebrid {
 	rl := request.ParseRateLimit(dc.RateLimit)
 	headers := map[string]string{

pkg/debrid/torbox/torbox.go

@@ -231,7 +231,7 @@ func (tb *Torbox) CheckStatus(torrent *torrent.Torrent, isSymlink bool) (*torren
 			}
 			break
 		} else if slices.Contains(tb.GetDownloadingStatus(), status) {
-			if !tb.DownloadUncached && !torrent.DownloadUncached {
+			if !torrent.DownloadUncached {
 				return torrent, fmt.Errorf("torrent: %s not cached", torrent.Name)
 			}
 			// Break out of the loop if the torrent is downloading.
@@ -332,6 +332,10 @@ func (tb *Torbox) GetTorrents() ([]*torrent.Torrent, error) {
 	return nil, fmt.Errorf("not implemented")
 }
 
+func (tb *Torbox) GetDownloadUncached() bool {
+	return tb.DownloadUncached
+}
+
 func New(dc config.Debrid, cache *cache.Cache) *Torbox {
 	rl := request.ParseRateLimit(dc.RateLimit)
 	headers := map[string]string{

pkg/qbit/downloader.go

@@ -6,6 +6,7 @@ import (
 	"github.com/cavaliergopher/grab/v3"
 	"github.com/sirrobot01/debrid-blackhole/internal/utils"
 	debrid "github.com/sirrobot01/debrid-blackhole/pkg/debrid/torrent"
+	"io"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -202,4 +203,56 @@ func (q *QBit) createSymLink(path string, torrentMountPath string, file debrid.F
 		// It's okay if the symlink already exists
 		q.logger.Debug().Msgf("Failed to create symlink: %s: %v", fullPath, err)
 	}
+	if q.SkipPreCache {
+		return
+	}
+	go func() {
+		err := q.preCacheFile(torrentFilePath)
+		if err != nil {
+			q.logger.Debug().Msgf("Failed to pre-cache file: %s: %v", torrentFilePath, err)
+		}
+	}()
+}
+
+func (q *QBit) preCacheFile(filePath string) error {
+	q.logger.Trace().Msgf("Pre-caching file: %s", filePath)
+	file, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("error opening file: %v", err)
+	}
+	defer file.Close()
+
+	// Pre-cache the file header (first 256KB) using 16KB chunks.
+	q.readSmallChunks(file, 0, 256*1024, 16*1024)
+	q.readSmallChunks(file, 1024*1024, 64*1024, 16*1024)
+
+	return nil
+}
+
+func (q *QBit) readSmallChunks(file *os.File, startPos int64, totalToRead int, chunkSize int) {
+	_, err := file.Seek(startPos, 0)
+	if err != nil {
+		return
+	}
+
+	buf := make([]byte, chunkSize)
+	bytesRemaining := totalToRead
+
+	for bytesRemaining > 0 {
+		toRead := chunkSize
+		if bytesRemaining < chunkSize {
+			toRead = bytesRemaining
+		}
+
+		n, err := file.Read(buf[:toRead])
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return
+		}
+
+		bytesRemaining -= n
+	}
+	return
 }

pkg/qbit/http.go

@@ -59,7 +59,8 @@ func (q *QBit) authContext(next http.Handler) http.Handler {
 		// Check if arr exists
 		a := svc.Arr.Get(category)
 		if a == nil {
-			a = arr.New(category, "", "", false, false, false)
+			downloadUncached := false
+			a = arr.New(category, "", "", false, false, &downloadUncached)
 		}
 		if err == nil {
 			host = strings.TrimSpace(host)

pkg/qbit/qbit.go

@@ -19,6 +19,7 @@ type QBit struct {
 	logger          zerolog.Logger
 	Tags            []string
 	RefreshInterval int
+	SkipPreCache    bool
 }
 
 func New() *QBit {
@@ -35,5 +36,6 @@ func New() *QBit {
 		Storage:         NewTorrentStorage(filepath.Join(_cfg.Path, "torrents.json")),
 		logger:          logger.NewLogger("qbit", _cfg.LogLevel, os.Stdout),
 		RefreshInterval: refreshInterval,
+		SkipPreCache:    cfg.SkipPreCache,
 	}
 }

pkg/web/server.go

@@ -311,7 +311,7 @@ func (ui *Handler) handleAddContent(w http.ResponseWriter, r *http.Request) {
 
 	_arr := svc.Arr.Get(arrName)
 	if _arr == nil {
-		_arr = arr.New(arrName, "", "", false, false, false)
+		_arr = arr.New(arrName, "", "", false, false, &downloadUncached)
 	}
 
 	// Handle URLs