Changelog 0.2.4

Wrap up file downloading feature
Adds Support for Downloader
2024-09-22 16:33:26 +01:00 · 2024-09-22 16:28:31 +01:00 · 2024-09-20 21:09:26 +01:00 · 2024-09-17 00:29:02 +01:00 · 2024-09-15 22:28:07 +01:00
24 changed files with 519 additions and 156 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -54,4 +54,17 @@
 #### 0.2.2
 - Fix name mismatch in the cache
 - Fix directory mapping with mounts
- Add Support for refreshing the *arrs
+- Add Support for refreshing the *arrs
+
+#### 0.2.3
+
+- Delete uncached items from RD
+- Fail if the torrent is not cached(optional)
+- Fix cache not being updated
+
+#### 0.2.4
+
+- Add file download support(Sequential Download)
+- Fix http handler error
+- Fix *arrs map failing concurrently
+- Fix cache not being updated
--- a/1
+++ b/1
@@ -18,6 +18,7 @@ ADD . .
 RUN CGO_ENABLED=0 GOOS=$(echo $TARGETPLATFORM | cut -d '/' -f1) GOARCH=$(echo $TARGETPLATFORM | cut -d '/' -f2) go build -o /blackhole

 FROM scratch
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder /blackhole /blackhole

 EXPOSE 8181
--- a/README.md
+++ b/README.md
@@ -124,6 +124,7 @@ Setting Up Qbittorrent in Arr
  - Password: `sonarr_token` # Your arr token
  - Category: e.g `sonarr`, `radarr`
  - Use SSL -> `No`
+  - Sequential Download -> `No`|`Yes` (If you want to download the torrents locally instead of symlink)
  - Test
  - Save

--- a/common/regex.go
+++ b/common/regex.go
@@ -8,6 +8,7 @@ import (

 var (
 	VIDEOMATCH  = "(?i)(\\.)(YUV|WMV|WEBM|VOB|VIV|SVI|ROQ|RMVB|RM|OGV|OGG|NSV|MXF|MPG|MPEG|M2V|MP2|MPE|MPV|MP4|M4P|M4V|MOV|QT|MNG|MKV|FLV|DRC|AVI|ASF|AMV|MKA|F4V|3GP|3G2|DIVX|X264|X265)$"
+	MUSICMATCH  = "(?i)(\\.)(?:MP3|WAV|FLAC|AAC|OGG|WMA|AIFF|ALAC|M4A|APE|AC3|DTS|M4P|MID|MIDI|MKA|MP2|MPA|RA|VOC|WV|AMR)$"
 	SUBMATCH    = "(?i)(\\.)(SRT|SUB|SBV|ASS|VTT|TTML|DFXP|STL|SCC|CAP|SMI|TTXT|TDS|USF|JSS|SSA|PSB|RT|LRC|SSB)$"
 	SAMPLEMATCH = `(?i)(^|[\\/]|[._-])(sample|trailer|thumb)s?([._-]|$)`
 )
@@ -36,7 +37,7 @@ func RemoveInvalidChars(value string) string {
 }

 func RemoveExtension(value string) string {
-	re := regexp.MustCompile(VIDEOMATCH + "|" + SUBMATCH + "|" + SAMPLEMATCH)
+	re := regexp.MustCompile(VIDEOMATCH + "|" + SUBMATCH + "|" + SAMPLEMATCH + "|" + MUSICMATCH)

 	// Find the last index of the matched extension
 	loc := re.FindStringIndex(value)
--- a/common/utils.go
+++ b/common/utils.go
@@ -217,7 +217,7 @@ func NewLogger(prefix string, output *os.File) *log.Logger {
 func GetInfohashFromURL(url string) (string, error) {
 	// Download the torrent file
 	var magnetLink string
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	client := &http.Client{
 		Timeout: 30 * time.Second,
--- a/go.mod
+++ b/go.mod
@@ -4,10 +4,12 @@ go 1.22

 require (
 	github.com/anacrolix/torrent v1.55.0
+	github.com/cavaliergopher/grab/v3 v3.0.1
 	github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380
 	github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/google/uuid v1.6.0
+	github.com/valyala/fasthttp v1.55.0
 	github.com/valyala/fastjson v1.6.4
 	golang.org/x/time v0.6.0
 )
@@ -15,9 +17,12 @@ require (
 require (
 	github.com/anacrolix/missinggo v1.3.0 // indirect
 	github.com/anacrolix/missinggo/v2 v2.7.3 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -35,6 +35,8 @@ github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pm
 github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8=
 github.com/anacrolix/torrent v1.55.0 h1:s9yh/YGdPmbN9dTa+0Inh2dLdrLQRvEAj1jdFW/Hdd8=
 github.com/anacrolix/torrent v1.55.0/go.mod h1:sBdZHBSZNj4de0m+EbYg7vvs/G/STubxu/GzzNbojsE=
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -44,6 +46,8 @@ github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2w
 github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
+github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4=
+github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -121,6 +125,8 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -187,6 +193,10 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
+github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
 github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
 github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
--- a/pkg/debrid/debrid.go
+++ b/pkg/debrid/debrid.go
@@ -10,8 +10,9 @@ import (

 type Service interface {
 	SubmitMagnet(torrent *Torrent) (*Torrent, error)
-	CheckStatus(torrent *Torrent) (*Torrent, error)
-	DownloadLink(torrent *Torrent) error
+	CheckStatus(torrent *Torrent, isSymlink bool) (*Torrent, error)
+	GetDownloadLinks(torrent *Torrent) error
+	DeleteTorrent(torrent *Torrent)
 	IsAvailable(infohashes []string) map[string]bool
 	GetMountPath() string
 	GetDownloadUncached() bool
@@ -119,7 +120,7 @@ func GetLocalCache(infohashes []string, cache *common.Cache) ([]string, map[stri
 	return hashes, result
 }

-func ProcessQBitTorrent(d Service, magnet *common.Magnet, arr *Arr) (*Torrent, error) {
+func ProcessQBitTorrent(d Service, magnet *common.Magnet, arr *Arr, isSymlink bool) (*Torrent, error) {
 	debridTorrent := &Torrent{
 		InfoHash: magnet.InfoHash,
 		Magnet:   magnet,
@@ -134,7 +135,7 @@ func ProcessQBitTorrent(d Service, magnet *common.Magnet, arr *Arr) (*Torrent, e
 		if !exists || !hash {
 			return debridTorrent, fmt.Errorf("torrent: %s is not cached", debridTorrent.Name)
 		} else {
-			logger.Printf("Torrent: %s is cached", debridTorrent.Name)
+			logger.Printf("Torrent: %s is cached(or downloading)", debridTorrent.Name)
 		}
 	}

@@ -143,5 +144,5 @@ func ProcessQBitTorrent(d Service, magnet *common.Magnet, arr *Arr) (*Torrent, e
 		logger.Printf("Error submitting magnet: %s", err)
 		return nil, err
 	}
-	return d.CheckStatus(debridTorrent)
+	return d.CheckStatus(debridTorrent, isSymlink)
 }
--- a/pkg/debrid/realdebrid.go
+++ b/pkg/debrid/realdebrid.go
@@ -40,7 +40,9 @@ func GetTorrentFiles(data structs.RealDebridTorrentInfo) []TorrentFile {
 	files := make([]TorrentFile, 0)
 	for _, f := range data.Files {
 		name := filepath.Base(f.Path)
-		if (!common.RegexMatch(common.VIDEOMATCH, name) && !common.RegexMatch(common.SUBMATCH, name)) || common.RegexMatch(common.SAMPLEMATCH, name) {
+		if (!common.RegexMatch(common.VIDEOMATCH, name) &&
+			!common.RegexMatch(common.SUBMATCH, name) &&
+			!common.RegexMatch(common.MUSICMATCH, name)) || common.RegexMatch(common.SAMPLEMATCH, name) {
 			continue
 		}
 		fileId := f.ID
@@ -149,12 +151,13 @@ func (r *RealDebrid) GetTorrent(id string) (*Torrent, error) {
 	torrent.Seeders = data.Seeders
 	torrent.Filename = data.Filename
 	torrent.OriginalFilename = data.OriginalFilename
+	torrent.Links = data.Links
 	files := GetTorrentFiles(data)
 	torrent.Files = files
 	return torrent, nil
 }

-func (r *RealDebrid) CheckStatus(torrent *Torrent) (*Torrent, error) {
+func (r *RealDebrid) CheckStatus(torrent *Torrent, isSymlink bool) (*Torrent, error) {
 	url := fmt.Sprintf("%s/torrents/info/%s", r.Host, torrent.Id)
 	for {
 		resp, err := r.client.MakeRequest(http.MethodGet, url, nil)
@@ -174,6 +177,8 @@ func (r *RealDebrid) CheckStatus(torrent *Torrent) (*Torrent, error) {
 		torrent.Progress = data.Progress
 		torrent.Speed = data.Speed
 		torrent.Seeders = data.Seeders
+		torrent.Links = data.Links
+		torrent.Status = status
 		if status == "error" || status == "dead" || status == "magnet_error" {
 			return torrent, fmt.Errorf("torrent: %s has error", torrent.Name)
 		} else if status == "waiting_files_selection" {
@@ -195,11 +200,24 @@ func (r *RealDebrid) CheckStatus(torrent *Torrent) (*Torrent, error) {
 				return torrent, err
 			}
 		} else if status == "downloaded" {
-			log.Printf("Torrent: %s downloaded\n", torrent.Name)
-			err = r.DownloadLink(torrent)
-			if err != nil {
-				return torrent, err
+			files := GetTorrentFiles(data)
+			torrent.Files = files
+			log.Printf("Torrent: %s downloaded to RD\n", torrent.Name)
+			if !isSymlink {
+				err = r.GetDownloadLinks(torrent)
+				if err != nil {
+					return torrent, err
+				}
 			}
+
+			break
+		} else if status == "downloading" {
+			if !r.DownloadUncached {
+				go r.DeleteTorrent(torrent)
+				return torrent, fmt.Errorf("torrent: %s not cached", torrent.Name)
+			}
+			// Break out of the loop if the torrent is downloading.
+			// This is necessary to prevent infinite loop since we moved to sync downloading and async processing
 			break
 		}

@@ -207,7 +225,42 @@ func (r *RealDebrid) CheckStatus(torrent *Torrent) (*Torrent, error) {
 	return torrent, nil
 }

-func (r *RealDebrid) DownloadLink(torrent *Torrent) error {
+func (r *RealDebrid) DeleteTorrent(torrent *Torrent) {
+	url := fmt.Sprintf("%s/torrents/delete/%s", r.Host, torrent.Id)
+	_, err := r.client.MakeRequest(http.MethodDelete, url, nil)
+	if err == nil {
+		r.logger.Printf("Torrent: %s deleted\n", torrent.Name)
+	} else {
+		r.logger.Printf("Error deleting torrent: %s", err)
+	}
+}
+
+func (r *RealDebrid) GetDownloadLinks(torrent *Torrent) error {
+	url := fmt.Sprintf("%s/unrestrict/link/", r.Host)
+	downloadLinks := make([]TorrentDownloadLinks, 0)
+	for _, link := range torrent.Links {
+		if link == "" {
+			continue
+		}
+		payload := gourl.Values{
+			"link": {link},
+		}
+		resp, err := r.client.MakeRequest(http.MethodPost, url, strings.NewReader(payload.Encode()))
+		if err != nil {
+			return err
+		}
+		var data structs.RealDebridUnrestrictResponse
+		if err = json.Unmarshal(resp, &data); err != nil {
+			return err
+		}
+		download := TorrentDownloadLinks{
+			Link:         data.Link,
+			Filename:     data.Filename,
+			DownloadLink: data.Download,
+		}
+		downloadLinks = append(downloadLinks, download)
+	}
+	torrent.DownloadLinks = downloadLinks
 	return nil
 }

--- a/pkg/debrid/structs/realdebrid.go
+++ b/pkg/debrid/structs/realdebrid.go
@@ -93,4 +93,15 @@ type RealDebridTorrentInfo struct {
 	Seeders int      `json:"seeders,omitempty"`
 }

-// 5e6e2e77fd3921a7903a41336c844cc409bf8788/14527C07BDFDDFC642963238BB6E7507B9742947/66A1CD1A5C7F4014877A51AC2620E857E3BB4D16
+type RealDebridUnrestrictResponse struct {
+	Id         string `json:"id"`
+	Filename   string `json:"filename"`
+	MimeType   string `json:"mimeType"`
+	Filesize   int64  `json:"filesize"`
+	Link       string `json:"link"`
+	Host       string `json:"host"`
+	Chunks     int64  `json:"chunks"`
+	Crc        int64  `json:"crc"`
+	Download   string `json:"download"`
+	Streamable int    `json:"streamable"`
+}
--- a/pkg/debrid/torrent.go
+++ b/pkg/debrid/torrent.go
@@ -25,25 +25,33 @@ type ArrHistorySchema struct {
 }

 type Torrent struct {
-	Id               string         `json:"id"`
-	InfoHash         string         `json:"info_hash"`
-	Name             string         `json:"name"`
-	Folder           string         `json:"folder"`
-	Filename         string         `json:"filename"`
-	OriginalFilename string         `json:"original_filename"`
-	Size             int64          `json:"size"`
-	Bytes            int64          `json:"bytes"` // Size of only the files that are downloaded
-	Magnet           *common.Magnet `json:"magnet"`
-	Files            []TorrentFile  `json:"files"`
-	Status           string         `json:"status"`
-	Progress         float64        `json:"progress"`
-	Speed            int64          `json:"speed"`
-	Seeders          int            `json:"seeders"`
+	Id               string                 `json:"id"`
+	InfoHash         string                 `json:"info_hash"`
+	Name             string                 `json:"name"`
+	Folder           string                 `json:"folder"`
+	Filename         string                 `json:"filename"`
+	OriginalFilename string                 `json:"original_filename"`
+	Size             int64                  `json:"size"`
+	Bytes            int64                  `json:"bytes"` // Size of only the files that are downloaded
+	Magnet           *common.Magnet         `json:"magnet"`
+	Files            []TorrentFile          `json:"files"`
+	Status           string                 `json:"status"`
+	Progress         float64                `json:"progress"`
+	Speed            int64                  `json:"speed"`
+	Seeders          int                    `json:"seeders"`
+	Links            []string               `json:"links"`
+	DownloadLinks    []TorrentDownloadLinks `json:"download_links"`

 	Debrid *Debrid
 	Arr    *Arr
 }

+type TorrentDownloadLinks struct {
+	Filename     string `json:"filename"`
+	Link         string `json:"link"`
+	DownloadLink string `json:"download_link"`
+}
+
 func (t *Torrent) GetSymlinkFolder(parent string) string {
 	return filepath.Join(parent, t.Arr.Name, t.Folder)
 }
--- a/pkg/proxy/proxy.go
+++ b/pkg/proxy/proxy.go
@@ -211,13 +211,6 @@ func (item Item) getHash() string {
 	}
 	infohash = hash
 	if infohash == "" {
-		//Get torrent file from http link
-		//Takes too long, not worth it
-		//magnet, err := common.OpenMagnetHttpURL(magnetLink)
-		//if err == nil && magnet != nil && magnet.InfoHash != "" {
-		//	log.Printf("Magnet: %s", magnet.InfoHash)
-		//}
-
 		if strings.Contains(magnetLink, "http") {
 			h, _ := common.GetInfohashFromURL(magnetLink)
 			if h != "" {
@@ -325,9 +318,7 @@ func (p *Proxy) Start() {
 		})
 	}

-	proxy.OnRequest(
-		goproxy.ReqHostMatches(regexp.MustCompile("^.443$")),
-		UrlMatches(regexp.MustCompile("^.*/api\\?t=(search|tvsearch|movie)(&.*)?$"))).HandleConnect(goproxy.AlwaysMitm)
+	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.443$"))).HandleConnect(goproxy.AlwaysMitm)
 	proxy.OnResponse(
 		UrlMatches(regexp.MustCompile("^.*/api\\?t=(search|tvsearch|movie)(&.*)?$")),
 		goproxy.StatusCodeIs(http.StatusOK, http.StatusAccepted)).DoFunc(
--- a/pkg/qbit/arr.go
+++ b/pkg/qbit/arr.go
@@ -40,7 +40,9 @@ func (q *QBit) RefreshArr(arr *debrid.Arr) {
 	if reqErr == nil {
 		statusOk := strconv.Itoa(resp.StatusCode)[0] == '2'
 		if statusOk {
-			q.logger.Printf("Refreshed monitored downloads for %s", cmp.Or(arr.Name, arr.Host))
+			if q.debug {
+				q.logger.Printf("Refreshed monitored downloads for %s", cmp.Or(arr.Name, arr.Host))
+			}
 		}
 	}
 	if reqErr != nil {
--- a/pkg/qbit/downloader.go
+++ b/pkg/qbit/downloader.go
@@ -0,0 +1,134 @@
+package qbit
+
+import (
+	"goBlack/common"
+	"goBlack/pkg/debrid"
+	"goBlack/pkg/qbit/downloaders"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+func (q *QBit) processManualFiles(torrent *Torrent, debridTorrent *debrid.Torrent, arr *debrid.Arr) {
+	q.logger.Printf("Downloading %d files...", len(debridTorrent.DownloadLinks))
+	torrentPath := common.RemoveExtension(debridTorrent.OriginalFilename)
+	parent := common.RemoveInvalidChars(filepath.Join(q.DownloadFolder, debridTorrent.Arr.Name, torrentPath))
+	err := os.MkdirAll(parent, os.ModePerm)
+	if err != nil {
+		q.logger.Printf("Failed to create directory: %s\n", parent)
+		q.MarkAsFailed(torrent)
+		return
+	}
+	torrent.TorrentPath = torrentPath
+	q.downloadFiles(debridTorrent, parent)
+	q.UpdateTorrent(torrent, debridTorrent)
+	q.RefreshArr(arr)
+}
+
+func (q *QBit) downloadFiles(debridTorrent *debrid.Torrent, parent string) {
+	var wg sync.WaitGroup
+	semaphore := make(chan struct{}, 5)
+	client := downloaders.GetHTTPClient()
+	for _, link := range debridTorrent.DownloadLinks {
+		if link.DownloadLink == "" {
+			q.logger.Printf("No download link found for %s\n", link.Filename)
+			continue
+		}
+		wg.Add(1)
+		semaphore <- struct{}{}
+		go func(link debrid.TorrentDownloadLinks) {
+			defer wg.Done()
+			defer func() { <-semaphore }()
+			err := downloaders.NormalHTTP(client, link.DownloadLink, filepath.Join(parent, link.Filename))
+			if err != nil {
+				q.logger.Printf("Error downloading %s: %v\n", link.DownloadLink, err)
+			} else {
+				q.logger.Printf("Downloaded %s successfully\n", link.DownloadLink)
+			}
+		}(link)
+	}
+	wg.Wait()
+	q.logger.Printf("Downloaded all files for %s\n", debridTorrent.Name)
+}
+
+func (q *QBit) processSymlink(torrent *Torrent, debridTorrent *debrid.Torrent, arr *debrid.Arr) {
+	var wg sync.WaitGroup
+	files := debridTorrent.Files
+	ready := make(chan debrid.TorrentFile, len(files))
+
+	q.logger.Printf("Checking %d files...", len(files))
+	rCloneBase := q.debrid.GetMountPath()
+	torrentPath, err := q.getTorrentPath(rCloneBase, debridTorrent) // /MyTVShow/
+	if err != nil {
+		q.MarkAsFailed(torrent)
+		q.logger.Printf("Error: %v", err)
+		return
+	}
+
+	torrentSymlinkPath := filepath.Join(q.DownloadFolder, debridTorrent.Arr.Name, torrentPath) // /mnt/symlinks/{category}/MyTVShow/
+	err = os.MkdirAll(torrentSymlinkPath, os.ModePerm)
+	if err != nil {
+		q.logger.Printf("Failed to create directory: %s\n", torrentSymlinkPath)
+		q.MarkAsFailed(torrent)
+		return
+	}
+	torrentRclonePath := filepath.Join(rCloneBase, torrentPath)
+	for _, file := range files {
+		wg.Add(1)
+		go checkFileLoop(&wg, torrentRclonePath, file, ready)
+	}
+
+	go func() {
+		wg.Wait()
+		close(ready)
+	}()
+
+	for f := range ready {
+		q.logger.Println("File is ready:", f.Path)
+		q.createSymLink(torrentSymlinkPath, torrentRclonePath, f)
+	}
+	// Update the torrent when all files are ready
+	torrent.TorrentPath = filepath.Base(torrentPath) // Quite important
+	q.UpdateTorrent(torrent, debridTorrent)
+	q.RefreshArr(arr)
+}
+
+func (q *QBit) getTorrentPath(rclonePath string, debridTorrent *debrid.Torrent) (string, error) {
+	pathChan := make(chan string)
+	errChan := make(chan error)
+
+	go func() {
+		for {
+			torrentPath := debridTorrent.GetMountFolder(rclonePath)
+			if torrentPath != "" {
+				pathChan <- torrentPath
+				return
+			}
+			time.Sleep(time.Second)
+		}
+	}()
+
+	select {
+	case path := <-pathChan:
+		return path, nil
+	case err := <-errChan:
+		return "", err
+	}
+}
+
+func (q *QBit) createSymLink(path string, torrentMountPath string, file debrid.TorrentFile) {
+
+	// Combine the directory and filename to form a full path
+	fullPath := filepath.Join(path, file.Name) // /mnt/symlinks/{category}/MyTVShow/MyTVShow.S01E01.720p.mkv
+	// Create a symbolic link if file doesn't exist
+	torrentFilePath := filepath.Join(torrentMountPath, file.Name) // debridFolder/MyTVShow/MyTVShow.S01E01.720p.mkv
+	err := os.Symlink(torrentFilePath, fullPath)
+	if err != nil {
+		q.logger.Printf("Failed to create symlink: %s\n", fullPath)
+	}
+	// Check if the file exists
+	if !common.FileReady(fullPath) {
+		q.logger.Printf("Symlink not ready: %s\n", fullPath)
+	}
+}
--- a/pkg/qbit/downloaders/fasthttp.go
+++ b/pkg/qbit/downloaders/fasthttp.go
@@ -0,0 +1,59 @@
+package downloaders
+
+import (
+	"crypto/tls"
+	"fmt"
+	"github.com/valyala/fasthttp"
+	"io"
+	"os"
+)
+
+func GetFastHTTPClient() *fasthttp.Client {
+	return &fasthttp.Client{
+		TLSConfig:          &tls.Config{InsecureSkipVerify: true},
+		StreamResponseBody: true,
+	}
+}
+
+func NormalFastHTTP(client *fasthttp.Client, url, filename string) error {
+	req := fasthttp.AcquireRequest()
+	resp := fasthttp.AcquireResponse()
+	defer fasthttp.ReleaseRequest(req)
+	defer fasthttp.ReleaseResponse(resp)
+
+	req.SetRequestURI(url)
+	req.Header.SetMethod(fasthttp.MethodGet)
+
+	if err := client.Do(req, resp); err != nil {
+		return err
+	}
+
+	// Check the response status code
+	if resp.StatusCode() != fasthttp.StatusOK {
+		return fmt.Errorf("unexpected status code: %d", resp.StatusCode())
+	}
+	file, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer func(file *os.File) {
+		err := file.Close()
+		if err != nil {
+			fmt.Println("Error closing file:", err)
+			return
+		}
+	}(file)
+	bodyStream := resp.BodyStream()
+	if bodyStream == nil {
+		// Write to memory and then to file
+		_, err := file.Write(resp.Body())
+		if err != nil {
+			return err
+		}
+	} else {
+		if _, err := io.Copy(file, bodyStream); err != nil {
+			return err
+		}
+	}
+	return nil
+}
--- a/pkg/qbit/downloaders/grab.go
+++ b/pkg/qbit/downloaders/grab.go
@@ -0,0 +1,55 @@
+package downloaders
+
+import (
+	"crypto/tls"
+	"fmt"
+	"github.com/cavaliergopher/grab/v3"
+	"net/http"
+	"time"
+)
+
+func GetGrabClient() *grab.Client {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		Proxy:           http.ProxyFromEnvironment,
+	}
+	return &grab.Client{
+		UserAgent: "qBitTorrent",
+		HTTPClient: &http.Client{
+			Transport: tr,
+		},
+	}
+}
+
+func NormalGrab(client *grab.Client, url, filename string) error {
+	req, err := grab.NewRequest(filename, url)
+	if err != nil {
+		return err
+	}
+	resp := client.Do(req)
+	if err := resp.Err(); err != nil {
+		return err
+	}
+
+	t := time.NewTicker(2 * time.Second)
+	defer t.Stop()
+Loop:
+	for {
+		select {
+		case <-t.C:
+			fmt.Printf("  %s: transferred %d / %d bytes (%.2f%%)\n",
+				resp.Filename,
+				resp.BytesComplete(),
+				resp.Size,
+				100*resp.Progress())
+
+		case <-resp.Done:
+			// download is complete
+			break Loop
+		}
+	}
+	if err := resp.Err(); err != nil {
+		return err
+	}
+	return nil
+}
--- a/pkg/qbit/downloaders/http.go
+++ b/pkg/qbit/downloaders/http.go
@@ -0,0 +1,44 @@
+package downloaders
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+)
+
+func GetHTTPClient() *http.Client {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	return &http.Client{Transport: tr}
+}
+
+func NormalHTTP(client *http.Client, url, filename string) error {
+	file, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// Send the HTTP GET request
+	resp, err := client.Get(url)
+	if err != nil {
+		fmt.Println("Error downloading file:", err)
+		return err
+	}
+	defer resp.Body.Close()
+
+	// Check server response
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server returned non-200 status: %d %s", resp.StatusCode, resp.Status)
+	}
+
+	// Write the response body to file
+	_, err = io.Copy(file, resp.Body)
+	if err != nil {
+		return err
+	}
+	return nil
+}
--- a/pkg/qbit/handlers_app.go
+++ b/pkg/qbit/handlers_app.go
@@ -7,12 +7,10 @@ import (

 func (q *QBit) handleVersion(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write([]byte("v4.3.2"))
-	w.WriteHeader(http.StatusOK)
 }

 func (q *QBit) handleWebAPIVersion(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write([]byte("2.7"))
-	w.WriteHeader(http.StatusOK)
 }

 func (q *QBit) handlePreferences(w http.ResponseWriter, r *http.Request) {
--- a/pkg/qbit/handlers_torrent.go
+++ b/pkg/qbit/handlers_torrent.go
@@ -1,8 +1,7 @@
 package qbit

 import (
-	"goBlack/common"
-	"io"
+	"context"
 	"net/http"
 	"path/filepath"
 	"strings"
@@ -38,6 +37,8 @@ func (q *QBit) handleTorrentsAdd(w http.ResponseWriter, r *http.Request) {
 		}
 	}

+	isSymlink := strings.ToLower(r.FormValue("sequentialDownload")) != "true"
+	q.logger.Printf("isSymlink: %v\n", isSymlink)
 	urls := r.FormValue("urls")
 	category := r.FormValue("category")

@@ -46,30 +47,24 @@ func (q *QBit) handleTorrentsAdd(w http.ResponseWriter, r *http.Request) {
 		urlList = strings.Split(urls, "\n")
 	}

+	ctx = context.WithValue(ctx, "isSymlink", isSymlink)
+
 	for _, url := range urlList {
-		magnet, err := common.GetMagnetFromUrl(url)
-		if err != nil {
-			q.logger.Printf("Error parsing magnet link: %v\n", err)
+		if err := q.AddMagnet(ctx, url, category); err != nil {
+			q.logger.Printf("Error adding magnet: %v\n", err)
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-		go q.Process(ctx, magnet, category)
-
 	}

 	if contentType == "multipart/form-data" {
 		files := r.MultipartForm.File["torrents"]
 		for _, fileHeader := range files {
-			file, _ := fileHeader.Open()
-			defer file.Close()
-			var reader io.Reader = file
-			magnet, err := common.GetMagnetFromFile(reader, fileHeader.Filename)
-			if err != nil {
+			if err := q.AddTorrent(ctx, fileHeader, category); err != nil {
+				q.logger.Printf("Error adding torrent: %v\n", err)
 				http.Error(w, err.Error(), http.StatusBadRequest)
-				q.logger.Printf("Error reading file: %s", fileHeader.Filename)
 				return
 			}
-			go q.Process(ctx, magnet, category)
 		}
 	}

--- a/pkg/qbit/main.go
+++ b/pkg/qbit/main.go
@@ -11,6 +11,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"sync"
 	"time"
 )

@@ -34,7 +35,7 @@ type QBit struct {
 	storage         *TorrentStorage
 	debug           bool
 	logger          *log.Logger
-	arrs            map[string]string // host:token (Used for refreshing in worker)
+	arrs            sync.Map // host:token (Used for refreshing in worker)
 	RefreshInterval int
 }

@@ -54,7 +55,7 @@ func NewQBit(config *common.Config, deb debrid.Service, cache *common.Cache) *QB
 		debug:           cfg.Debug,
 		storage:         storage,
 		logger:          common.NewLogger("QBit", os.Stdout),
-		arrs:            make(map[string]string),
+		arrs:            sync.Map{},
 		RefreshInterval: refreshInterval,
 	}
 }
@@ -62,7 +63,9 @@ func NewQBit(config *common.Config, deb debrid.Service, cache *common.Cache) *QB
 func (q *QBit) Start() {

 	r := chi.NewRouter()
-	r.Use(middleware.Logger)
+	if q.debug {
+		r.Use(middleware.Logger)
+	}
 	r.Use(middleware.Recoverer)

 	q.AddRoutes(r)
--- a/pkg/qbit/middleware.go
+++ b/pkg/qbit/middleware.go
@@ -52,7 +52,7 @@ func (q *QBit) authContext(next http.Handler) http.Handler {
 		if err == nil {
 			ctx = context.WithValue(r.Context(), "host", host)
 			ctx = context.WithValue(ctx, "token", token)
-			q.arrs[host] = token
+			q.arrs.Store(host, token)
 			next.ServeHTTP(w, r.WithContext(ctx))
 			return
 		}
--- a/pkg/qbit/qbit.go
+++ b/pkg/qbit/qbit.go
@@ -2,36 +2,68 @@ package qbit

 import (
 	"context"
+	"fmt"
 	"github.com/google/uuid"
 	"goBlack/common"
 	"goBlack/pkg/debrid"
-	"os"
-	"path/filepath"
+	"io"
+	"mime/multipart"
 	"strings"
-	"sync"
 	"time"
 )

-func (q *QBit) Process(ctx context.Context, magnet *common.Magnet, category string) (*Torrent, error) {
+func (q *QBit) AddMagnet(ctx context.Context, url, category string) error {
+	magnet, err := common.GetMagnetFromUrl(url)
+	if err != nil {
+		q.logger.Printf("Error parsing magnet link: %v\n", err)
+		return err
+	}
+	err = q.Process(ctx, magnet, category)
+	if err != nil {
+		q.logger.Println("Failed to process magnet:", err)
+		return err
+	}
+	return nil
+}
+
+func (q *QBit) AddTorrent(ctx context.Context, fileHeader *multipart.FileHeader, category string) error {
+	file, _ := fileHeader.Open()
+	defer file.Close()
+	var reader io.Reader = file
+	magnet, err := common.GetMagnetFromFile(reader, fileHeader.Filename)
+	if err != nil {
+		q.logger.Printf("Error reading file: %s", fileHeader.Filename)
+		return err
+	}
+	err = q.Process(ctx, magnet, category)
+	if err != nil {
+		q.logger.Println("Failed to process torrent:", err)
+		return err
+	}
+	return nil
+}
+
+func (q *QBit) Process(ctx context.Context, magnet *common.Magnet, category string) error {
 	torrent := q.CreateTorrentFromMagnet(magnet, category)
-	go q.storage.AddOrUpdate(torrent)
 	arr := &debrid.Arr{
 		Name:  category,
 		Token: ctx.Value("token").(string),
 		Host:  ctx.Value("host").(string),
 	}
-	debridTorrent, err := debrid.ProcessQBitTorrent(q.debrid, magnet, arr)
+	isSymlink := ctx.Value("isSymlink").(bool)
+	debridTorrent, err := debrid.ProcessQBitTorrent(q.debrid, magnet, arr, isSymlink)
 	if err != nil || debridTorrent == nil {
-		// Mark as failed
-		q.logger.Printf("Failed to process torrent: %s: %v", magnet.Name, err)
-		q.MarkAsFailed(torrent)
-		return torrent, err
+		if err == nil {
+			err = fmt.Errorf("failed to process torrent")
+		}
+		return err
 	}
 	torrent.ID = debridTorrent.Id
 	torrent.DebridTorrent = debridTorrent
 	torrent.Name = debridTorrent.Name
-	q.processFiles(torrent, debridTorrent, arr)
-	return torrent, nil
+	q.storage.AddOrUpdate(torrent)
+	go q.processFiles(torrent, debridTorrent, arr, isSymlink) // We can send async for file processing not to delay the response
+	return nil
 }

 func (q *QBit) CreateTorrentFromMagnet(magnet *common.Magnet, category string) *Torrent {
@@ -63,81 +95,22 @@ func (q *QBit) CreateTorrentFromMagnet(magnet *common.Magnet, category string) *
 	return torrent
 }

-func (q *QBit) processFiles(torrent *Torrent, debridTorrent *debrid.Torrent, arr *debrid.Arr) {
-	var wg sync.WaitGroup
-	files := debridTorrent.Files
-	ready := make(chan debrid.TorrentFile, len(files))
-
-	q.logger.Printf("Checking %d files...", len(files))
-	rCloneBase := q.debrid.GetMountPath()
-	torrentPath, err := q.getTorrentPath(rCloneBase, debridTorrent) // /MyTVShow/
-	if err != nil {
-		q.logger.Printf("Error: %v", err)
-		return
-	}
-
-	torrentSymlinkPath := filepath.Join(q.DownloadFolder, debridTorrent.Arr.Name, torrentPath) // /mnt/symlinks/{category}/MyTVShow/
-	err = os.MkdirAll(torrentSymlinkPath, os.ModePerm)
-	if err != nil {
-		q.logger.Printf("Failed to create directory: %s\n", torrentSymlinkPath)
-		return
-	}
-	torrentRclonePath := filepath.Join(rCloneBase, torrentPath)
-	for _, file := range files {
-		wg.Add(1)
-		go checkFileLoop(&wg, torrentRclonePath, file, ready)
-	}
-
-	go func() {
-		wg.Wait()
-		close(ready)
-	}()
-
-	for f := range ready {
-		q.logger.Println("File is ready:", f.Path)
-		q.createSymLink(torrentSymlinkPath, torrentRclonePath, f)
-	}
-	// Update the torrent when all files are ready
-	torrent.TorrentPath = filepath.Base(torrentPath) // Quite important
-	q.UpdateTorrent(torrent, debridTorrent)
-	q.RefreshArr(arr)
-}
-
-func (q *QBit) getTorrentPath(rclonePath string, debridTorrent *debrid.Torrent) (string, error) {
-	pathChan := make(chan string)
-	errChan := make(chan error)
-
-	go func() {
-		for {
-			torrentPath := debridTorrent.GetMountFolder(rclonePath)
-			if torrentPath != "" {
-				pathChan <- torrentPath
-				return
-			}
-			time.Sleep(time.Second)
+func (q *QBit) processFiles(torrent *Torrent, debridTorrent *debrid.Torrent, arr *debrid.Arr, isSymlink bool) {
+	for debridTorrent.Status != "downloaded" {
+		progress := debridTorrent.Progress
+		q.logger.Printf("Progress: %.2f%%", progress)
+		time.Sleep(5 * time.Second)
+		dbT, err := q.debrid.CheckStatus(debridTorrent, isSymlink)
+		if err != nil {
+			q.logger.Printf("Error checking status: %v", err)
+			q.MarkAsFailed(torrent)
+			return
 		}
-	}()
-
-	select {
-	case path := <-pathChan:
-		return path, nil
-	case err := <-errChan:
-		return "", err
-	}
-}
-
-func (q *QBit) createSymLink(path string, torrentMountPath string, file debrid.TorrentFile) {
-
-	// Combine the directory and filename to form a full path
-	fullPath := filepath.Join(path, file.Name) // /mnt/symlinks/{category}/MyTVShow/MyTVShow.S01E01.720p.mkv
-	// Create a symbolic link if file doesn't exist
-	torrentFilePath := filepath.Join(torrentMountPath, file.Name) // debridFolder/MyTVShow/MyTVShow.S01E01.720p.mkv
-	err := os.Symlink(torrentFilePath, fullPath)
-	if err != nil {
-		q.logger.Printf("Failed to create symlink: %s\n", fullPath)
-	}
-	// Check if the file exists
-	if !common.FileReady(fullPath) {
-		q.logger.Printf("Symlink not ready: %s\n", fullPath)
+		debridTorrent = dbT
+	}
+	if isSymlink {
+		q.processSymlink(torrent, debridTorrent, arr)
+	} else {
+		q.processManualFiles(torrent, debridTorrent, arr)
 	}
 }
--- a/pkg/qbit/storage.go
+++ b/pkg/qbit/storage.go
@@ -77,7 +77,7 @@ func (ts *TorrentStorage) Get(hash string) *Torrent {
 func (ts *TorrentStorage) GetAll(category string, filter string, hashes []string) []*Torrent {
 	ts.mu.RLock()
 	defer ts.mu.RUnlock()
-	torrents := make([]*Torrent, 0, len(ts.torrents))
+	torrents := make([]*Torrent, 0)
 	for _, id := range ts.order {
 		torrent := ts.torrents[id]
 		if category != "" && torrent.Category != category {
--- a/pkg/qbit/worker.go
+++ b/pkg/qbit/worker.go
@@ -20,22 +20,27 @@ func (q *QBit) StartRefreshWorker(ctx context.Context) {
 			q.logger.Println("Qbit Refresh Worker stopped")
 			return
 		case <-refreshTicker.C:
-			q.RefreshArrs()
+			torrents := q.storage.GetAll("", "", nil)
+			if len(torrents) > 0 {
+				q.RefreshArrs()
+			}
 		}
 	}
 }

 func (q *QBit) RefreshArrs() {
-	torrents := q.storage.GetAll("", "", nil)
-	if len(torrents) == 0 {
-		return
-	}
-	for host, token := range q.arrs {
+	q.arrs.Range(func(key, value interface{}) bool {
+		host, ok := key.(string)
+		token, ok2 := value.(string)
+		if !ok || !ok2 {
+			return true
+		}
 		arr := &debrid.Arr{
 			Name:  "",
 			Token: token,
 			Host:  host,
 		}
 		q.RefreshArr(arr)
-	}
+		return true
+	})
 }
Author	SHA1	Message	Date
Mukhtar Akere	f6c6144601	Changelog 0.2.4	2024-09-22 16:33:26 +01:00
Mukhtar Akere	ff74e279d9	Wrap up file downloading feature	2024-09-22 16:28:31 +01:00
Mukhtar Akere	ba147ac56c	Adds Support for Downloader	2024-09-20 21:09:26 +01:00
Mukhtar Akere	01981114cb	Changelog 0.2.3	2024-09-17 00:29:02 +01:00
Mukhtar Akere	2ec0354881	Hotfix: Download Uncached	2024-09-15 22:28:07 +01:00