Changelog: 0.1.4

- Make Blackhole and Proxy optional
- Query all Hashes at once, speeeding up the process
- refractor the code a little bit

.goreleaser.yaml

@@ -1,11 +1,3 @@
-# This is an example .goreleaser.yml file with some sensible defaults.
-# Make sure to check the documentation at https://goreleaser.com
-
-# The lines below are called `modelines`. See `:help modeline`
-# Feel free to remove those if you don't want/need to use them.
-# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
-# vim: set ts=2 sw=2 tw=0 fo=cnqoj
-
 version: 2
 
 before:
@@ -20,6 +12,11 @@ builds:
       - linux
       - windows
       - darwin
+    goarch:
+      - amd64
+      - arm
+      - arm64
+
 
 archives:
   - format: tar.gz
@@ -42,3 +39,7 @@ changelog:
     exclude:
       - "^docs:"
       - "^test:"
+
+# Environment setup
+env:
+  - CGO_ENABLED=0

CHANGELOG.md

@@ -0,0 +1,34 @@
+#### 0.1.0
+- Initial Release
+- Added Real Debrid Support
+- Added Arrs Support
+- Added Proxy Support
+- Added Basic Authentication for Proxy
+- Added Rate Limiting for Debrid Providers
+
+#### 0.1.1
+- Added support for "No Blackhole" for Arrs
+- Added support for "Cached Only" for Proxy
+- Bug Fixes
+
+#### 0.1.2
+- Bug fixes
+- Code cleanup
+- Get available hashes at once
+
+#### 0.1.3
+
+- Searching for infohashes in the xml description/summary/comments
+- Added local cache support
+- Added max cache size
+- Rewrite blackhole.go
+- Bug fixes
+  - Fixed indexer getting disabled
+  - Fixed blackhole not working
+
+#### 0.1.4
+
+- Rewrote Report log
+- Fix YTS, 1337x not grabbing infohash
+- Fix Torrent symlink bug
+- 

Dockerfile

@@ -20,5 +20,7 @@ RUN CGO_ENABLED=0 GOOS=$(echo $TARGETPLATFORM | cut -d '/' -f1) GOARCH=$(echo $T
 FROM scratch
 COPY --from=builder /blackhole /blackhole
 
+EXPOSE 8181
+
 # Run
 CMD ["/blackhole", "--config", "/app/config.json"]

README.md

@@ -3,11 +3,15 @@
 This is a Golang implementation go Torrent Blackhole with a **Real Debrid Proxy Support**.
 
 #### Uses
-- Torrent Blackhole that supports the Arrs.
+- Torrent Blackhole that supports the Arrs(Sonarr, Radarr, etc)
 - Proxy support for the Arrs
 
 The proxy is useful in filtering out un-cached Real Debrid torrents
 
+### Changelog
+
+- View the [CHANGELOG.md](CHANGELOG.md) for the latest changes
+
 
 #### Installation
 ##### Docker Compose
@@ -15,7 +19,7 @@ The proxy is useful in filtering out un-cached Real Debrid torrents
 version: '3.7'
 services:
   blackhole:
-    image: cy01/blackhole:latest
+    image: cy01/blackhole:latest # or cy01/blackhole:beta
     container_name: blackhole
     user: "1000:1000"
     volumes:
@@ -73,12 +77,31 @@ Download the binary from the releases page and run it with the config file.
     "port": "8181",
     "debug": false,
     "username": "username",
-    "password": "password"
-  }
+    "password": "password",
+    "cached_only": true
+  },
+  "max_cache_size": 1000
 }
 ```
 
-#### Proxy
+#### Config Notes
+##### Debrid Config
+- This config key is important as it's used for both Blackhole and Proxy
+
+##### Arrs Config
+- An empty array will disable Blackhole for the Arrs
+- The `watch_folder` is the folder where the Blackhole will watch for torrents
+- The `completed_folder` is the folder where the Blackhole will move the completed torrents
+- The `token` is the API key for the Arr(This is optional, I think)
+
+##### Proxy Config
+- The `enabled` key is used to enable the proxy
+- The `port` key is the port the proxy will listen on
+- The `debug` key is used to enable debug logs
+- The `username` and `password` keys are used for basic authentication
+- The `cached_only` means only cached torrents will be returned
+- 
+### Proxy
 
 The proxy is useful in filtering out un-cached Real Debrid torrents. 
 The proxy is a simple HTTP proxy that requires basic authentication. The proxy can be enabled by setting the `proxy.enabled` to `true` in the config file. 

cmd/blackhole.go

@@ -12,6 +12,21 @@ import (
 	"time"
 )
 
+type Blackhole struct {
+	config *common.Config
+	deb    debrid.Service
+	cache  *common.Cache
+}
+
+func NewBlackhole(config *common.Config, deb debrid.Service, cache *common.Cache) *Blackhole {
+	return &Blackhole{
+		config: config,
+		deb:    deb,
+		cache:  cache,
+	}
+
+}
+
 func fileReady(path string) bool {
 	_, err := os.Stat(path)
 	return !os.IsNotExist(err) // Returns true if the file exists
@@ -33,12 +48,12 @@ func checkFileLoop(wg *sync.WaitGroup, dir string, file debrid.TorrentFile, read
 	}
 }
 
-func ProcessFiles(arr *debrid.Arr, torrent *debrid.Torrent) {
+func (b *Blackhole) processFiles(arr *debrid.Arr, torrent *debrid.Torrent) {
 	var wg sync.WaitGroup
 	files := torrent.Files
 	ready := make(chan debrid.TorrentFile, len(files))
 
-	log.Println("Checking files...")
+	log.Printf("Checking %d files...", len(files))
 
 	for _, file := range files {
 		wg.Add(1)
@@ -52,29 +67,30 @@ func ProcessFiles(arr *debrid.Arr, torrent *debrid.Torrent) {
 
 	for r := range ready {
 		log.Println("File is ready:", r.Name)
-		CreateSymLink(arr, torrent)
+		b.createSymLink(arr, torrent)
 
 	}
 	go torrent.Cleanup(true)
 	fmt.Printf("%s downloaded", torrent.Name)
 }
 
-func CreateSymLink(config *debrid.Arr, torrent *debrid.Torrent) {
-	path := filepath.Join(config.CompletedFolder, torrent.Folder)
+func (b *Blackhole) createSymLink(arr *debrid.Arr, torrent *debrid.Torrent) {
+	path := filepath.Join(arr.CompletedFolder, torrent.Folder)
 	err := os.MkdirAll(path, os.ModePerm)
 	if err != nil {
 		log.Printf("Failed to create directory: %s\n", path)
 	}
+
 	for _, file := range torrent.Files {
 		// Combine the directory and filename to form a full path
-		fullPath := filepath.Join(config.CompletedFolder, file.Path)
-
+		fullPath := filepath.Join(path, file.Name) // completedFolder/MyTVShow/MyTVShow.S01E01.720p.mkv
 		// Create a symbolic link if file doesn't exist
-		_ = os.Symlink(filepath.Join(config.Debrid.Folder, file.Path), fullPath)
+		torrentPath := filepath.Join(arr.Debrid.Folder, torrent.Folder, file.Name) // debridFolder/MyTVShow/MyTVShow.S01E01.720p.mkv
+		_ = os.Symlink(torrentPath, fullPath)
 	}
 }
 
-func watchFiles(watcher *fsnotify.Watcher, events map[string]time.Time) {
+func watcher(watcher *fsnotify.Watcher, events map[string]time.Time) {
 	for {
 		select {
 		case event, ok := <-watcher.Events:
@@ -96,7 +112,7 @@ func watchFiles(watcher *fsnotify.Watcher, events map[string]time.Time) {
 	}
 }
 
-func processFilesDebounced(arr *debrid.Arr, db debrid.Service, events map[string]time.Time, debouncePeriod time.Duration) {
+func (b *Blackhole) processFilesDebounced(arr *debrid.Arr, events map[string]time.Time, debouncePeriod time.Duration) {
 	ticker := time.NewTicker(1 * time.Second) // Check every second
 	defer ticker.Stop()
 
@@ -105,7 +121,7 @@ func processFilesDebounced(arr *debrid.Arr, db debrid.Service, events map[string
 			if time.Since(lastEventTime) >= debouncePeriod {
 				log.Printf("Torrent file detected: %s", file)
 				// Process the torrent file
-				torrent, err := db.Process(arr, file)
+				torrent, err := b.deb.Process(arr, file)
 				if err != nil && torrent != nil {
 					// remove torrent file
 					torrent.Cleanup(true)
@@ -113,7 +129,7 @@ func processFilesDebounced(arr *debrid.Arr, db debrid.Service, events map[string
 					log.Printf("Error processing torrent file: %s", err)
 				}
 				if err == nil && torrent != nil && len(torrent.Files) > 0 {
-					go ProcessFiles(arr, torrent)
+					go b.processFiles(arr, torrent)
 				}
 				delete(events, file) // remove file from channel
 
@@ -122,32 +138,33 @@ func processFilesDebounced(arr *debrid.Arr, db debrid.Service, events map[string
 	}
 }
 
-func StartArr(conf *debrid.Arr, db debrid.Service) {
-	log.Printf("Watching: %s", conf.WatchFolder)
+func (b *Blackhole) startArr(arr *debrid.Arr) {
+	log.Printf("Watching: %s", arr.WatchFolder)
 	w, err := fsnotify.NewWatcher()
 	if err != nil {
-		log.Fatal(err)
+		log.Println(err)
 	}
 	defer func(w *fsnotify.Watcher) {
 		err := w.Close()
 		if err != nil {
-			log.Fatal(err)
+			log.Println(err)
 		}
 	}(w)
 	events := make(map[string]time.Time)
 
-	go watchFiles(w, events)
-	if err = w.Add(conf.WatchFolder); err != nil {
+	go watcher(w, events)
+	if err = w.Add(arr.WatchFolder); err != nil {
 		log.Println("Error Watching folder:", err)
 		return
 	}
 
-	processFilesDebounced(conf, db, events, 1*time.Second)
+	b.processFilesDebounced(arr, events, 1*time.Second)
 }
 
-func StartBlackhole(config *common.Config, deb debrid.Service) {
+func (b *Blackhole) Start() {
+	log.Println("[*] Starting Blackhole")
 	var wg sync.WaitGroup
-	for _, conf := range config.Arrs {
+	for _, conf := range b.config.Arrs {
 		wg.Add(1)
 		defer wg.Done()
 		headers := map[string]string{
@@ -156,14 +173,14 @@ func StartBlackhole(config *common.Config, deb debrid.Service) {
 		client := common.NewRLHTTPClient(nil, headers)
 
 		arr := &debrid.Arr{
-			Debrid:          config.Debrid,
+			Debrid:          b.config.Debrid,
 			WatchFolder:     conf.WatchFolder,
 			CompletedFolder: conf.CompletedFolder,
 			Token:           conf.Token,
 			URL:             conf.URL,
 			Client:          client,
 		}
-		go StartArr(arr, deb)
+		go b.startArr(arr)
 	}
 	wg.Wait()
 }

cmd/main.go

@@ -1,18 +1,39 @@
 package cmd
 
 import (
+	"cmp"
 	"goBlack/common"
 	"goBlack/debrid"
-	"log"
+	"sync"
 )
 
 func Start(config *common.Config) {
+	maxCacheSize := cmp.Or(config.MaxCacheSize, 1000)
+	cache := common.NewCache(maxCacheSize)
+	
+	deb := debrid.NewDebrid(config.Debrid, cache)
+
+	var wg sync.WaitGroup
 
-	log.Print("[*] BlackHole running")
-	deb := debrid.NewDebrid(config.Debrid)
 	if config.Proxy.Enabled {
-		go StartProxy(config, deb)
+		proxy := NewProxy(*config, deb, cache)
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			proxy.Start()
+		}()
 	}
-	StartBlackhole(config, deb)
+
+	if len(config.Arrs) > 0 {
+		blackhole := NewBlackhole(config, deb, cache)
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			blackhole.Start()
+		}()
+	}
+
+	// Wait indefinitely
+	wg.Wait()
 
 }

cmd/proxy.go

@@ -13,6 +13,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"os"
 	"regexp"
 	"strings"
 	"sync"
@@ -20,80 +21,83 @@ import (
 
 type RSS struct {
 	XMLName xml.Name `xml:"rss"`
+	Text    string   `xml:",chardata"`
 	Version string   `xml:"version,attr"`
-	Channel Channel  `xml:"channel"`
-}
-
-type Channel struct {
-	XMLName  xml.Name `xml:"channel"`
-	Title    string   `xml:"title"`
-	AtomLink AtomLink `xml:"link"`
-	Items    []Item   `xml:"item"`
-}
-
-type AtomLink struct {
-	XMLName xml.Name `xml:"link"`
-	Rel     string   `xml:"rel,attr"`
-	Type    string   `xml:"type,attr"`
+	Atom    string   `xml:"atom,attr"`
+	Torznab string   `xml:"torznab,attr"`
+	Channel struct {
+		Text string `xml:",chardata"`
+		Link struct {
+			Text string `xml:",chardata"`
+			Rel  string `xml:"rel,attr"`
+			Type string `xml:"type,attr"`
+		} `xml:"link"`
+		Title string `xml:"title"`
+		Items []Item `xml:"item"`
+	} `xml:"channel"`
 }
 
 type Item struct {
-	XMLName         xml.Name        `xml:"item"`
-	Title           string          `xml:"title"`
-	Description     string          `xml:"description"`
-	GUID            string          `xml:"guid"`
-	ProwlarrIndexer ProwlarrIndexer `xml:"prowlarrindexer"`
-	Comments        string          `xml:"comments"`
-	PubDate         string          `xml:"pubDate"`
-	Size            int64           `xml:"size"`
-	Link            string          `xml:"link"`
-	Categories      []string        `xml:"category"`
-	Enclosure       Enclosure       `xml:"enclosure"`
-	TorznabAttrs    []TorznabAttr   `xml:"torznab:attr"`
+	Text            string `xml:",chardata"`
+	Title           string `xml:"title"`
+	Description     string `xml:"description"`
+	GUID            string `xml:"guid"`
+	ProwlarrIndexer struct {
+		Text string `xml:",chardata"`
+		ID   string `xml:"id,attr"`
+		Type string `xml:"type,attr"`
+	} `xml:"prowlarrindexer"`
+	Comments  string   `xml:"comments"`
+	PubDate   string   `xml:"pubDate"`
+	Size      string   `xml:"size"`
+	Link      string   `xml:"link"`
+	Category  []string `xml:"category"`
+	Enclosure struct {
+		Text   string `xml:",chardata"`
+		URL    string `xml:"url,attr"`
+		Length string `xml:"length,attr"`
+		Type   string `xml:"type,attr"`
+	} `xml:"enclosure"`
+	TorznabAttrs []struct {
+		Text  string `xml:",chardata"`
+		Name  string `xml:"name,attr"`
+		Value string `xml:"value,attr"`
+	} `xml:"attr"`
 }
 
-type ProwlarrIndexer struct {
-	ID    string `xml:"id,attr"`
-	Type  string `xml:"type,attr"`
-	Value string `xml:",chardata"`
+type Proxy struct {
+	port       string
+	enabled    bool
+	debug      bool
+	username   string
+	password   string
+	cachedOnly bool
+	debrid     debrid.Service
+	cache      *common.Cache
 }
 
-type Enclosure struct {
-	URL    string `xml:"url,attr"`
-	Length int64  `xml:"length,attr"`
-	Type   string `xml:"type,attr"`
+func NewProxy(config common.Config, deb debrid.Service, cache *common.Cache) *Proxy {
+	cfg := config.Proxy
+	port := cmp.Or(os.Getenv("PORT"), cfg.Port, "8181")
+	return &Proxy{
+		port:       port,
+		enabled:    cfg.Enabled,
+		debug:      cfg.Debug,
+		username:   cfg.Username,
+		password:   cfg.Password,
+		cachedOnly: cfg.CachedOnly,
+		debrid:     deb,
+		cache:      cache,
+	}
 }
 
-type TorznabAttr struct {
-	Name  string `xml:"name,attr"`
-	Value string `xml:"value,attr"`
-}
-
-type SafeItems struct {
-	mu    sync.Mutex
-	Items []Item
-}
-
-func (s *SafeItems) Add(item Item) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.Items = append(s.Items, item)
-}
-
-func (s *SafeItems) Get() []Item {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return s.Items
-}
-
-func ProcessJSONResponse(resp *http.Response, deb debrid.Service) *http.Response {
+func (p *Proxy) ProcessJSONResponse(resp *http.Response) *http.Response {
 	if resp == nil || resp.Body == nil {
 		return resp
 	}
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		log.Println("Error reading response body:", err)
 		return resp
 	}
 	err = resp.Body.Close()
@@ -101,8 +105,8 @@ func ProcessJSONResponse(resp *http.Response, deb debrid.Service) *http.Response
 		return nil
 	}
 
-	var p fastjson.Parser
-	v, err := p.ParseBytes(body)
+	var par fastjson.Parser
+	v, err := par.ParseBytes(body)
 	if err != nil {
 		// If it's not JSON, return the original response
 		resp.Body = io.NopCloser(bytes.NewReader(body))
@@ -123,69 +127,100 @@ func ProcessJSONResponse(resp *http.Response, deb debrid.Service) *http.Response
 
 }
 
-func ProcessResponse(resp *http.Response, deb debrid.Service) *http.Response {
+func (p *Proxy) ProcessResponse(resp *http.Response) *http.Response {
 	if resp == nil || resp.Body == nil {
 		return resp
 	}
 	contentType := resp.Header.Get("Content-Type")
 	switch contentType {
 	case "application/json":
-		return ProcessJSONResponse(resp, deb)
+		return resp // p.ProcessJSONResponse(resp)
 	case "application/xml":
-		return ProcessXMLResponse(resp, deb)
+		return p.ProcessXMLResponse(resp)
 	case "application/rss+xml":
-		return ProcessXMLResponse(resp, deb)
+		return p.ProcessXMLResponse(resp)
 	default:
 		return resp
 	}
 }
 
-func XMLItemIsCached(item Item, deb debrid.Service) bool {
-	magnetLink := ""
+func getItemsHash(items []Item) map[string]string {
+
+	var wg sync.WaitGroup
+	idHashMap := sync.Map{} // Use sync.Map for concurrent access
+
+	for _, item := range items {
+		wg.Add(1)
+		go func(item Item) {
+			defer wg.Done()
+			hash := strings.ToLower(item.getHash())
+			if hash != "" {
+				idHashMap.Store(item.GUID, hash) // Store directly into sync.Map
+			}
+		}(item)
+	}
+	wg.Wait()
+
+	// Convert sync.Map to regular map
+	finalMap := make(map[string]string)
+	idHashMap.Range(func(key, value interface{}) bool {
+		finalMap[key.(string)] = value.(string)
+		return true
+	})
+
+	return finalMap
+}
+
+func (item Item) getHash() string {
 	infohash := ""
 
-	// Extract magnet link from the link or comments
-	if strings.Contains(item.Link, "magnet:?") {
-		magnetLink = item.Link
-	} else if strings.Contains(item.GUID, "magnet:?") {
-		magnetLink = item.GUID
-	}
-
-	// Extract infohash from <torznab:attr> elements
 	for _, attr := range item.TorznabAttrs {
 		if attr.Name == "infohash" {
-			infohash = attr.Value
+			return attr.Value
 		}
 	}
-	if magnetLink == "" && infohash == "" {
-		// We can't check the availability of the torrent without a magnet link or infohash
-		return false
-	}
-	var magnet *common.Magnet
-	var err error
 
+	if strings.Contains(item.GUID, "magnet:?") {
+		magnet, err := common.GetMagnetInfo(item.GUID)
+		if err == nil && magnet != nil && magnet.InfoHash != "" {
+			return magnet.InfoHash
+		}
+	}
+
+	magnetLink := item.Link
+
+	if magnetLink == "" {
+		// We can't check the availability of the torrent without a magnet link or infohash
+		return ""
+	}
+
+	if strings.Contains(magnetLink, "magnet:?") {
+		magnet, err := common.GetMagnetInfo(magnetLink)
+		if err == nil && magnet != nil && magnet.InfoHash != "" {
+			return magnet.InfoHash
+		}
+	}
+
+	//Check Description for infohash
+	hash := common.ExtractInfoHash(item.Description)
+	if hash == "" {
+		// Check Title for infohash
+		hash = common.ExtractInfoHash(item.Comments)
+	}
+	infohash = hash
 	if infohash == "" {
-		magnet, err = common.GetMagnetInfo(magnetLink)
-		if err != nil {
-			log.Println("Error getting magnet info:", err)
-			return false
-		}
-	} else {
-		magnet = &common.Magnet{
-			InfoHash: infohash,
-			Name:     item.Title,
-			Link:     magnetLink,
-		}
+		//Get torrent file from http link
+		//Takes too long, not worth it
+		//magnet, err := common.OpenMagnetHttpURL(magnetLink)
+		//if err == nil && magnet != nil && magnet.InfoHash != "" {
+		//	log.Printf("Magnet: %s", magnet.InfoHash)
+		//}
 	}
-	if magnet == nil {
-		log.Println("Error getting magnet info")
-		return false
-	}
-	return deb.IsAvailable(magnet)
+	return infohash
 
 }
 
-func ProcessXMLResponse(resp *http.Response, deb debrid.Service) *http.Response {
+func (p *Proxy) ProcessXMLResponse(resp *http.Response) *http.Response {
 	if resp == nil || resp.Body == nil {
 		return resp
 	}
@@ -193,6 +228,7 @@ func ProcessXMLResponse(resp *http.Response, deb debrid.Service) *http.Response
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		log.Println("Error reading response body:", err)
+		resp.Body = io.NopCloser(bytes.NewReader(body))
 		return resp
 	}
 	err = resp.Body.Close()
@@ -203,43 +239,54 @@ func ProcessXMLResponse(resp *http.Response, deb debrid.Service) *http.Response
 	var rss RSS
 	err = xml.Unmarshal(body, &rss)
 	if err != nil {
-		log.Fatalf("Error unmarshalling XML: %v", err)
+		log.Printf("Error unmarshalling XML: %v", err)
+		resp.Body = io.NopCloser(bytes.NewReader(body))
 		return resp
 	}
-	newItems := &SafeItems{}
-	var wg sync.WaitGroup
+	indexer := ""
+	if len(rss.Channel.Items) > 0 {
+		indexer = rss.Channel.Items[0].ProwlarrIndexer.Text
+	}
 
 	// Step 4: Extract infohash or magnet URI, manipulate data
-	for _, item := range rss.Channel.Items {
-		wg.Add(1)
-		go func(item Item) {
-			defer wg.Done()
-			if XMLItemIsCached(item, deb) {
-				newItems.Add(item)
-			}
-		}(item)
+	IdsHashMap := getItemsHash(rss.Channel.Items)
+	hashes := make([]string, 0)
+	for _, hash := range IdsHashMap {
+		if hash != "" {
+			hashes = append(hashes, hash)
+		}
 	}
-	wg.Wait()
-	rss.Channel.Items = newItems.Get()
+	availableHashesMap := p.debrid.IsAvailable(hashes)
+	newItems := make([]Item, 0, len(rss.Channel.Items))
+
+	if len(hashes) > 0 {
+		for _, item := range rss.Channel.Items {
+			hash := IdsHashMap[item.GUID]
+			if hash == "" {
+				continue
+			}
+			isCached, exists := availableHashesMap[hash]
+			if !exists || !isCached {
+				continue
+			}
+			newItems = append(newItems, item)
+		}
+	}
+
+	log.Printf("[%s Report]: %d/%d items are cached || Found %d infohash", indexer, len(newItems), len(rss.Channel.Items), len(hashes))
+	rss.Channel.Items = newItems
 
 	// rss.Channel.Items = newItems
 	modifiedBody, err := xml.MarshalIndent(rss, "", "  ")
 	if err != nil {
 		log.Printf("Error marshalling XML: %v", err)
+		resp.Body = io.NopCloser(bytes.NewReader(body))
 		return resp
 	}
 	modifiedBody = append([]byte(xml.Header), modifiedBody...)
 
-	if err != nil {
-		log.Fatalf("Error marshalling XML: %v", err)
-		return resp
-	}
-
 	// Set the modified body back to the response
 	resp.Body = io.NopCloser(bytes.NewReader(modifiedBody))
-	resp.ContentLength = int64(len(modifiedBody))
-	resp.Header.Set("Content-Length", string(rune(len(modifiedBody))))
-
 	return resp
 }
 
@@ -249,9 +296,8 @@ func UrlMatches(re *regexp.Regexp) goproxy.ReqConditionFunc {
 	}
 }
 
-func StartProxy(config *common.Config, deb debrid.Service) {
-	username, password := config.Proxy.Username, config.Proxy.Password
-	cfg := config.Proxy
+func (p *Proxy) Start() {
+	username, password := p.username, p.password
 	proxy := goproxy.NewProxyHttpServer()
 	if username != "" || password != "" {
 		// Set up basic auth for proxy
@@ -261,14 +307,15 @@ func StartProxy(config *common.Config, deb debrid.Service) {
 	}
 
 	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.443$"))).HandleConnect(goproxy.AlwaysMitm)
-	proxy.OnResponse(UrlMatches(regexp.MustCompile("^.*/api\\?t=(search|tvsearch|movie)(&.*)?$"))).DoFunc(
+	proxy.OnResponse(
+		UrlMatches(regexp.MustCompile("^.*/api\\?t=(search|tvsearch|movie)(&.*)?$")),
+		goproxy.StatusCodeIs(http.StatusOK, http.StatusAccepted)).DoFunc(
 		func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
-			return ProcessResponse(resp, deb)
+			return p.ProcessResponse(resp)
 		})
 
-	port := cmp.Or(cfg.Port, "8181")
-	proxy.Verbose = cfg.Debug
-	port = fmt.Sprintf(":%s", port)
-	log.Printf("Starting proxy server on %s\n", port)
-	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s", port), proxy))
+	proxy.Verbose = p.debug
+	portFmt := fmt.Sprintf(":%s", p.port)
+	log.Printf("[*] Starting proxy server on %s\n", portFmt)
+	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s", portFmt), proxy))
 }

common/cache.go

@@ -0,0 +1,88 @@
+package common
+
+import (
+	"sync"
+)
+
+type Cache struct {
+	data     map[string]struct{}
+	order    []string
+	maxItems int
+	mu       sync.RWMutex
+}
+
+func NewCache(maxItems int) *Cache {
+	if maxItems <= 0 {
+		maxItems = 1000
+	}
+	return &Cache{
+		data:     make(map[string]struct{}, maxItems),
+		order:    make([]string, 0, maxItems),
+		maxItems: maxItems,
+	}
+}
+
+func (c *Cache) Add(value string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if _, exists := c.data[value]; !exists {
+		if len(c.order) >= c.maxItems {
+			delete(c.data, c.order[0])
+			c.order = c.order[1:]
+		}
+		c.data[value] = struct{}{}
+		c.order = append(c.order, value)
+	}
+}
+
+func (c *Cache) AddMultiple(values map[string]bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for value := range values {
+		if _, exists := c.data[value]; !exists {
+			if len(c.order) >= c.maxItems {
+				delete(c.data, c.order[0])
+				c.order = c.order[1:]
+			}
+			c.data[value] = struct{}{}
+			c.order = append(c.order, value)
+		}
+	}
+}
+
+func (c *Cache) Get(index int) (string, bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if index < 0 || index >= len(c.order) {
+		return "", false
+	}
+	return c.order[index], true
+}
+
+func (c *Cache) GetMultiple(values []string) map[string]bool {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	result := make(map[string]bool, len(values))
+	for _, value := range values {
+		if _, exists := c.data[value]; exists {
+			result[value] = true
+		}
+	}
+	return result
+}
+
+func (c *Cache) Exists(value string) bool {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	_, exists := c.data[value]
+	return exists
+}
+
+func (c *Cache) Len() int {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return len(c.order)
+}

common/config.go

@@ -16,7 +16,6 @@ type DebridConfig struct {
 }
 
 type Config struct {
-	DbDSN  string       `json:"db_dsn"`
 	Debrid DebridConfig `json:"debrid"`
 	Arrs   []struct {
 		WatchFolder     string `json:"watch_folder"`
@@ -25,12 +24,14 @@ type Config struct {
 		URL             string `json:"url"`
 	} `json:"arrs"`
 	Proxy struct {
-		Port     string `json:"port"`
-		Enabled  bool   `json:"enabled"`
-		Debug    bool   `json:"debug"`
-		Username string `json:"username"`
-		Password string `json:"password"`
+		Port       string `json:"port"`
+		Enabled    bool   `json:"enabled"`
+		Debug      bool   `json:"debug"`
+		Username   string `json:"username"`
+		Password   string `json:"password"`
+		CachedOnly bool   `json:"cached_only"`
 	}
+	MaxCacheSize int `json:"max_cache_size"`
 }
 
 func LoadConfig(path string) (*Config, error) {

common/regex.go

@@ -5,8 +5,9 @@ import (
 )
 
 var (
-	VIDEOMATCH = "(?i)(\\.)(YUV|WMV|WEBM|VOB|VIV|SVI|ROQ|RMVB|RM|OGV|OGG|NSV|MXF|MTS|M2TS|TS|MPG|MPEG|M2V|MP2|MPE|MPV|MP4|M4P|M4V|MOV|QT|MNG|MKV|FLV|DRC|AVI|ASF|AMV)$"
-	SUBMATCH   = "(?i)(\\.)(SRT|SUB|SBV|ASS|VTT|TTML|DFXP|STL|SCC|CAP|SMI|TTXT|TDS|USF|JSS|SSA|PSB|RT|LRC|SSB)$"
+	VIDEOMATCH  = "(?i)(\\.)(YUV|WMV|WEBM|VOB|VIV|SVI|ROQ|RMVB|RM|OGV|OGG|NSV|MXF|MTS|M2TS|TS|MPG|MPEG|M2V|MP2|MPE|MPV|MP4|M4P|M4V|MOV|QT|MNG|MKV|FLV|DRC|AVI|ASF|AMV)$"
+	SUBMATCH    = "(?i)(\\.)(SRT|SUB|SBV|ASS|VTT|TTML|DFXP|STL|SCC|CAP|SMI|TTXT|TDS|USF|JSS|SSA|PSB|RT|LRC|SSB)$"
+	SeasonMatch = "(?i)(?:season|s)[.\\-_\\s]?(\\d+)"
 )
 
 func RegexMatch(regex string, value string) bool {
@@ -25,3 +26,13 @@ func RemoveExtension(value string) string {
 		return value
 	}
 }
+
+func RegexFind(regex string, value string) string {
+	re := regexp.MustCompile(regex)
+	match := re.FindStringSubmatch(value)
+	if len(match) > 0 {
+		return match[0]
+	} else {
+		return ""
+	}
+}

common/utils.go

@@ -2,11 +2,16 @@ package common
 
 import (
 	"bufio"
+	"encoding/base32"
+	"encoding/hex"
 	"fmt"
+	"github.com/anacrolix/torrent/metainfo"
 	"log"
 	"math/rand"
+	"net/http"
 	"net/url"
 	"os"
+	"regexp"
 	"strings"
 )
 
@@ -46,14 +51,51 @@ func OpenMagnetFile(filePath string) string {
 	return ""
 }
 
+func OpenMagnetHttpURL(magnetLink string) (*Magnet, error) {
+	resp, err := http.Get(magnetLink)
+	if err != nil {
+		return nil, fmt.Errorf("error making GET request: %v", err)
+	}
+	defer func(resp *http.Response) {
+		err := resp.Body.Close()
+		if err != nil {
+			return
+		}
+	}(resp) // Ensure the response is closed after the function ends
+
+	// Create a scanner to read the file line by line
+
+	mi, err := metainfo.Load(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	hash := mi.HashInfoBytes()
+	infoHash := hash.HexString()
+	info, err := mi.UnmarshalInfo()
+	if err != nil {
+		return nil, err
+	}
+	log.Println("InfoHash: ", infoHash)
+	magnet := &Magnet{
+		InfoHash: infoHash,
+		Name:     info.Name,
+		Size:     info.Length,
+		Link:     mi.Magnet(&hash, &info).String(),
+	}
+	return magnet, nil
+
+}
+
 func GetMagnetInfo(magnetLink string) (*Magnet, error) {
 	if magnetLink == "" {
 		return nil, fmt.Errorf("error getting magnet from file")
 	}
+
 	magnetURI, err := url.Parse(magnetLink)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing magnet link")
 	}
+
 	query := magnetURI.Query()
 	xt := query.Get("xt")
 	dn := query.Get("dn")
@@ -81,3 +123,47 @@ func RandomString(length int) string {
 	}
 	return string(b)
 }
+
+func ExtractInfoHash(magnetDesc string) string {
+	const prefix = "xt=urn:btih:"
+	start := strings.Index(magnetDesc, prefix)
+	if start == -1 {
+		return ""
+	}
+	hash := ""
+	start += len(prefix)
+	end := strings.IndexAny(magnetDesc[start:], "&#")
+	if end == -1 {
+		hash = magnetDesc[start:]
+	} else {
+		hash = magnetDesc[start : start+end]
+	}
+	hash, _ = processInfoHash(hash) // Convert to hex if needed
+	return hash
+}
+
+func processInfoHash(input string) (string, error) {
+	// Regular expression for a valid 40-character hex infohash
+	hexRegex := regexp.MustCompile("^[0-9a-fA-F]{40}$")
+
+	// If it's already a valid hex infohash, return it as is
+	if hexRegex.MatchString(input) {
+		return strings.ToLower(input), nil
+	}
+
+	// If it's 32 characters long, it might be Base32 encoded
+	if len(input) == 32 {
+		// Ensure the input is uppercase and remove any padding
+		input = strings.ToUpper(strings.TrimRight(input, "="))
+
+		// Try to decode from Base32
+		decoded, err := base32.StdEncoding.DecodeString(input)
+		if err == nil && len(decoded) == 20 {
+			// If successful and the result is 20 bytes, encode to hex
+			return hex.EncodeToString(decoded), nil
+		}
+	}
+
+	// If we get here, it's not a valid infohash and we couldn't convert it
+	return "", fmt.Errorf("invalid infohash: %s", input)
+}

debrid/debrid.go

@@ -11,21 +11,23 @@ type Service interface {
 	CheckStatus(torrent *Torrent) (*Torrent, error)
 	DownloadLink(torrent *Torrent) error
 	Process(arr *Arr, magnet string) (*Torrent, error)
-	IsAvailable(magnet *common.Magnet) bool
+	IsAvailable(infohashes []string) map[string]bool
 }
 
 type Debrid struct {
 	Host             string `json:"host"`
 	APIKey           string
 	DownloadUncached bool
+	client           *common.RLHTTPClient
+	cache            *common.Cache
 }
 
-func NewDebrid(dc common.DebridConfig) Service {
+func NewDebrid(dc common.DebridConfig, cache *common.Cache) Service {
 	switch dc.Name {
 	case "realdebrid":
-		return NewRealDebrid(dc)
+		return NewRealDebrid(dc, cache)
 	default:
-		return NewRealDebrid(dc)
+		return NewRealDebrid(dc, cache)
 	}
 }
 
@@ -81,3 +83,30 @@ func getTorrentInfo(filePath string) (*Torrent, error) {
 	}
 	return torrent, nil
 }
+
+func GetLocalCache(infohashes []string, cache *common.Cache) ([]string, map[string]bool) {
+	result := make(map[string]bool)
+	hashes := make([]string, len(infohashes))
+
+	if len(infohashes) == 0 {
+		return hashes, result
+	}
+	if len(infohashes) == 1 {
+		if cache.Exists(infohashes[0]) {
+			return hashes, map[string]bool{infohashes[0]: true}
+		}
+		return infohashes, result
+	}
+
+	cachedHashes := cache.GetMultiple(infohashes)
+	for _, h := range infohashes {
+		_, exists := cachedHashes[h]
+		if !exists {
+			hashes = append(hashes, h)
+		} else {
+			result[h] = true
+		}
+	}
+
+	return hashes, result
+}

debrid/realdebrid.go

@@ -18,6 +18,7 @@ type RealDebrid struct {
 	APIKey           string
 	DownloadUncached bool
 	client           *common.RLHTTPClient
+	cache            *common.Cache
 }
 
 func (r *RealDebrid) Process(arr *Arr, magnet string) (*Torrent, error) {
@@ -28,32 +29,72 @@ func (r *RealDebrid) Process(arr *Arr, magnet string) (*Torrent, error) {
 	}
 	log.Printf("Torrent Name: %s", torrent.Name)
 	if !r.DownloadUncached {
-		if !r.IsAvailable(torrent.Magnet) {
+		hash, exists := r.IsAvailable([]string{torrent.InfoHash})[torrent.InfoHash]
+		if !exists || !hash {
 			return torrent, fmt.Errorf("torrent is not cached")
 		}
+		log.Printf("Torrent: %s is cached", torrent.Name)
 	}
 
+	torrent, err = r.SubmitMagnet(torrent)
+	if err != nil || torrent.Id == "" {
+		return nil, err
+	}
 	return r.CheckStatus(torrent)
 }
 
-func (r *RealDebrid) IsAvailable(magnet *common.Magnet) bool {
-	url := fmt.Sprintf("%s/torrents/instantAvailability/%s", r.Host, magnet.InfoHash)
-	resp, err := r.client.MakeRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return false
+func (r *RealDebrid) IsAvailable(infohashes []string) map[string]bool {
+	// Check if the infohashes are available in the local cache
+	hashes, result := GetLocalCache(infohashes, r.cache)
+
+	if len(hashes) == 0 {
+		// Either all the infohashes are locally cached or none are
+		r.cache.AddMultiple(result)
+		return result
 	}
-	var data structs.RealDebridAvailabilityResponse
-	err = json.Unmarshal(resp, &data)
-	if err != nil {
-		return false
+
+	// Divide hashes into groups of 100
+	for i := 0; i < len(hashes); i += 200 {
+		end := i + 200
+		if end > len(hashes) {
+			end = len(hashes)
+		}
+
+		// Filter out empty strings
+		validHashes := make([]string, 0, end-i)
+		for _, hash := range hashes[i:end] {
+			if hash != "" {
+				validHashes = append(validHashes, hash)
+			}
+		}
+
+		// If no valid hashes in this batch, continue to the next batch
+		if len(validHashes) == 0 {
+			continue
+		}
+
+		hashStr := strings.Join(validHashes, "/")
+		url := fmt.Sprintf("%s/torrents/instantAvailability/%s", r.Host, hashStr)
+		resp, err := r.client.MakeRequest(http.MethodGet, url, nil)
+		if err != nil {
+			log.Println("Error checking availability:", err)
+			return result
+		}
+		var data structs.RealDebridAvailabilityResponse
+		err = json.Unmarshal(resp, &data)
+		if err != nil {
+			log.Println("Error marshalling availability:", err)
+			return result
+		}
+		for _, h := range hashes[i:end] {
+			hosters, exists := data[strings.ToLower(h)]
+			if exists && len(hosters.Rd) > 0 {
+				result[h] = true
+			}
+		}
 	}
-	hosters, exists := data[strings.ToLower(magnet.InfoHash)]
-	if !exists || len(hosters) < 1 {
-		log.Printf("Torrent: %s not cached", magnet.Name)
-		return false
-	}
-	log.Printf("Torrent: %s is cached", magnet.Name)
-	return true
+	r.cache.AddMultiple(result) // Add the results to the cache
+	return result
 }
 
 func (r *RealDebrid) SubmitMagnet(torrent *Torrent) (*Torrent, error) {
@@ -89,7 +130,7 @@ func (r *RealDebrid) CheckStatus(torrent *Torrent) (*Torrent, error) {
 		} else if status == "waiting_files_selection" {
 			files := make([]TorrentFile, 0)
 			for _, f := range data.Files {
-				name := f.Path
+				name := filepath.Base(f.Path)
 				if !common.RegexMatch(common.VIDEOMATCH, name) && !common.RegexMatch(common.SUBMATCH, name) {
 					continue
 				}
@@ -137,7 +178,7 @@ func (r *RealDebrid) DownloadLink(torrent *Torrent) error {
 	return nil
 }
 
-func NewRealDebrid(dc common.DebridConfig) *RealDebrid {
+func NewRealDebrid(dc common.DebridConfig, cache *common.Cache) *RealDebrid {
 	rl := common.ParseRateLimit(dc.RateLimit)
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
@@ -148,5 +189,6 @@ func NewRealDebrid(dc common.DebridConfig) *RealDebrid {
 		APIKey:           dc.APIKey,
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
+		cache:            cache,
 	}
 }

debrid/structs/realdebrid.go

@@ -1,10 +1,63 @@
 package structs
 
-type RealDebridAvailabilityResponse map[string]Hosters
+import (
+	"encoding/json"
+	"fmt"
+)
 
-type Hosters map[string][]FileIDs
+type RealDebridAvailabilityResponse map[string]Hoster
 
-type FileIDs map[string]FileVariant
+func (r *RealDebridAvailabilityResponse) UnmarshalJSON(data []byte) error {
+	// First, try to unmarshal as an object
+	var objectData map[string]Hoster
+	err := json.Unmarshal(data, &objectData)
+	if err == nil {
+		*r = objectData
+		return nil
+	}
+
+	// If that fails, try to unmarshal as an array
+	var arrayData []map[string]Hoster
+	err = json.Unmarshal(data, &arrayData)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal as both object and array: %v", err)
+	}
+
+	// If it's an array, use the first element
+	if len(arrayData) > 0 {
+		*r = arrayData[0]
+		return nil
+	}
+
+	// If it's an empty array, initialize as an empty map
+	*r = make(map[string]Hoster)
+	return nil
+}
+
+type Hoster struct {
+	Rd []map[string]FileVariant `json:"rd"`
+}
+
+func (h *Hoster) UnmarshalJSON(data []byte) error {
+	// Attempt to unmarshal into the expected structure (an object with an "rd" key)
+	type Alias Hoster
+	var obj Alias
+	if err := json.Unmarshal(data, &obj); err == nil {
+		*h = Hoster(obj)
+		return nil
+	}
+
+	// If unmarshalling into an object fails, check if it's an empty array
+	var arr []interface{}
+	if err := json.Unmarshal(data, &arr); err == nil && len(arr) == 0 {
+		// It's an empty array; initialize with no entries
+		*h = Hoster{Rd: nil}
+		return nil
+	}
+
+	// If both attempts fail, return an error
+	return fmt.Errorf("hoster: cannot unmarshal JSON data: %s", string(data))
+}
 
 type FileVariant struct {
 	Filename string `json:"filename"`
@@ -39,3 +92,5 @@ type RealDebridTorrentInfo struct {
 	Speed   int      `json:"speed,omitempty"`
 	Seeders int      `json:"seeders,omitempty"`
 }
+
+// 5e6e2e77fd3921a7903a41336c844cc409bf8788/14527C07BDFDDFC642963238BB6E7507B9742947/66A1CD1A5C7F4014877A51AC2620E857E3BB4D16