Changelog: 0.1.4

CHANGELOG.md

@@ -0,0 +1,34 @@
+#### 0.1.0
+- Initial Release
+- Added Real Debrid Support
+- Added Arrs Support
+- Added Proxy Support
+- Added Basic Authentication for Proxy
+- Added Rate Limiting for Debrid Providers
+
+#### 0.1.1
+- Added support for "No Blackhole" for Arrs
+- Added support for "Cached Only" for Proxy
+- Bug Fixes
+
+#### 0.1.2
+- Bug fixes
+- Code cleanup
+- Get available hashes at once
+
+#### 0.1.3
+
+- Searching for infohashes in the xml description/summary/comments
+- Added local cache support
+- Added max cache size
+- Rewrite blackhole.go
+- Bug fixes
+  - Fixed indexer getting disabled
+  - Fixed blackhole not working
+
+#### 0.1.4
+
+- Rewrote Report log
+- Fix YTS, 1337x not grabbing infohash
+- Fix Torrent symlink bug
+- 

README.md

@@ -8,6 +8,10 @@ This is a Golang implementation go Torrent Blackhole with a **Real Debrid Proxy
 
 The proxy is useful in filtering out un-cached Real Debrid torrents
 
+### Changelog
+
+- View the [CHANGELOG.md](CHANGELOG.md) for the latest changes
+
 
 #### Installation
 ##### Docker Compose
@@ -15,7 +19,7 @@ The proxy is useful in filtering out un-cached Real Debrid torrents
 version: '3.7'
 services:
   blackhole:
-    image: cy01/blackhole:latest
+    image: cy01/blackhole:latest # or cy01/blackhole:beta
     container_name: blackhole
     user: "1000:1000"
     volumes:
@@ -75,7 +79,8 @@ Download the binary from the releases page and run it with the config file.
     "username": "username",
     "password": "password",
     "cached_only": true
-  }
+  },
+  "max_cache_size": 1000
 }
 ```
 

cmd/blackhole.go

@@ -12,6 +12,21 @@ import (
 	"time"
 )
 
+type Blackhole struct {
+	config *common.Config
+	deb    debrid.Service
+	cache  *common.Cache
+}
+
+func NewBlackhole(config *common.Config, deb debrid.Service, cache *common.Cache) *Blackhole {
+	return &Blackhole{
+		config: config,
+		deb:    deb,
+		cache:  cache,
+	}
+
+}
+
 func fileReady(path string) bool {
 	_, err := os.Stat(path)
 	return !os.IsNotExist(err) // Returns true if the file exists
@@ -33,12 +48,12 @@ func checkFileLoop(wg *sync.WaitGroup, dir string, file debrid.TorrentFile, read
 	}
 }
 
-func ProcessFiles(arr *debrid.Arr, torrent *debrid.Torrent) {
+func (b *Blackhole) processFiles(arr *debrid.Arr, torrent *debrid.Torrent) {
 	var wg sync.WaitGroup
 	files := torrent.Files
 	ready := make(chan debrid.TorrentFile, len(files))
 
-	log.Println("Checking files...")
+	log.Printf("Checking %d files...", len(files))
 
 	for _, file := range files {
 		wg.Add(1)
@@ -52,29 +67,30 @@ func ProcessFiles(arr *debrid.Arr, torrent *debrid.Torrent) {
 
 	for r := range ready {
 		log.Println("File is ready:", r.Name)
-		CreateSymLink(arr, torrent)
+		b.createSymLink(arr, torrent)
 
 	}
 	go torrent.Cleanup(true)
 	fmt.Printf("%s downloaded", torrent.Name)
 }
 
-func CreateSymLink(config *debrid.Arr, torrent *debrid.Torrent) {
+func (b *Blackhole) createSymLink(arr *debrid.Arr, torrent *debrid.Torrent) {
-	path := filepath.Join(config.CompletedFolder, torrent.Folder)
+	path := filepath.Join(arr.CompletedFolder, torrent.Folder)
 	err := os.MkdirAll(path, os.ModePerm)
 	if err != nil {
 		log.Printf("Failed to create directory: %s\n", path)
 	}
+
 	for _, file := range torrent.Files {
 		// Combine the directory and filename to form a full path
-		fullPath := filepath.Join(config.CompletedFolder, file.Path)
+		fullPath := filepath.Join(path, file.Name) // completedFolder/MyTVShow/MyTVShow.S01E01.720p.mkv
-
 		// Create a symbolic link if file doesn't exist
-		_ = os.Symlink(filepath.Join(config.Debrid.Folder, file.Path), fullPath)
+		torrentPath := filepath.Join(arr.Debrid.Folder, torrent.Folder, file.Name) // debridFolder/MyTVShow/MyTVShow.S01E01.720p.mkv
+		_ = os.Symlink(torrentPath, fullPath)
 	}
 }
 
-func watchFiles(watcher *fsnotify.Watcher, events map[string]time.Time) {
+func watcher(watcher *fsnotify.Watcher, events map[string]time.Time) {
 	for {
 		select {
 		case event, ok := <-watcher.Events:
@@ -96,7 +112,7 @@ func watchFiles(watcher *fsnotify.Watcher, events map[string]time.Time) {
 	}
 }
 
-func processFilesDebounced(arr *debrid.Arr, db debrid.Service, events map[string]time.Time, debouncePeriod time.Duration) {
+func (b *Blackhole) processFilesDebounced(arr *debrid.Arr, events map[string]time.Time, debouncePeriod time.Duration) {
 	ticker := time.NewTicker(1 * time.Second) // Check every second
 	defer ticker.Stop()
 
@@ -105,7 +121,7 @@ func processFilesDebounced(arr *debrid.Arr, db debrid.Service, events map[string
 			if time.Since(lastEventTime) >= debouncePeriod {
 				log.Printf("Torrent file detected: %s", file)
 				// Process the torrent file
-				torrent, err := db.Process(arr, file)
+				torrent, err := b.deb.Process(arr, file)
 				if err != nil && torrent != nil {
 					// remove torrent file
 					torrent.Cleanup(true)
@@ -113,7 +129,7 @@ func processFilesDebounced(arr *debrid.Arr, db debrid.Service, events map[string
 					log.Printf("Error processing torrent file: %s", err)
 				}
 				if err == nil && torrent != nil && len(torrent.Files) > 0 {
-					go ProcessFiles(arr, torrent)
+					go b.processFiles(arr, torrent)
 				}
 				delete(events, file) // remove file from channel
 
@@ -122,8 +138,8 @@ func processFilesDebounced(arr *debrid.Arr, db debrid.Service, events map[string
 	}
 }
 
-func StartArr(conf *debrid.Arr, db debrid.Service) {
+func (b *Blackhole) startArr(arr *debrid.Arr) {
-	log.Printf("Watching: %s", conf.WatchFolder)
+	log.Printf("Watching: %s", arr.WatchFolder)
 	w, err := fsnotify.NewWatcher()
 	if err != nil {
 		log.Println(err)
@@ -136,19 +152,19 @@ func StartArr(conf *debrid.Arr, db debrid.Service) {
 	}(w)
 	events := make(map[string]time.Time)
 
-	go watchFiles(w, events)
+	go watcher(w, events)
-	if err = w.Add(conf.WatchFolder); err != nil {
+	if err = w.Add(arr.WatchFolder); err != nil {
 		log.Println("Error Watching folder:", err)
 		return
 	}
 
-	processFilesDebounced(conf, db, events, 1*time.Second)
+	b.processFilesDebounced(arr, events, 1*time.Second)
 }
 
-func StartBlackhole(config *common.Config, deb debrid.Service) {
+func (b *Blackhole) Start() {
 	log.Println("[*] Starting Blackhole")
 	var wg sync.WaitGroup
-	for _, conf := range config.Arrs {
+	for _, conf := range b.config.Arrs {
 		wg.Add(1)
 		defer wg.Done()
 		headers := map[string]string{
@@ -157,14 +173,14 @@ func StartBlackhole(config *common.Config, deb debrid.Service) {
 		client := common.NewRLHTTPClient(nil, headers)
 
 		arr := &debrid.Arr{
-			Debrid:          config.Debrid,
+			Debrid:          b.config.Debrid,
 			WatchFolder:     conf.WatchFolder,
 			CompletedFolder: conf.CompletedFolder,
 			Token:           conf.Token,
 			URL:             conf.URL,
 			Client:          client,
 		}
-		go StartArr(arr, deb)
+		go b.startArr(arr)
 	}
 	wg.Wait()
 }

cmd/main.go

@@ -1,19 +1,22 @@
 package cmd
 
 import (
+	"cmp"
 	"goBlack/common"
 	"goBlack/debrid"
 	"sync"
 )
 
 func Start(config *common.Config) {
-
+	maxCacheSize := cmp.Or(config.MaxCacheSize, 1000)
-	deb := debrid.NewDebrid(config.Debrid)
+	cache := common.NewCache(maxCacheSize)
+	
+	deb := debrid.NewDebrid(config.Debrid, cache)
 
 	var wg sync.WaitGroup
 
 	if config.Proxy.Enabled {
-		proxy := NewProxy(*config, deb)
+		proxy := NewProxy(*config, deb, cache)
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
@@ -22,10 +25,11 @@ func Start(config *common.Config) {
 	}
 
 	if len(config.Arrs) > 0 {
+		blackhole := NewBlackhole(config, deb, cache)
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			StartBlackhole(config, deb)
+			blackhole.Start()
 		}()
 	}
 

cmd/proxy.go

@@ -16,80 +16,78 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 )
 
 type RSS struct {
 	XMLName xml.Name `xml:"rss"`
+	Text    string   `xml:",chardata"`
 	Version string   `xml:"version,attr"`
-	Channel Channel  `xml:"channel"`
+	Atom    string   `xml:"atom,attr"`
-}
+	Torznab string   `xml:"torznab,attr"`
-
+	Channel struct {
-type Channel struct {
+		Text string `xml:",chardata"`
-	XMLName  xml.Name `xml:"channel"`
+		Link struct {
-	Title    string   `xml:"title"`
+			Text string `xml:",chardata"`
-	AtomLink AtomLink `xml:"link"`
+			Rel  string `xml:"rel,attr"`
-	Items    []Item   `xml:"item"`
+			Type string `xml:"type,attr"`
-}
+		} `xml:"link"`
-
+		Title string `xml:"title"`
-type AtomLink struct {
+		Items []Item `xml:"item"`
-	XMLName xml.Name `xml:"link"`
+	} `xml:"channel"`
-	Rel     string   `xml:"rel,attr"`
-	Type    string   `xml:"type,attr"`
 }
 
 type Item struct {
-	XMLName         xml.Name        `xml:"item"`
+	Text            string `xml:",chardata"`
-	Title           string          `xml:"title"`
+	Title           string `xml:"title"`
-	Description     string          `xml:"description"`
+	Description     string `xml:"description"`
-	GUID            string          `xml:"guid"`
+	GUID            string `xml:"guid"`
-	ProwlarrIndexer ProwlarrIndexer `xml:"prowlarrindexer"`
+	ProwlarrIndexer struct {
-	Comments        string          `xml:"comments"`
+		Text string `xml:",chardata"`
-	PubDate         string          `xml:"pubDate"`
+		ID   string `xml:"id,attr"`
-	Size            int64           `xml:"size"`
+		Type string `xml:"type,attr"`
-	Link            string          `xml:"link"`
+	} `xml:"prowlarrindexer"`
-	Categories      []string        `xml:"category"`
+	Comments  string   `xml:"comments"`
-	Enclosure       Enclosure       `xml:"enclosure"`
+	PubDate   string   `xml:"pubDate"`
-	TorznabAttrs    []TorznabAttr   `xml:"torznab:attr"`
+	Size      string   `xml:"size"`
-}
+	Link      string   `xml:"link"`
-
+	Category  []string `xml:"category"`
-type ProwlarrIndexer struct {
+	Enclosure struct {
-	ID    string `xml:"id,attr"`
+		Text   string `xml:",chardata"`
-	Type  string `xml:"type,attr"`
+		URL    string `xml:"url,attr"`
-	Value string `xml:",chardata"`
+		Length string `xml:"length,attr"`
-}
+		Type   string `xml:"type,attr"`
-
+	} `xml:"enclosure"`
-type Enclosure struct {
+	TorznabAttrs []struct {
-	URL    string `xml:"url,attr"`
+		Text  string `xml:",chardata"`
-	Length int64  `xml:"length,attr"`
+		Name  string `xml:"name,attr"`
-	Type   string `xml:"type,attr"`
+		Value string `xml:"value,attr"`
-}
+	} `xml:"attr"`
-
-type TorznabAttr struct {
-	Name  string `xml:"name,attr"`
-	Value string `xml:"value,attr"`
 }
 
 type Proxy struct {
-	Port       string `json:"port"`
+	port       string
-	Enabled    bool   `json:"enabled"`
+	enabled    bool
-	Debug      bool   `json:"debug"`
+	debug      bool
-	Username   string `json:"username"`
+	username   string
-	Password   string `json:"password"`
+	password   string
-	CachedOnly bool   `json:"cached_only"`
+	cachedOnly bool
-	Debrid     debrid.Service
+	debrid     debrid.Service
+	cache      *common.Cache
 }
 
-func NewProxy(config common.Config, deb debrid.Service) *Proxy {
+func NewProxy(config common.Config, deb debrid.Service, cache *common.Cache) *Proxy {
 	cfg := config.Proxy
 	port := cmp.Or(os.Getenv("PORT"), cfg.Port, "8181")
 	return &Proxy{
-		Port:       port,
+		port:       port,
-		Enabled:    cfg.Enabled,
+		enabled:    cfg.Enabled,
-		Debug:      cfg.Debug,
+		debug:      cfg.Debug,
-		Username:   cfg.Username,
+		username:   cfg.Username,
-		Password:   cfg.Password,
+		password:   cfg.Password,
-		CachedOnly: cfg.CachedOnly,
+		cachedOnly: cfg.CachedOnly,
-		Debrid:     deb,
+		debrid:     deb,
+		cache:      cache,
 	}
 }
 
@@ -147,45 +145,76 @@ func (p *Proxy) ProcessResponse(resp *http.Response) *http.Response {
 }
 
 func getItemsHash(items []Item) map[string]string {
-	IdHashMap := make(map[string]string)
+
+	var wg sync.WaitGroup
+	idHashMap := sync.Map{} // Use sync.Map for concurrent access
+
 	for _, item := range items {
-		hash := getItemHash(item)
+		wg.Add(1)
-		IdHashMap[item.GUID] = hash
+		go func(item Item) {
+			defer wg.Done()
+			hash := strings.ToLower(item.getHash())
+			if hash != "" {
+				idHashMap.Store(item.GUID, hash) // Store directly into sync.Map
+			}
+		}(item)
 	}
-	return IdHashMap
+	wg.Wait()
+
+	// Convert sync.Map to regular map
+	finalMap := make(map[string]string)
+	idHashMap.Range(func(key, value interface{}) bool {
+		finalMap[key.(string)] = value.(string)
+		return true
+	})
+
+	return finalMap
 }
 
-func getItemHash(item Item) string {
+func (item Item) getHash() string {
-	magnetLink := ""
 	infohash := ""
 
-	// Extract magnet link from the link or comments
-	if strings.Contains(item.Link, "magnet:?") {
-		magnetLink = item.Link
-	} else if strings.Contains(item.GUID, "magnet:?") {
-		magnetLink = item.GUID
-	}
-
-	// Extract infohash from <torznab:attr> elements
 	for _, attr := range item.TorznabAttrs {
 		if attr.Name == "infohash" {
-			infohash = attr.Value
+			return attr.Value
 		}
 	}
-	if magnetLink == "" && infohash == "" {
+
+	if strings.Contains(item.GUID, "magnet:?") {
+		magnet, err := common.GetMagnetInfo(item.GUID)
+		if err == nil && magnet != nil && magnet.InfoHash != "" {
+			return magnet.InfoHash
+		}
+	}
+
+	magnetLink := item.Link
+
+	if magnetLink == "" {
 		// We can't check the availability of the torrent without a magnet link or infohash
 		return ""
 	}
-	var magnet *common.Magnet
-	var err error
 
-	if infohash == "" {
+	if strings.Contains(magnetLink, "magnet:?") {
-		magnet, err = common.GetMagnetInfo(magnetLink)
+		magnet, err := common.GetMagnetInfo(magnetLink)
-		if err != nil || magnet == nil || magnet.InfoHash == "" {
+		if err == nil && magnet != nil && magnet.InfoHash != "" {
-			log.Println("Error getting magnet info:", err)
+			return magnet.InfoHash
-			return ""
 		}
-		infohash = magnet.InfoHash
+	}
+
+	//Check Description for infohash
+	hash := common.ExtractInfoHash(item.Description)
+	if hash == "" {
+		// Check Title for infohash
+		hash = common.ExtractInfoHash(item.Comments)
+	}
+	infohash = hash
+	if infohash == "" {
+		//Get torrent file from http link
+		//Takes too long, not worth it
+		//magnet, err := common.OpenMagnetHttpURL(magnetLink)
+		//if err == nil && magnet != nil && magnet.InfoHash != "" {
+		//	log.Printf("Magnet: %s", magnet.InfoHash)
+		//}
 	}
 	return infohash
 
@@ -198,9 +227,8 @@ func (p *Proxy) ProcessXMLResponse(resp *http.Response) *http.Response {
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		if p.Debug {
+		log.Println("Error reading response body:", err)
-			log.Println("Error reading response body:", err)
+		resp.Body = io.NopCloser(bytes.NewReader(body))
-		}
 		return resp
 	}
 	err = resp.Body.Close()
@@ -211,12 +239,14 @@ func (p *Proxy) ProcessXMLResponse(resp *http.Response) *http.Response {
 	var rss RSS
 	err = xml.Unmarshal(body, &rss)
 	if err != nil {
-		if p.Debug {
+		log.Printf("Error unmarshalling XML: %v", err)
-			log.Printf("Error unmarshalling XML: %v", err)
+		resp.Body = io.NopCloser(bytes.NewReader(body))
-		}
 		return resp
 	}
-	newItems := make([]Item, 0)
+	indexer := ""
+	if len(rss.Channel.Items) > 0 {
+		indexer = rss.Channel.Items[0].ProwlarrIndexer.Text
+	}
 
 	// Step 4: Extract infohash or magnet URI, manipulate data
 	IdsHashMap := getItemsHash(rss.Channel.Items)
@@ -226,45 +256,37 @@ func (p *Proxy) ProcessXMLResponse(resp *http.Response) *http.Response {
 			hashes = append(hashes, hash)
 		}
 	}
-	if len(hashes) == 0 {
+	availableHashesMap := p.debrid.IsAvailable(hashes)
-		// No infohashes or magnet links found, should we return the original response?
+	newItems := make([]Item, 0, len(rss.Channel.Items))
-		return resp
+
+	if len(hashes) > 0 {
+		for _, item := range rss.Channel.Items {
+			hash := IdsHashMap[item.GUID]
+			if hash == "" {
+				continue
+			}
+			isCached, exists := availableHashesMap[hash]
+			if !exists || !isCached {
+				continue
+			}
+			newItems = append(newItems, item)
+		}
 	}
-	availableHashes := p.Debrid.IsAvailable(hashes)
+
-	for _, item := range rss.Channel.Items {
+	log.Printf("[%s Report]: %d/%d items are cached || Found %d infohash", indexer, len(newItems), len(rss.Channel.Items), len(hashes))
-		hash := IdsHashMap[item.GUID]
-		if hash == "" {
-			// newItems = append(newItems, item)
-			continue
-		}
-		isCached, exists := availableHashes[hash]
-		if !exists {
-			// newItems = append(newItems, item)
-			continue
-		}
-		if !isCached {
-			continue
-		}
-		newItems = append(newItems, item)
-	}
-	log.Printf("Report: %d/%d items are cached", len(newItems), len(rss.Channel.Items))
 	rss.Channel.Items = newItems
 
 	// rss.Channel.Items = newItems
 	modifiedBody, err := xml.MarshalIndent(rss, "", "  ")
 	if err != nil {
-		if p.Debug {
+		log.Printf("Error marshalling XML: %v", err)
-			log.Printf("Error marshalling XML: %v", err)
+		resp.Body = io.NopCloser(bytes.NewReader(body))
-		}
 		return resp
 	}
 	modifiedBody = append([]byte(xml.Header), modifiedBody...)
 
 	// Set the modified body back to the response
 	resp.Body = io.NopCloser(bytes.NewReader(modifiedBody))
-	resp.ContentLength = int64(len(modifiedBody))
-	resp.Header.Set("Content-Length", string(rune(len(modifiedBody))))
-
 	return resp
 }
 
@@ -275,7 +297,7 @@ func UrlMatches(re *regexp.Regexp) goproxy.ReqConditionFunc {
 }
 
 func (p *Proxy) Start() {
-	username, password := p.Username, p.Password
+	username, password := p.username, p.password
 	proxy := goproxy.NewProxyHttpServer()
 	if username != "" || password != "" {
 		// Set up basic auth for proxy
@@ -285,13 +307,15 @@ func (p *Proxy) Start() {
 	}
 
 	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.443$"))).HandleConnect(goproxy.AlwaysMitm)
-	proxy.OnResponse(UrlMatches(regexp.MustCompile("^.*/api\\?t=(search|tvsearch|movie)(&.*)?$"))).DoFunc(
+	proxy.OnResponse(
+		UrlMatches(regexp.MustCompile("^.*/api\\?t=(search|tvsearch|movie)(&.*)?$")),
+		goproxy.StatusCodeIs(http.StatusOK, http.StatusAccepted)).DoFunc(
 		func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
 			return p.ProcessResponse(resp)
 		})
 
-	proxy.Verbose = p.Debug
+	proxy.Verbose = p.debug
-	portFmt := fmt.Sprintf(":%s", p.Port)
+	portFmt := fmt.Sprintf(":%s", p.port)
 	log.Printf("[*] Starting proxy server on %s\n", portFmt)
 	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s", portFmt), proxy))
 }

common/cache.go

@@ -0,0 +1,88 @@
+package common
+
+import (
+	"sync"
+)
+
+type Cache struct {
+	data     map[string]struct{}
+	order    []string
+	maxItems int
+	mu       sync.RWMutex
+}
+
+func NewCache(maxItems int) *Cache {
+	if maxItems <= 0 {
+		maxItems = 1000
+	}
+	return &Cache{
+		data:     make(map[string]struct{}, maxItems),
+		order:    make([]string, 0, maxItems),
+		maxItems: maxItems,
+	}
+}
+
+func (c *Cache) Add(value string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if _, exists := c.data[value]; !exists {
+		if len(c.order) >= c.maxItems {
+			delete(c.data, c.order[0])
+			c.order = c.order[1:]
+		}
+		c.data[value] = struct{}{}
+		c.order = append(c.order, value)
+	}
+}
+
+func (c *Cache) AddMultiple(values map[string]bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for value := range values {
+		if _, exists := c.data[value]; !exists {
+			if len(c.order) >= c.maxItems {
+				delete(c.data, c.order[0])
+				c.order = c.order[1:]
+			}
+			c.data[value] = struct{}{}
+			c.order = append(c.order, value)
+		}
+	}
+}
+
+func (c *Cache) Get(index int) (string, bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if index < 0 || index >= len(c.order) {
+		return "", false
+	}
+	return c.order[index], true
+}
+
+func (c *Cache) GetMultiple(values []string) map[string]bool {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	result := make(map[string]bool, len(values))
+	for _, value := range values {
+		if _, exists := c.data[value]; exists {
+			result[value] = true
+		}
+	}
+	return result
+}
+
+func (c *Cache) Exists(value string) bool {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	_, exists := c.data[value]
+	return exists
+}
+
+func (c *Cache) Len() int {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return len(c.order)
+}

common/config.go

@@ -16,7 +16,6 @@ type DebridConfig struct {
 }
 
 type Config struct {
-	DbDSN  string       `json:"db_dsn"`
 	Debrid DebridConfig `json:"debrid"`
 	Arrs   []struct {
 		WatchFolder     string `json:"watch_folder"`
@@ -32,6 +31,7 @@ type Config struct {
 		Password   string `json:"password"`
 		CachedOnly bool   `json:"cached_only"`
 	}
+	MaxCacheSize int `json:"max_cache_size"`
 }
 
 func LoadConfig(path string) (*Config, error) {

common/regex.go

@@ -5,8 +5,9 @@ import (
 )
 
 var (
-	VIDEOMATCH = "(?i)(\\.)(YUV|WMV|WEBM|VOB|VIV|SVI|ROQ|RMVB|RM|OGV|OGG|NSV|MXF|MTS|M2TS|TS|MPG|MPEG|M2V|MP2|MPE|MPV|MP4|M4P|M4V|MOV|QT|MNG|MKV|FLV|DRC|AVI|ASF|AMV)$"
+	VIDEOMATCH  = "(?i)(\\.)(YUV|WMV|WEBM|VOB|VIV|SVI|ROQ|RMVB|RM|OGV|OGG|NSV|MXF|MTS|M2TS|TS|MPG|MPEG|M2V|MP2|MPE|MPV|MP4|M4P|M4V|MOV|QT|MNG|MKV|FLV|DRC|AVI|ASF|AMV)$"
-	SUBMATCH   = "(?i)(\\.)(SRT|SUB|SBV|ASS|VTT|TTML|DFXP|STL|SCC|CAP|SMI|TTXT|TDS|USF|JSS|SSA|PSB|RT|LRC|SSB)$"
+	SUBMATCH    = "(?i)(\\.)(SRT|SUB|SBV|ASS|VTT|TTML|DFXP|STL|SCC|CAP|SMI|TTXT|TDS|USF|JSS|SSA|PSB|RT|LRC|SSB)$"
+	SeasonMatch = "(?i)(?:season|s)[.\\-_\\s]?(\\d+)"
 )
 
 func RegexMatch(regex string, value string) bool {
@@ -25,3 +26,13 @@ func RemoveExtension(value string) string {
 		return value
 	}
 }
+
+func RegexFind(regex string, value string) string {
+	re := regexp.MustCompile(regex)
+	match := re.FindStringSubmatch(value)
+	if len(match) > 0 {
+		return match[0]
+	} else {
+		return ""
+	}
+}

common/utils.go

@@ -2,11 +2,16 @@ package common
 
 import (
 	"bufio"
+	"encoding/base32"
+	"encoding/hex"
 	"fmt"
+	"github.com/anacrolix/torrent/metainfo"
 	"log"
 	"math/rand"
+	"net/http"
 	"net/url"
 	"os"
+	"regexp"
 	"strings"
 )
 
@@ -46,14 +51,51 @@ func OpenMagnetFile(filePath string) string {
 	return ""
 }
 
+func OpenMagnetHttpURL(magnetLink string) (*Magnet, error) {
+	resp, err := http.Get(magnetLink)
+	if err != nil {
+		return nil, fmt.Errorf("error making GET request: %v", err)
+	}
+	defer func(resp *http.Response) {
+		err := resp.Body.Close()
+		if err != nil {
+			return
+		}
+	}(resp) // Ensure the response is closed after the function ends
+
+	// Create a scanner to read the file line by line
+
+	mi, err := metainfo.Load(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	hash := mi.HashInfoBytes()
+	infoHash := hash.HexString()
+	info, err := mi.UnmarshalInfo()
+	if err != nil {
+		return nil, err
+	}
+	log.Println("InfoHash: ", infoHash)
+	magnet := &Magnet{
+		InfoHash: infoHash,
+		Name:     info.Name,
+		Size:     info.Length,
+		Link:     mi.Magnet(&hash, &info).String(),
+	}
+	return magnet, nil
+
+}
+
 func GetMagnetInfo(magnetLink string) (*Magnet, error) {
 	if magnetLink == "" {
 		return nil, fmt.Errorf("error getting magnet from file")
 	}
+
 	magnetURI, err := url.Parse(magnetLink)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing magnet link")
 	}
+
 	query := magnetURI.Query()
 	xt := query.Get("xt")
 	dn := query.Get("dn")
@@ -81,3 +123,47 @@ func RandomString(length int) string {
 	}
 	return string(b)
 }
+
+func ExtractInfoHash(magnetDesc string) string {
+	const prefix = "xt=urn:btih:"
+	start := strings.Index(magnetDesc, prefix)
+	if start == -1 {
+		return ""
+	}
+	hash := ""
+	start += len(prefix)
+	end := strings.IndexAny(magnetDesc[start:], "&#")
+	if end == -1 {
+		hash = magnetDesc[start:]
+	} else {
+		hash = magnetDesc[start : start+end]
+	}
+	hash, _ = processInfoHash(hash) // Convert to hex if needed
+	return hash
+}
+
+func processInfoHash(input string) (string, error) {
+	// Regular expression for a valid 40-character hex infohash
+	hexRegex := regexp.MustCompile("^[0-9a-fA-F]{40}$")
+
+	// If it's already a valid hex infohash, return it as is
+	if hexRegex.MatchString(input) {
+		return strings.ToLower(input), nil
+	}
+
+	// If it's 32 characters long, it might be Base32 encoded
+	if len(input) == 32 {
+		// Ensure the input is uppercase and remove any padding
+		input = strings.ToUpper(strings.TrimRight(input, "="))
+
+		// Try to decode from Base32
+		decoded, err := base32.StdEncoding.DecodeString(input)
+		if err == nil && len(decoded) == 20 {
+			// If successful and the result is 20 bytes, encode to hex
+			return hex.EncodeToString(decoded), nil
+		}
+	}
+
+	// If we get here, it's not a valid infohash and we couldn't convert it
+	return "", fmt.Errorf("invalid infohash: %s", input)
+}

debrid/debrid.go

@@ -18,14 +18,16 @@ type Debrid struct {
 	Host             string `json:"host"`
 	APIKey           string
 	DownloadUncached bool
+	client           *common.RLHTTPClient
+	cache            *common.Cache
 }
 
-func NewDebrid(dc common.DebridConfig) Service {
+func NewDebrid(dc common.DebridConfig, cache *common.Cache) Service {
 	switch dc.Name {
 	case "realdebrid":
-		return NewRealDebrid(dc)
+		return NewRealDebrid(dc, cache)
 	default:
-		return NewRealDebrid(dc)
+		return NewRealDebrid(dc, cache)
 	}
 }
 
@@ -81,3 +83,30 @@ func getTorrentInfo(filePath string) (*Torrent, error) {
 	}
 	return torrent, nil
 }
+
+func GetLocalCache(infohashes []string, cache *common.Cache) ([]string, map[string]bool) {
+	result := make(map[string]bool)
+	hashes := make([]string, len(infohashes))
+
+	if len(infohashes) == 0 {
+		return hashes, result
+	}
+	if len(infohashes) == 1 {
+		if cache.Exists(infohashes[0]) {
+			return hashes, map[string]bool{infohashes[0]: true}
+		}
+		return infohashes, result
+	}
+
+	cachedHashes := cache.GetMultiple(infohashes)
+	for _, h := range infohashes {
+		_, exists := cachedHashes[h]
+		if !exists {
+			hashes = append(hashes, h)
+		} else {
+			result[h] = true
+		}
+	}
+
+	return hashes, result
+}

debrid/realdebrid.go

@@ -18,6 +18,7 @@ type RealDebrid struct {
 	APIKey           string
 	DownloadUncached bool
 	client           *common.RLHTTPClient
+	cache            *common.Cache
 }
 
 func (r *RealDebrid) Process(arr *Arr, magnet string) (*Torrent, error) {
@@ -28,7 +29,8 @@ func (r *RealDebrid) Process(arr *Arr, magnet string) (*Torrent, error) {
 	}
 	log.Printf("Torrent Name: %s", torrent.Name)
 	if !r.DownloadUncached {
-		if !r.IsAvailable([]string{torrent.InfoHash})[torrent.InfoHash] {
+		hash, exists := r.IsAvailable([]string{torrent.InfoHash})[torrent.InfoHash]
+		if !exists || !hash {
 			return torrent, fmt.Errorf("torrent is not cached")
 		}
 		log.Printf("Torrent: %s is cached", torrent.Name)
@@ -42,29 +44,56 @@ func (r *RealDebrid) Process(arr *Arr, magnet string) (*Torrent, error) {
 }
 
 func (r *RealDebrid) IsAvailable(infohashes []string) map[string]bool {
-	hashes := strings.Join(infohashes, "/")
+	// Check if the infohashes are available in the local cache
-	result := make(map[string]bool)
+	hashes, result := GetLocalCache(infohashes, r.cache)
-	url := fmt.Sprintf("%s/torrents/instantAvailability/%s", r.Host, hashes)
+
-	resp, err := r.client.MakeRequest(http.MethodGet, url, nil)
+	if len(hashes) == 0 {
-	if err != nil {
+		// Either all the infohashes are locally cached or none are
-		log.Println(url)
+		r.cache.AddMultiple(result)
-		log.Println("Error checking availability:", err)
 		return result
 	}
-	var data structs.RealDebridAvailabilityResponse
+
-	err = json.Unmarshal(resp, &data)
+	// Divide hashes into groups of 100
-	if err != nil {
+	for i := 0; i < len(hashes); i += 200 {
-		log.Println("Error marshalling availability:", err)
+		end := i + 200
-		return result
+		if end > len(hashes) {
-	}
+			end = len(hashes)
-	for _, h := range infohashes {
+		}
-		hosters, exists := data[strings.ToLower(h)]
+
-		if !exists || len(hosters.Rd) < 1 {
+		// Filter out empty strings
-			result[h] = false
+		validHashes := make([]string, 0, end-i)
-		} else {
+		for _, hash := range hashes[i:end] {
-			result[h] = true
+			if hash != "" {
+				validHashes = append(validHashes, hash)
+			}
+		}
+
+		// If no valid hashes in this batch, continue to the next batch
+		if len(validHashes) == 0 {
+			continue
+		}
+
+		hashStr := strings.Join(validHashes, "/")
+		url := fmt.Sprintf("%s/torrents/instantAvailability/%s", r.Host, hashStr)
+		resp, err := r.client.MakeRequest(http.MethodGet, url, nil)
+		if err != nil {
+			log.Println("Error checking availability:", err)
+			return result
+		}
+		var data structs.RealDebridAvailabilityResponse
+		err = json.Unmarshal(resp, &data)
+		if err != nil {
+			log.Println("Error marshalling availability:", err)
+			return result
+		}
+		for _, h := range hashes[i:end] {
+			hosters, exists := data[strings.ToLower(h)]
+			if exists && len(hosters.Rd) > 0 {
+				result[h] = true
+			}
 		}
 	}
+	r.cache.AddMultiple(result) // Add the results to the cache
 	return result
 }
 
@@ -101,7 +130,7 @@ func (r *RealDebrid) CheckStatus(torrent *Torrent) (*Torrent, error) {
 		} else if status == "waiting_files_selection" {
 			files := make([]TorrentFile, 0)
 			for _, f := range data.Files {
-				name := f.Path
+				name := filepath.Base(f.Path)
 				if !common.RegexMatch(common.VIDEOMATCH, name) && !common.RegexMatch(common.SUBMATCH, name) {
 					continue
 				}
@@ -149,7 +178,7 @@ func (r *RealDebrid) DownloadLink(torrent *Torrent) error {
 	return nil
 }
 
-func NewRealDebrid(dc common.DebridConfig) *RealDebrid {
+func NewRealDebrid(dc common.DebridConfig, cache *common.Cache) *RealDebrid {
 	rl := common.ParseRateLimit(dc.RateLimit)
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", dc.APIKey),
@@ -160,5 +189,6 @@ func NewRealDebrid(dc common.DebridConfig) *RealDebrid {
 		APIKey:           dc.APIKey,
 		DownloadUncached: dc.DownloadUncached,
 		client:           client,
+		cache:            cache,
 	}
 }

debrid/structs/realdebrid.go

@@ -7,6 +7,33 @@ import (
 
 type RealDebridAvailabilityResponse map[string]Hoster
 
+func (r *RealDebridAvailabilityResponse) UnmarshalJSON(data []byte) error {
+	// First, try to unmarshal as an object
+	var objectData map[string]Hoster
+	err := json.Unmarshal(data, &objectData)
+	if err == nil {
+		*r = objectData
+		return nil
+	}
+
+	// If that fails, try to unmarshal as an array
+	var arrayData []map[string]Hoster
+	err = json.Unmarshal(data, &arrayData)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal as both object and array: %v", err)
+	}
+
+	// If it's an array, use the first element
+	if len(arrayData) > 0 {
+		*r = arrayData[0]
+		return nil
+	}
+
+	// If it's an empty array, initialize as an empty map
+	*r = make(map[string]Hoster)
+	return nil
+}
+
 type Hoster struct {
 	Rd []map[string]FileVariant `json:"rd"`
 }