Performance improvements; import speedup
This commit is contained in:
@@ -32,6 +32,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
|||||||
# Stage 2: Create directory structure
|
# Stage 2: Create directory structure
|
||||||
FROM alpine:3.19 as dirsetup
|
FROM alpine:3.19 as dirsetup
|
||||||
RUN mkdir -p /app/logs && \
|
RUN mkdir -p /app/logs && \
|
||||||
|
mkdir -p /app/cache && \
|
||||||
chmod 777 /app/logs && \
|
chmod 777 /app/logs && \
|
||||||
touch /app/logs/decypharr.log && \
|
touch /app/logs/decypharr.log && \
|
||||||
chmod 666 /app/logs/decypharr.log
|
chmod 666 /app/logs/decypharr.log
|
||||||
|
|||||||
@@ -218,7 +218,10 @@ This is particularly useful if you want to use the Repair tool without using Qbi
|
|||||||
|
|
||||||
The repair worker is a simple worker that checks for missing files in the Arrs(Sonarr, Radarr, etc). It's particularly useful for files either deleted by the Debrid provider or files with bad symlinks.
|
The repair worker is a simple worker that checks for missing files in the Arrs(Sonarr, Radarr, etc). It's particularly useful for files either deleted by the Debrid provider or files with bad symlinks.
|
||||||
|
|
||||||
**Note**: If you're using zurg, set the `zurg_url` under repair config. This will speed up the repair process, exponentially.
|
**Notes**
|
||||||
|
|
||||||
|
- For those using the webdav server, set the `use_webdav` key to `true` in the debrid provider config. This will speed up the repair process, exponentially.
|
||||||
|
- For those using zurg, set the `zurg_url` under repair config. This will speed up the repair process, exponentially.
|
||||||
|
|
||||||
- Search for broken symlinks/files
|
- Search for broken symlinks/files
|
||||||
- Search for missing files
|
- Search for missing files
|
||||||
|
|||||||
@@ -86,7 +86,8 @@
|
|||||||
"enabled": false,
|
"enabled": false,
|
||||||
"interval": "12h",
|
"interval": "12h",
|
||||||
"run_on_start": false,
|
"run_on_start": false,
|
||||||
"zurg_url": "http://zurg:9999",
|
"zurg_url": "",
|
||||||
|
"use_webdav": false,
|
||||||
"auto_process": false
|
"auto_process": false
|
||||||
},
|
},
|
||||||
"log_level": "info",
|
"log_level": "info",
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ var (
|
|||||||
MUSICMATCH = "(?i)(\\.)(mp2|mp3|m4a|m4b|m4p|ogg|oga|opus|wma|wav|wv|flac|ape|aif|aiff|aifc)$"
|
MUSICMATCH = "(?i)(\\.)(mp2|mp3|m4a|m4b|m4p|ogg|oga|opus|wma|wav|wv|flac|ape|aif|aiff|aifc)$"
|
||||||
)
|
)
|
||||||
|
|
||||||
var SAMPLEMATCH = `(?i)(^|[\\/]|\s|[._-])(sample|trailer|thumb|special|extras?)s?(\s|[._-]|$|/)`
|
var SAMPLEMATCH = `(?i)(^|[\\/]|\s|[(])(?:sample|trailer|thumb|special|extras?)s?([\s._\-/)]|$)`
|
||||||
|
|
||||||
func RegexMatch(regex string, value string) bool {
|
func RegexMatch(regex string, value string) bool {
|
||||||
re := regexp.MustCompile(regex)
|
re := regexp.MustCompile(regex)
|
||||||
|
|||||||
@@ -5,7 +5,10 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"github.com/sirrobot01/debrid-blackhole/cmd/decypharr"
|
"github.com/sirrobot01/debrid-blackhole/cmd/decypharr"
|
||||||
"github.com/sirrobot01/debrid-blackhole/internal/config"
|
"github.com/sirrobot01/debrid-blackhole/internal/config"
|
||||||
|
"github.com/sirrobot01/debrid-blackhole/pkg/version"
|
||||||
"log"
|
"log"
|
||||||
|
"net/http"
|
||||||
|
_ "net/http/pprof" // registers pprof handlers
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
@@ -19,6 +22,15 @@ func main() {
|
|||||||
debug.PrintStack()
|
debug.PrintStack()
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
if version.GetInfo().Channel == "dev" {
|
||||||
|
log.Println("Running in dev mode")
|
||||||
|
go func() {
|
||||||
|
if err := http.ListenAndServe(":6060", nil); err != nil {
|
||||||
|
log.Fatalf("pprof server failed: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
var configPath string
|
var configPath string
|
||||||
flag.StringVar(&configPath, "config", "/data", "path to the data folder")
|
flag.StringVar(&configPath, "config", "/data", "path to the data folder")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|||||||
@@ -137,6 +137,10 @@ func (a *Arr) CleanupQueue() error {
|
|||||||
isMessedUp = true
|
isMessedUp = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
if strings.Contains(m.Title, "One or more episodes expected in this release were not imported or missing from the release") {
|
||||||
|
isMessedUp = true
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -228,7 +228,7 @@ func (ad *AllDebrid) CheckStatus(torrent *types.Torrent, isSymlink bool) (*types
|
|||||||
}
|
}
|
||||||
// Break out of the loop if the torrent is downloading.
|
// Break out of the loop if the torrent is downloading.
|
||||||
// This is necessary to prevent infinite loop since we moved to sync downloading and async processing
|
// This is necessary to prevent infinite loop since we moved to sync downloading and async processing
|
||||||
break
|
return torrent, nil
|
||||||
} else {
|
} else {
|
||||||
return torrent, fmt.Errorf("torrent: %s has error", torrent.Name)
|
return torrent, fmt.Errorf("torrent: %s has error", torrent.Name)
|
||||||
}
|
}
|
||||||
|
|||||||
+173
-152
@@ -126,7 +126,7 @@ func New(dc config.Debrid, client types.Client) *Cache {
|
|||||||
folderNaming: WebDavFolderNaming(dc.FolderNaming),
|
folderNaming: WebDavFolderNaming(dc.FolderNaming),
|
||||||
autoExpiresLinksAfter: autoExpiresLinksAfter,
|
autoExpiresLinksAfter: autoExpiresLinksAfter,
|
||||||
repairsInProgress: xsync.NewMapOf[string, bool](),
|
repairsInProgress: xsync.NewMapOf[string, bool](),
|
||||||
saveSemaphore: make(chan struct{}, 10),
|
saveSemaphore: make(chan struct{}, 50),
|
||||||
ctx: context.Background(),
|
ctx: context.Background(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -159,53 +159,6 @@ func (c *Cache) Start(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Cache) GetTorrentFolder(torrent *types.Torrent) string {
|
|
||||||
switch c.folderNaming {
|
|
||||||
case WebDavUseFileName:
|
|
||||||
return torrent.Filename
|
|
||||||
case WebDavUseOriginalName:
|
|
||||||
return torrent.OriginalFilename
|
|
||||||
case WebDavUseFileNameNoExt:
|
|
||||||
return utils.RemoveExtension(torrent.Filename)
|
|
||||||
case WebDavUseOriginalNameNoExt:
|
|
||||||
return utils.RemoveExtension(torrent.OriginalFilename)
|
|
||||||
case WebDavUseID:
|
|
||||||
return torrent.Id
|
|
||||||
default:
|
|
||||||
return torrent.Filename
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) setTorrent(t *CachedTorrent) {
|
|
||||||
c.torrents.Store(t.Id, t)
|
|
||||||
|
|
||||||
c.torrentsNames.Store(c.GetTorrentFolder(t.Torrent), t)
|
|
||||||
|
|
||||||
c.SaveTorrent(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) setTorrents(torrents map[string]*CachedTorrent) {
|
|
||||||
for _, t := range torrents {
|
|
||||||
c.torrents.Store(t.Id, t)
|
|
||||||
c.torrentsNames.Store(c.GetTorrentFolder(t.Torrent), t)
|
|
||||||
}
|
|
||||||
|
|
||||||
c.refreshListings()
|
|
||||||
|
|
||||||
c.SaveTorrents()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) GetListing() []os.FileInfo {
|
|
||||||
if v, ok := c.listings.Load().([]os.FileInfo); ok {
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) Close() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) load() (map[string]*CachedTorrent, error) {
|
func (c *Cache) load() (map[string]*CachedTorrent, error) {
|
||||||
torrents := make(map[string]*CachedTorrent)
|
torrents := make(map[string]*CachedTorrent)
|
||||||
if err := os.MkdirAll(c.dir, 0755); err != nil {
|
if err := os.MkdirAll(c.dir, 0755); err != nil {
|
||||||
@@ -238,13 +191,11 @@ func (c *Cache) load() (map[string]*CachedTorrent, error) {
|
|||||||
}
|
}
|
||||||
isComplete := true
|
isComplete := true
|
||||||
if len(ct.Files) != 0 {
|
if len(ct.Files) != 0 {
|
||||||
// We can assume the torrent is complete
|
// Check if all files are valid, if not, delete the file.json and remove from cache.
|
||||||
|
|
||||||
for _, f := range ct.Files {
|
for _, f := range ct.Files {
|
||||||
if f.Link == "" {
|
if !f.IsValid() {
|
||||||
c.logger.Debug().Msgf("Torrent %s is not complete, missing link for file %s", ct.Id, f.Name)
|
|
||||||
isComplete = false
|
isComplete = false
|
||||||
continue
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if isComplete {
|
if isComplete {
|
||||||
@@ -255,6 +206,9 @@ func (c *Cache) load() (map[string]*CachedTorrent, error) {
|
|||||||
ct.AddedOn = addedOn
|
ct.AddedOn = addedOn
|
||||||
ct.IsComplete = true
|
ct.IsComplete = true
|
||||||
torrents[ct.Id] = &ct
|
torrents[ct.Id] = &ct
|
||||||
|
} else {
|
||||||
|
// Delete the file if it's not complete
|
||||||
|
_ = os.Remove(filePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -263,100 +217,6 @@ func (c *Cache) load() (map[string]*CachedTorrent, error) {
|
|||||||
return torrents, nil
|
return torrents, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Cache) GetTorrents() map[string]*CachedTorrent {
|
|
||||||
torrents := make(map[string]*CachedTorrent)
|
|
||||||
c.torrents.Range(func(key string, value *CachedTorrent) bool {
|
|
||||||
torrents[key] = value
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
return torrents
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) GetTorrent(id string) *CachedTorrent {
|
|
||||||
if t, ok := c.torrents.Load(id); ok {
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) GetTorrentByName(name string) *CachedTorrent {
|
|
||||||
if t, ok := c.torrentsNames.Load(name); ok {
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) SaveTorrents() {
|
|
||||||
c.torrents.Range(func(key string, value *CachedTorrent) bool {
|
|
||||||
c.SaveTorrent(value)
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) SaveTorrent(ct *CachedTorrent) {
|
|
||||||
// Try to acquire semaphore without blocking
|
|
||||||
select {
|
|
||||||
case c.saveSemaphore <- struct{}{}:
|
|
||||||
go func() {
|
|
||||||
defer func() { <-c.saveSemaphore }()
|
|
||||||
c.saveTorrent(ct)
|
|
||||||
}()
|
|
||||||
default:
|
|
||||||
go c.saveTorrent(ct) // If the semaphore is full, just run the save in the background
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) saveTorrent(ct *CachedTorrent) {
|
|
||||||
data, err := json.MarshalIndent(ct, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
c.logger.Debug().Err(err).Msgf("Failed to marshal torrent: %s", ct.Id)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
fileName := ct.Torrent.Id + ".json"
|
|
||||||
filePath := filepath.Join(c.dir, fileName)
|
|
||||||
|
|
||||||
// Use a unique temporary filename for concurrent safety
|
|
||||||
tmpFile := filePath + ".tmp." + strconv.FormatInt(time.Now().UnixNano(), 10)
|
|
||||||
|
|
||||||
f, err := os.Create(tmpFile)
|
|
||||||
if err != nil {
|
|
||||||
c.logger.Debug().Err(err).Msgf("Failed to create file: %s", tmpFile)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Track if we've closed the file
|
|
||||||
fileClosed := false
|
|
||||||
defer func() {
|
|
||||||
// Only close if not already closed
|
|
||||||
if !fileClosed {
|
|
||||||
_ = f.Close()
|
|
||||||
}
|
|
||||||
// Clean up the temp file if it still exists and rename failed
|
|
||||||
_ = os.Remove(tmpFile)
|
|
||||||
}()
|
|
||||||
|
|
||||||
w := bufio.NewWriter(f)
|
|
||||||
if _, err := w.Write(data); err != nil {
|
|
||||||
c.logger.Debug().Err(err).Msgf("Failed to write data: %s", tmpFile)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := w.Flush(); err != nil {
|
|
||||||
c.logger.Debug().Err(err).Msgf("Failed to flush data: %s", tmpFile)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close the file before renaming
|
|
||||||
_ = f.Close()
|
|
||||||
fileClosed = true
|
|
||||||
|
|
||||||
if err := os.Rename(tmpFile, filePath); err != nil {
|
|
||||||
c.logger.Debug().Err(err).Msgf("Failed to rename file: %s", tmpFile)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cache) Sync() error {
|
func (c *Cache) Sync() error {
|
||||||
cachedTorrents, err := c.load()
|
cachedTorrents, err := c.load()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -414,7 +274,7 @@ func (c *Cache) Sync() error {
|
|||||||
func (c *Cache) sync(torrents []*types.Torrent) error {
|
func (c *Cache) sync(torrents []*types.Torrent) error {
|
||||||
|
|
||||||
// Create channels with appropriate buffering
|
// Create channels with appropriate buffering
|
||||||
workChan := make(chan *types.Torrent, min(1000, len(torrents)))
|
workChan := make(chan *types.Torrent, min(c.workers, len(torrents)))
|
||||||
|
|
||||||
// Use an atomic counter for progress tracking
|
// Use an atomic counter for progress tracking
|
||||||
var processed int64
|
var processed int64
|
||||||
@@ -474,6 +334,157 @@ func (c *Cache) sync(torrents []*types.Torrent) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetTorrentFolder(torrent *types.Torrent) string {
|
||||||
|
switch c.folderNaming {
|
||||||
|
case WebDavUseFileName:
|
||||||
|
return torrent.Filename
|
||||||
|
case WebDavUseOriginalName:
|
||||||
|
return torrent.OriginalFilename
|
||||||
|
case WebDavUseFileNameNoExt:
|
||||||
|
return utils.RemoveExtension(torrent.Filename)
|
||||||
|
case WebDavUseOriginalNameNoExt:
|
||||||
|
return utils.RemoveExtension(torrent.OriginalFilename)
|
||||||
|
case WebDavUseID:
|
||||||
|
return torrent.Id
|
||||||
|
default:
|
||||||
|
return torrent.Filename
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) setTorrent(t *CachedTorrent) {
|
||||||
|
c.torrents.Store(t.Id, t)
|
||||||
|
|
||||||
|
c.torrentsNames.Store(c.GetTorrentFolder(t.Torrent), t)
|
||||||
|
|
||||||
|
c.SaveTorrent(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) setTorrents(torrents map[string]*CachedTorrent) {
|
||||||
|
for _, t := range torrents {
|
||||||
|
c.torrents.Store(t.Id, t)
|
||||||
|
c.torrentsNames.Store(c.GetTorrentFolder(t.Torrent), t)
|
||||||
|
}
|
||||||
|
|
||||||
|
c.refreshListings()
|
||||||
|
|
||||||
|
c.SaveTorrents()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetListing() []os.FileInfo {
|
||||||
|
if v, ok := c.listings.Load().([]os.FileInfo); ok {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetTorrents() map[string]*CachedTorrent {
|
||||||
|
torrents := make(map[string]*CachedTorrent)
|
||||||
|
c.torrents.Range(func(key string, value *CachedTorrent) bool {
|
||||||
|
torrents[key] = value
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
return torrents
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetTorrent(id string) *CachedTorrent {
|
||||||
|
if t, ok := c.torrents.Load(id); ok {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetTorrentByName(name string) *CachedTorrent {
|
||||||
|
if t, ok := c.torrentsNames.Load(name); ok {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) SaveTorrents() {
|
||||||
|
c.torrents.Range(func(key string, value *CachedTorrent) bool {
|
||||||
|
c.SaveTorrent(value)
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) SaveTorrent(ct *CachedTorrent) {
|
||||||
|
marshaled, err := json.MarshalIndent(ct, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
c.logger.Debug().Err(err).Msgf("Failed to marshal torrent: %s", ct.Id)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store just the essential info needed for the file operation
|
||||||
|
saveInfo := struct {
|
||||||
|
id string
|
||||||
|
jsonData []byte
|
||||||
|
}{
|
||||||
|
id: ct.Torrent.Id,
|
||||||
|
jsonData: marshaled,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to acquire semaphore without blocking
|
||||||
|
select {
|
||||||
|
case c.saveSemaphore <- struct{}{}:
|
||||||
|
go func() {
|
||||||
|
defer func() { <-c.saveSemaphore }()
|
||||||
|
c.saveTorrent(saveInfo.id, saveInfo.jsonData)
|
||||||
|
}()
|
||||||
|
default:
|
||||||
|
c.saveTorrent(saveInfo.id, saveInfo.jsonData)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) saveTorrent(id string, data []byte) {
|
||||||
|
|
||||||
|
fileName := id + ".json"
|
||||||
|
filePath := filepath.Join(c.dir, fileName)
|
||||||
|
|
||||||
|
// Use a unique temporary filename for concurrent safety
|
||||||
|
tmpFile := filePath + ".tmp." + strconv.FormatInt(time.Now().UnixNano(), 10)
|
||||||
|
|
||||||
|
f, err := os.Create(tmpFile)
|
||||||
|
if err != nil {
|
||||||
|
c.logger.Debug().Err(err).Msgf("Failed to create file: %s", tmpFile)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track if we've closed the file
|
||||||
|
fileClosed := false
|
||||||
|
defer func() {
|
||||||
|
// Only close if not already closed
|
||||||
|
if !fileClosed {
|
||||||
|
_ = f.Close()
|
||||||
|
}
|
||||||
|
// Clean up the temp file if it still exists and rename failed
|
||||||
|
_ = os.Remove(tmpFile)
|
||||||
|
}()
|
||||||
|
|
||||||
|
w := bufio.NewWriter(f)
|
||||||
|
if _, err := w.Write(data); err != nil {
|
||||||
|
c.logger.Debug().Err(err).Msgf("Failed to write data: %s", tmpFile)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := w.Flush(); err != nil {
|
||||||
|
c.logger.Debug().Err(err).Msgf("Failed to flush data: %s", tmpFile)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close the file before renaming
|
||||||
|
_ = f.Close()
|
||||||
|
fileClosed = true
|
||||||
|
|
||||||
|
if err := os.Rename(tmpFile, filePath); err != nil {
|
||||||
|
c.logger.Debug().Err(err).Msgf("Failed to rename file: %s", tmpFile)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Cache) ProcessTorrent(t *types.Torrent, refreshRclone bool) error {
|
func (c *Cache) ProcessTorrent(t *types.Torrent, refreshRclone bool) error {
|
||||||
if len(t.Files) == 0 {
|
if len(t.Files) == 0 {
|
||||||
if err := c.client.UpdateTorrent(t); err != nil {
|
if err := c.client.UpdateTorrent(t); err != nil {
|
||||||
@@ -481,15 +492,21 @@ func (c *Cache) ProcessTorrent(t *types.Torrent, refreshRclone bool) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Validate each file in the torrent
|
// Validate each file in the torrent
|
||||||
|
isComplete := true
|
||||||
for _, file := range t.Files {
|
for _, file := range t.Files {
|
||||||
if file.Link == "" {
|
if file.Link == "" {
|
||||||
c.logger.Debug().Msgf("Torrent %s is not complete, missing link for file %s. Triggering a reinsert", t.Id, file.Name)
|
isComplete = false
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !isComplete {
|
||||||
|
c.logger.Debug().Msgf("Torrent %s is not complete, missing link for file %s. Triggering a reinsert", t.Id)
|
||||||
if err := c.ReInsertTorrent(t); err != nil {
|
if err := c.ReInsertTorrent(t); err != nil {
|
||||||
c.logger.Error().Err(err).Msgf("Failed to reinsert torrent %s", t.Id)
|
c.logger.Error().Err(err).Msgf("Failed to reinsert torrent %s", t.Id)
|
||||||
return fmt.Errorf("failed to reinsert torrent: %w", err)
|
return fmt.Errorf("failed to reinsert torrent: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
addedOn, err := time.Parse(time.RFC3339, t.Added)
|
addedOn, err := time.Parse(time.RFC3339, t.Added)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -535,6 +552,8 @@ func (c *Cache) GetDownloadLink(torrentId, filename, fileLink string) string {
|
|||||||
downloadLink, err := c.client.GetDownloadLink(ct.Torrent, &file)
|
downloadLink, err := c.client.GetDownloadLink(ct.Torrent, &file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, request.HosterUnavailableError) {
|
if errors.Is(err, request.HosterUnavailableError) {
|
||||||
|
c.logger.Debug().Err(err).Msgf("Hoster is unavailable for %s/%s", ct.Name, filename)
|
||||||
|
return ""
|
||||||
// This code is commented iut due to the fact that if a torrent link is uncached, it's likely that we can't redownload it again
|
// This code is commented iut due to the fact that if a torrent link is uncached, it's likely that we can't redownload it again
|
||||||
// Do not attempt to repair the torrent if the hoster is unavailable
|
// Do not attempt to repair the torrent if the hoster is unavailable
|
||||||
// Check link here??
|
// Check link here??
|
||||||
@@ -565,8 +584,10 @@ func (c *Cache) GetDownloadLink(torrentId, filename, fileLink string) string {
|
|||||||
file.Generated = time.Now()
|
file.Generated = time.Now()
|
||||||
ct.Files[filename] = file
|
ct.Files[filename] = file
|
||||||
|
|
||||||
go c.updateDownloadLink(file.Link, downloadLink)
|
go func() {
|
||||||
go c.setTorrent(ct)
|
c.updateDownloadLink(file.Link, downloadLink)
|
||||||
|
c.setTorrent(ct)
|
||||||
|
}()
|
||||||
return file.DownloadLink
|
return file.DownloadLink
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ func (c *Cache) refreshListings() {
|
|||||||
c.listings.Store(files)
|
c.listings.Store(files)
|
||||||
_ = c.refreshXml()
|
_ = c.refreshXml()
|
||||||
if err := c.RefreshRclone(); err != nil {
|
if err := c.RefreshRclone(); err != nil {
|
||||||
c.logger.Debug().Err(err).Msg("Failed to refresh rclone")
|
c.logger.Trace().Err(err).Msg("Failed to refresh rclone") // silent error
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -131,6 +131,7 @@ func (c *Cache) ReInsertTorrent(torrent *types.Torrent) error {
|
|||||||
|
|
||||||
oldID := torrent.Id
|
oldID := torrent.Id
|
||||||
defer c.repairsInProgress.Delete(oldID)
|
defer c.repairsInProgress.Delete(oldID)
|
||||||
|
defer c.DeleteTorrent(oldID)
|
||||||
|
|
||||||
// Submit the magnet to the debrid service
|
// Submit the magnet to the debrid service
|
||||||
torrent.Id = ""
|
torrent.Id = ""
|
||||||
@@ -138,7 +139,6 @@ func (c *Cache) ReInsertTorrent(torrent *types.Torrent) error {
|
|||||||
torrent, err = c.client.SubmitMagnet(torrent)
|
torrent, err = c.client.SubmitMagnet(torrent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Remove the old torrent from the cache and debrid service
|
// Remove the old torrent from the cache and debrid service
|
||||||
_ = c.DeleteTorrent(oldID)
|
|
||||||
return fmt.Errorf("failed to submit magnet: %w", err)
|
return fmt.Errorf("failed to submit magnet: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -150,21 +150,24 @@ func (c *Cache) ReInsertTorrent(torrent *types.Torrent) error {
|
|||||||
torrent, err = c.client.CheckStatus(torrent, true)
|
torrent, err = c.client.CheckStatus(torrent, true)
|
||||||
if err != nil && torrent != nil {
|
if err != nil && torrent != nil {
|
||||||
// Torrent is likely in progress
|
// Torrent is likely in progress
|
||||||
// Delete the old and new torrent
|
|
||||||
_ = c.DeleteTorrent(oldID)
|
|
||||||
_ = c.DeleteTorrent(torrent.Id)
|
_ = c.DeleteTorrent(torrent.Id)
|
||||||
|
|
||||||
return fmt.Errorf("failed to check status: %w", err)
|
return fmt.Errorf("failed to check status: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.DeleteTorrent(oldID); err != nil {
|
|
||||||
return fmt.Errorf("failed to delete old torrent: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if torrent == nil {
|
if torrent == nil {
|
||||||
return fmt.Errorf("failed to check status: empty torrent")
|
return fmt.Errorf("failed to check status: empty torrent")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, file := range torrent.Files {
|
||||||
|
if file.Link == "" {
|
||||||
|
c.logger.Debug().Msgf("Torrent %s is still not complete, missing link for file %s.", torrent.Name, file.Name)
|
||||||
|
// Delete the torrent from the cache
|
||||||
|
_ = c.DeleteTorrent(torrent.Id)
|
||||||
|
return fmt.Errorf("torrent %s is still not complete, missing link for file %s", torrent.Name, file.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Update the torrent in the cache
|
// Update the torrent in the cache
|
||||||
addedOn, err := time.Parse(time.RFC3339, torrent.Added)
|
addedOn, err := time.Parse(time.RFC3339, torrent.Added)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -215,7 +215,7 @@ func (dl *DebridLink) CheckStatus(torrent *types.Torrent, isSymlink bool) (*type
|
|||||||
}
|
}
|
||||||
// Break out of the loop if the torrent is downloading.
|
// Break out of the loop if the torrent is downloading.
|
||||||
// This is necessary to prevent infinite loop since we moved to sync downloading and async processing
|
// This is necessary to prevent infinite loop since we moved to sync downloading and async processing
|
||||||
break
|
return torrent, nil
|
||||||
} else {
|
} else {
|
||||||
return torrent, fmt.Errorf("torrent: %s has error", torrent.Name)
|
return torrent, fmt.Errorf("torrent: %s has error", torrent.Name)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -244,6 +244,7 @@ func (r *RealDebrid) CheckStatus(t *types.Torrent, isSymlink bool) (*types.Torre
|
|||||||
if !t.DownloadUncached {
|
if !t.DownloadUncached {
|
||||||
return t, fmt.Errorf("torrent: %s not cached", t.Name)
|
return t, fmt.Errorf("torrent: %s not cached", t.Name)
|
||||||
}
|
}
|
||||||
|
return t, nil
|
||||||
} else {
|
} else {
|
||||||
return t, fmt.Errorf("torrent: %s has error: %s", t.Name, status)
|
return t, fmt.Errorf("torrent: %s has error: %s", t.Name, status)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -254,7 +254,7 @@ func (tb *Torbox) CheckStatus(torrent *types.Torrent, isSymlink bool) (*types.To
|
|||||||
}
|
}
|
||||||
// Break out of the loop if the torrent is downloading.
|
// Break out of the loop if the torrent is downloading.
|
||||||
// This is necessary to prevent infinite loop since we moved to sync downloading and async processing
|
// This is necessary to prevent infinite loop since we moved to sync downloading and async processing
|
||||||
break
|
return torrent, nil
|
||||||
} else {
|
} else {
|
||||||
return torrent, fmt.Errorf("torrent: %s has error", torrent.Name)
|
return torrent, fmt.Errorf("torrent: %s has error", torrent.Name)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package types
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/sirrobot01/debrid-blackhole/internal/config"
|
||||||
"github.com/sirrobot01/debrid-blackhole/internal/logger"
|
"github.com/sirrobot01/debrid-blackhole/internal/logger"
|
||||||
"github.com/sirrobot01/debrid-blackhole/internal/utils"
|
"github.com/sirrobot01/debrid-blackhole/internal/utils"
|
||||||
"github.com/sirrobot01/debrid-blackhole/pkg/arr"
|
"github.com/sirrobot01/debrid-blackhole/pkg/arr"
|
||||||
@@ -80,6 +81,26 @@ type File struct {
|
|||||||
Generated time.Time `json:"generated"`
|
Generated time.Time `json:"generated"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *File) IsValid() bool {
|
||||||
|
cfg := config.Get()
|
||||||
|
name := filepath.Base(f.Path)
|
||||||
|
if utils.IsSampleFile(f.Path) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if !cfg.IsAllowedFile(name) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !cfg.IsSizeAllowed(f.Size) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if f.Link == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func (t *Torrent) Cleanup(remove bool) {
|
func (t *Torrent) Cleanup(remove bool) {
|
||||||
if remove {
|
if remove {
|
||||||
err := os.Remove(t.Filename)
|
err := os.Remove(t.Filename)
|
||||||
|
|||||||
+32
-19
@@ -163,6 +163,7 @@ func (q *QBit) createSymlinks(debridTorrent *debrid.Torrent, rclonePath, torrent
|
|||||||
}
|
}
|
||||||
ticker := time.NewTicker(100 * time.Millisecond)
|
ticker := time.NewTicker(100 * time.Millisecond)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
filePaths := make([]string, 0, len(pending))
|
||||||
|
|
||||||
for len(pending) > 0 {
|
for len(pending) > 0 {
|
||||||
<-ticker.C
|
<-ticker.C
|
||||||
@@ -170,11 +171,24 @@ func (q *QBit) createSymlinks(debridTorrent *debrid.Torrent, rclonePath, torrent
|
|||||||
fullFilePath := filepath.Join(rclonePath, file.Path)
|
fullFilePath := filepath.Join(rclonePath, file.Path)
|
||||||
if _, err := os.Stat(fullFilePath); !os.IsNotExist(err) {
|
if _, err := os.Stat(fullFilePath); !os.IsNotExist(err) {
|
||||||
q.logger.Info().Msgf("File is ready: %s", file.Path)
|
q.logger.Info().Msgf("File is ready: %s", file.Path)
|
||||||
q.createSymLink(torrentSymlinkPath, rclonePath, file)
|
_filePath := q.createSymLink(torrentSymlinkPath, rclonePath, file)
|
||||||
|
filePaths = append(filePaths, _filePath)
|
||||||
delete(pending, path)
|
delete(pending, path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if q.SkipPreCache {
|
||||||
|
return torrentSymlinkPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
|
||||||
|
if err := q.preCacheFile(debridTorrent.Name, filePaths); err != nil {
|
||||||
|
q.logger.Error().Msgf("Failed to pre-cache file: %s", err)
|
||||||
|
}
|
||||||
|
}() // Pre-cache the files in the background
|
||||||
|
// Pre-cache the first 256KB and 1MB of the file
|
||||||
return torrentSymlinkPath, nil
|
return torrentSymlinkPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -189,7 +203,7 @@ func (q *QBit) getTorrentPath(rclonePath string, debridTorrent *debrid.Torrent)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (q *QBit) createSymLink(path string, torrentMountPath string, file debrid.File) {
|
func (q *QBit) createSymLink(path string, torrentMountPath string, file debrid.File) string {
|
||||||
|
|
||||||
// Combine the directory and filename to form a full path
|
// Combine the directory and filename to form a full path
|
||||||
fullPath := filepath.Join(path, file.Name) // /mnt/symlinks/{category}/MyTVShow/MyTVShow.S01E01.720p.mkv
|
fullPath := filepath.Join(path, file.Name) // /mnt/symlinks/{category}/MyTVShow/MyTVShow.S01E01.720p.mkv
|
||||||
@@ -200,28 +214,27 @@ func (q *QBit) createSymLink(path string, torrentMountPath string, file debrid.F
|
|||||||
// It's okay if the symlink already exists
|
// It's okay if the symlink already exists
|
||||||
q.logger.Debug().Msgf("Failed to create symlink: %s: %v", fullPath, err)
|
q.logger.Debug().Msgf("Failed to create symlink: %s: %v", fullPath, err)
|
||||||
}
|
}
|
||||||
if q.SkipPreCache {
|
return torrentFilePath
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *QBit) preCacheFile(name string, filePaths []string) error {
|
||||||
|
q.logger.Trace().Msgf("Pre-caching file: %s", name)
|
||||||
|
if len(filePaths) == 0 {
|
||||||
|
return fmt.Errorf("no file paths provided")
|
||||||
|
}
|
||||||
|
for _, filePath := range filePaths {
|
||||||
|
func() {
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
defer file.Close()
|
||||||
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
go func() {
|
|
||||||
err := q.preCacheFile(torrentFilePath)
|
|
||||||
if err != nil {
|
|
||||||
q.logger.Debug().Msgf("Failed to pre-cache file: %s: %v", torrentFilePath, err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *QBit) preCacheFile(filePath string) error {
|
|
||||||
q.logger.Trace().Msgf("Pre-caching file: %s", filePath)
|
|
||||||
file, err := os.Open(filePath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error opening file: %v", err)
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
// Pre-cache the file header (first 256KB) using 16KB chunks.
|
// Pre-cache the file header (first 256KB) using 16KB chunks.
|
||||||
q.readSmallChunks(file, 0, 256*1024, 16*1024)
|
q.readSmallChunks(file, 0, 256*1024, 16*1024)
|
||||||
q.readSmallChunks(file, 1024*1024, 64*1024, 16*1024)
|
q.readSmallChunks(file, 1024*1024, 64*1024, 16*1024)
|
||||||
|
return
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-2
@@ -124,8 +124,9 @@ func (q *QBit) ProcessFiles(torrent *Torrent, debridTorrent *debrid.Torrent, arr
|
|||||||
q.MarkAsFailed(torrent)
|
q.MarkAsFailed(torrent)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
rclonePath := filepath.Join(debridTorrent.MountPath, cache.GetTorrentFolder(debridTorrent))
|
rclonePath := filepath.Join(debridTorrent.MountPath, cache.GetTorrentFolder(debridTorrent)) // /mnt/remote/realdebrid/MyTVShow
|
||||||
torrentSymlinkPath, err = q.createSymlinks(debridTorrent, rclonePath, debridTorrent.Name)
|
torrentFolderNoExt := utils.RemoveExtension(debridTorrent.Name)
|
||||||
|
torrentSymlinkPath, err = q.createSymlinks(debridTorrent, rclonePath, torrentFolderNoExt) // /mnt/symlinks/{category}/MyTVShow/
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
// User is using either zurg or debrid webdav
|
// User is using either zurg or debrid webdav
|
||||||
|
|||||||
Reference in New Issue
Block a user