- Revamp decypharr arch \n
- Add callback_ur, download_folder to addContent API \n - Fix few bugs \n - More declarative UI keywords - Speed up repairs - Few other improvements/bug fixes
This commit is contained in:
Mukhtar Akere
@@ -0,0 +1,320 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/sirrobot01/decypharr/pkg/debrid/types"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cavaliergopher/grab/v3"
|
||||
"github.com/sirrobot01/decypharr/internal/utils"
|
||||
)
|
||||
|
||||
func Download(client *grab.Client, url, filename string, byterange *[2]int64, progressCallback func(int64, int64)) error {
|
||||
req, err := grab.NewRequest(filename, url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set byte range if specified
|
||||
if byterange != nil {
|
||||
byterangeStr := fmt.Sprintf("%d-%d", byterange[0], byterange[1])
|
||||
req.HTTPRequest.Header.Set("Range", "bytes="+byterangeStr)
|
||||
}
|
||||
|
||||
resp := client.Do(req)
|
||||
|
||||
t := time.NewTicker(time.Second * 2)
|
||||
defer t.Stop()
|
||||
|
||||
var lastReported int64
|
||||
Loop:
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
current := resp.BytesComplete()
|
||||
speed := int64(resp.BytesPerSecond())
|
||||
if current != lastReported {
|
||||
if progressCallback != nil {
|
||||
progressCallback(current-lastReported, speed)
|
||||
}
|
||||
lastReported = current
|
||||
}
|
||||
case <-resp.Done:
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
|
||||
// Report final bytes
|
||||
if progressCallback != nil {
|
||||
progressCallback(resp.BytesComplete()-lastReported, 0)
|
||||
}
|
||||
|
||||
return resp.Err()
|
||||
}
|
||||
|
||||
func (s *Store) ProcessManualFile(torrent *Torrent) (string, error) {
|
||||
debridTorrent := torrent.DebridTorrent
|
||||
s.logger.Info().Msgf("Downloading %d files...", len(debridTorrent.Files))
|
||||
torrentPath := filepath.Join(torrent.SavePath, utils.RemoveExtension(debridTorrent.OriginalFilename))
|
||||
torrentPath = utils.RemoveInvalidChars(torrentPath)
|
||||
err := os.MkdirAll(torrentPath, os.ModePerm)
|
||||
if err != nil {
|
||||
// add the previous error to the error and return
|
||||
return "", fmt.Errorf("failed to create directory: %s: %v", torrentPath, err)
|
||||
}
|
||||
s.downloadFiles(torrent, torrentPath)
|
||||
return torrentPath, nil
|
||||
}
|
||||
|
||||
func (s *Store) downloadFiles(torrent *Torrent, parent string) {
|
||||
debridTorrent := torrent.DebridTorrent
|
||||
var wg sync.WaitGroup
|
||||
|
||||
totalSize := int64(0)
|
||||
for _, file := range debridTorrent.GetFiles() {
|
||||
totalSize += file.Size
|
||||
}
|
||||
debridTorrent.Lock()
|
||||
debridTorrent.SizeDownloaded = 0 // Reset downloaded bytes
|
||||
debridTorrent.Progress = 0 // Reset progress
|
||||
debridTorrent.Unlock()
|
||||
progressCallback := func(downloaded int64, speed int64) {
|
||||
debridTorrent.Lock()
|
||||
defer debridTorrent.Unlock()
|
||||
torrent.Lock()
|
||||
defer torrent.Unlock()
|
||||
|
||||
// Update total downloaded bytes
|
||||
debridTorrent.SizeDownloaded += downloaded
|
||||
debridTorrent.Speed = speed
|
||||
|
||||
// Calculate overall progress
|
||||
if totalSize > 0 {
|
||||
debridTorrent.Progress = float64(debridTorrent.SizeDownloaded) / float64(totalSize) * 100
|
||||
}
|
||||
s.UpdateTorrentMin(torrent, debridTorrent)
|
||||
}
|
||||
client := &grab.Client{
|
||||
UserAgent: "Decypharr[QBitTorrent]",
|
||||
HTTPClient: &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
},
|
||||
},
|
||||
}
|
||||
errChan := make(chan error, len(debridTorrent.Files))
|
||||
for _, file := range debridTorrent.GetFiles() {
|
||||
if file.DownloadLink == nil {
|
||||
s.logger.Info().Msgf("No download link found for %s", file.Name)
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
s.downloadSemaphore <- struct{}{}
|
||||
go func(file types.File) {
|
||||
defer wg.Done()
|
||||
defer func() { <-s.downloadSemaphore }()
|
||||
filename := file.Name
|
||||
|
||||
err := Download(
|
||||
client,
|
||||
file.DownloadLink.DownloadLink,
|
||||
filepath.Join(parent, filename),
|
||||
file.ByteRange,
|
||||
progressCallback,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
s.logger.Error().Msgf("Failed to download %s: %v", filename, err)
|
||||
errChan <- err
|
||||
} else {
|
||||
s.logger.Info().Msgf("Downloaded %s", filename)
|
||||
}
|
||||
}(file)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
close(errChan)
|
||||
var errors []error
|
||||
for err := range errChan {
|
||||
if err != nil {
|
||||
errors = append(errors, err)
|
||||
}
|
||||
}
|
||||
if len(errors) > 0 {
|
||||
s.logger.Error().Msgf("Errors occurred during download: %v", errors)
|
||||
return
|
||||
}
|
||||
s.logger.Info().Msgf("Downloaded all files for %s", debridTorrent.Name)
|
||||
}
|
||||
|
||||
func (s *Store) ProcessSymlink(torrent *Torrent) (string, error) {
|
||||
debridTorrent := torrent.DebridTorrent
|
||||
files := debridTorrent.Files
|
||||
if len(files) == 0 {
|
||||
return "", fmt.Errorf("no video files found")
|
||||
}
|
||||
s.logger.Info().Msgf("Checking symlinks for %d files...", len(files))
|
||||
rCloneBase := debridTorrent.MountPath
|
||||
torrentPath, err := s.getTorrentPath(rCloneBase, debridTorrent) // /MyTVShow/
|
||||
// This returns filename.ext for alldebrid instead of the parent folder filename/
|
||||
torrentFolder := torrentPath
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get torrent path: %v", err)
|
||||
}
|
||||
// Check if the torrent path is a file
|
||||
torrentRclonePath := filepath.Join(rCloneBase, torrentPath) // leave it as is
|
||||
if debridTorrent.Debrid == "alldebrid" && utils.IsMediaFile(torrentPath) {
|
||||
// Alldebrid hotfix for single file torrents
|
||||
torrentFolder = utils.RemoveExtension(torrentFolder)
|
||||
torrentRclonePath = rCloneBase // /mnt/rclone/magnets/ // Remove the filename since it's in the root folder
|
||||
}
|
||||
torrentSymlinkPath := filepath.Join(torrent.SavePath, torrentFolder) // /mnt/symlinks/{category}/MyTVShow/
|
||||
err = os.MkdirAll(torrentSymlinkPath, os.ModePerm)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create directory: %s: %v", torrentSymlinkPath, err)
|
||||
}
|
||||
|
||||
realPaths := make(map[string]string)
|
||||
err = filepath.WalkDir(torrentRclonePath, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if !d.IsDir() {
|
||||
filename := d.Name()
|
||||
rel, _ := filepath.Rel(torrentRclonePath, path)
|
||||
realPaths[filename] = rel
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
s.logger.Warn().Msgf("Error while scanning rclone path: %v", err)
|
||||
}
|
||||
|
||||
pending := make(map[string]types.File)
|
||||
for _, file := range files {
|
||||
if realRelPath, ok := realPaths[file.Name]; ok {
|
||||
file.Path = realRelPath
|
||||
}
|
||||
pending[file.Path] = file
|
||||
}
|
||||
ticker := time.NewTicker(200 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
timeout := time.After(30 * time.Minute)
|
||||
filePaths := make([]string, 0, len(pending))
|
||||
|
||||
for len(pending) > 0 {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
for path, file := range pending {
|
||||
fullFilePath := filepath.Join(torrentRclonePath, file.Path)
|
||||
if _, err := os.Stat(fullFilePath); !os.IsNotExist(err) {
|
||||
fileSymlinkPath := filepath.Join(torrentSymlinkPath, file.Name)
|
||||
if err := os.Symlink(fullFilePath, fileSymlinkPath); err != nil && !os.IsExist(err) {
|
||||
s.logger.Debug().Msgf("Failed to create symlink: %s: %v", fileSymlinkPath, err)
|
||||
} else {
|
||||
filePaths = append(filePaths, fileSymlinkPath)
|
||||
delete(pending, path)
|
||||
s.logger.Info().Msgf("File is ready: %s", file.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
case <-timeout:
|
||||
s.logger.Warn().Msgf("Timeout waiting for files, %d files still pending", len(pending))
|
||||
return torrentSymlinkPath, fmt.Errorf("timeout waiting for files: %d files still pending", len(pending))
|
||||
}
|
||||
}
|
||||
if s.skipPreCache {
|
||||
return torrentSymlinkPath, nil
|
||||
}
|
||||
|
||||
go func() {
|
||||
s.logger.Debug().Msgf("Pre-caching %s", debridTorrent.Name)
|
||||
if err := utils.PreCacheFile(filePaths); err != nil {
|
||||
s.logger.Error().Msgf("Failed to pre-cache file: %s", err)
|
||||
} else {
|
||||
s.logger.Trace().Msgf("Pre-cached %d files", len(filePaths))
|
||||
}
|
||||
}()
|
||||
return torrentSymlinkPath, nil
|
||||
}
|
||||
|
||||
func (s *Store) createSymlinksWebdav(torrent *Torrent, debridTorrent *types.Torrent, rclonePath, torrentFolder string) (string, error) {
|
||||
files := debridTorrent.Files
|
||||
symlinkPath := filepath.Join(torrent.SavePath, torrentFolder) // /mnt/symlinks/{category}/MyTVShow/
|
||||
err := os.MkdirAll(symlinkPath, os.ModePerm)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create directory: %s: %v", symlinkPath, err)
|
||||
}
|
||||
|
||||
remainingFiles := make(map[string]types.File)
|
||||
for _, file := range files {
|
||||
remainingFiles[file.Name] = file
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
timeout := time.After(30 * time.Minute)
|
||||
filePaths := make([]string, 0, len(files))
|
||||
|
||||
for len(remainingFiles) > 0 {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
entries, err := os.ReadDir(rclonePath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check which files exist in this batch
|
||||
for _, entry := range entries {
|
||||
filename := entry.Name()
|
||||
if file, exists := remainingFiles[filename]; exists {
|
||||
fullFilePath := filepath.Join(rclonePath, filename)
|
||||
fileSymlinkPath := filepath.Join(symlinkPath, file.Name)
|
||||
|
||||
if err := os.Symlink(fullFilePath, fileSymlinkPath); err != nil && !os.IsExist(err) {
|
||||
s.logger.Debug().Msgf("Failed to create symlink: %s: %v", fileSymlinkPath, err)
|
||||
} else {
|
||||
filePaths = append(filePaths, fileSymlinkPath)
|
||||
delete(remainingFiles, filename)
|
||||
s.logger.Info().Msgf("File is ready: %s", file.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case <-timeout:
|
||||
s.logger.Warn().Msgf("Timeout waiting for files, %d files still pending", len(remainingFiles))
|
||||
return symlinkPath, fmt.Errorf("timeout waiting for files")
|
||||
}
|
||||
}
|
||||
|
||||
if s.skipPreCache {
|
||||
return symlinkPath, nil
|
||||
}
|
||||
|
||||
go func() {
|
||||
s.logger.Debug().Msgf("Pre-caching %s", debridTorrent.Name)
|
||||
if err := utils.PreCacheFile(filePaths); err != nil {
|
||||
s.logger.Error().Msgf("Failed to pre-cache file: %s", err)
|
||||
} else {
|
||||
s.logger.Debug().Msgf("Pre-cached %d files", len(filePaths))
|
||||
}
|
||||
}() // Pre-cache the files in the background
|
||||
// Pre-cache the first 256KB and 1MB of the file
|
||||
return symlinkPath, nil
|
||||
}
|
||||
|
||||
func (s *Store) getTorrentPath(rclonePath string, debridTorrent *types.Torrent) (string, error) {
|
||||
for {
|
||||
torrentPath, err := debridTorrent.GetMountFolder(rclonePath)
|
||||
if err == nil {
|
||||
s.logger.Debug().Msgf("Found torrent path: %s", torrentPath)
|
||||
return torrentPath, err
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func createTorrentFromMagnet(req *ImportRequest) *Torrent {
|
||||
magnet := req.Magnet
|
||||
arrName := req.Arr.Name
|
||||
torrent := &Torrent{
|
||||
ID: "",
|
||||
Hash: strings.ToLower(magnet.InfoHash),
|
||||
Name: magnet.Name,
|
||||
Size: magnet.Size,
|
||||
Category: arrName,
|
||||
Source: string(req.Type),
|
||||
State: "downloading",
|
||||
MagnetUri: magnet.Link,
|
||||
|
||||
Tracker: "udp://tracker.opentrackr.org:1337",
|
||||
UpLimit: -1,
|
||||
DlLimit: -1,
|
||||
AutoTmm: false,
|
||||
Ratio: 1,
|
||||
RatioLimit: 1,
|
||||
SavePath: filepath.Join(req.DownloadFolder, arrName) + string(os.PathSeparator),
|
||||
}
|
||||
return torrent
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"github.com/sirrobot01/decypharr/internal/request"
|
||||
"github.com/sirrobot01/decypharr/internal/utils"
|
||||
"github.com/sirrobot01/decypharr/pkg/arr"
|
||||
debridTypes "github.com/sirrobot01/decypharr/pkg/debrid/types"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ImportType string
|
||||
|
||||
const (
|
||||
ImportTypeQBitTorrent ImportType = "qbit"
|
||||
ImportTypeAPI ImportType = "api"
|
||||
)
|
||||
|
||||
func NewImportRequest(debrid string, downloadFolder string, magnet *utils.Magnet, arr *arr.Arr, isSymlink, downloadUncached bool, callBackUrl string, importType ImportType) *ImportRequest {
|
||||
return &ImportRequest{
|
||||
Status: "started",
|
||||
DownloadFolder: downloadFolder,
|
||||
Debrid: debrid,
|
||||
Magnet: magnet,
|
||||
Arr: arr,
|
||||
IsSymlink: isSymlink,
|
||||
DownloadUncached: downloadUncached,
|
||||
CallBackUrl: callBackUrl,
|
||||
Type: importType,
|
||||
}
|
||||
}
|
||||
|
||||
type ImportRequest struct {
|
||||
DownloadFolder string `json:"downloadFolder"`
|
||||
Debrid string `json:"debrid"`
|
||||
Magnet *utils.Magnet `json:"magnet"`
|
||||
Arr *arr.Arr `json:"arr"`
|
||||
IsSymlink bool `json:"isSymlink"`
|
||||
DownloadUncached bool `json:"downloadUncached"`
|
||||
CallBackUrl string `json:"callBackUrl"`
|
||||
|
||||
Status string `json:"status"`
|
||||
CompletedAt time.Time `json:"completedAt,omitempty"`
|
||||
Error error `json:"error,omitempty"`
|
||||
|
||||
Type ImportType `json:"type"`
|
||||
Async bool `json:"async"`
|
||||
}
|
||||
|
||||
type importResponse struct {
|
||||
Status string `json:"status"`
|
||||
CompletedAt time.Time `json:"completedAt"`
|
||||
Error error `json:"error"`
|
||||
Torrent *Torrent `json:"torrent"`
|
||||
Debrid *debridTypes.Torrent `json:"debrid"`
|
||||
}
|
||||
|
||||
func (i *ImportRequest) sendCallback(torrent *Torrent, debridTorrent *debridTypes.Torrent) {
|
||||
if i.CallBackUrl == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if the callback URL is valid
|
||||
if _, err := url.ParseRequestURI(i.CallBackUrl); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
client := request.New()
|
||||
payload, err := json.Marshal(&importResponse{
|
||||
Status: i.Status,
|
||||
Error: i.Error,
|
||||
CompletedAt: i.CompletedAt,
|
||||
Torrent: torrent,
|
||||
Debrid: debridTorrent,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
req, err := http.NewRequest("POST", i.CallBackUrl, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
_, _ = client.Do(req)
|
||||
|
||||
}
|
||||
|
||||
func (i *ImportRequest) markAsFailed(err error, torrent *Torrent, debridTorrent *debridTypes.Torrent) {
|
||||
i.Status = "failed"
|
||||
i.Error = err
|
||||
i.CompletedAt = time.Now()
|
||||
i.sendCallback(torrent, debridTorrent)
|
||||
}
|
||||
|
||||
func (i *ImportRequest) markAsCompleted(torrent *Torrent, debridTorrent *debridTypes.Torrent) {
|
||||
i.Status = "completed"
|
||||
i.Error = nil
|
||||
i.CompletedAt = time.Now()
|
||||
i.sendCallback(torrent, debridTorrent)
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/sirrobot01/decypharr/internal/config"
|
||||
"github.com/sirrobot01/decypharr/internal/logger"
|
||||
"github.com/sirrobot01/decypharr/pkg/arr"
|
||||
"github.com/sirrobot01/decypharr/pkg/debrid"
|
||||
"github.com/sirrobot01/decypharr/pkg/repair"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
repair *repair.Repair
|
||||
arr *arr.Storage
|
||||
debrid *debrid.Storage
|
||||
torrents *TorrentStorage
|
||||
logger zerolog.Logger
|
||||
refreshInterval time.Duration
|
||||
skipPreCache bool
|
||||
downloadSemaphore chan struct{}
|
||||
}
|
||||
|
||||
var (
|
||||
instance *Store
|
||||
once sync.Once
|
||||
)
|
||||
|
||||
// GetStore returns the singleton instance
|
||||
func GetStore() *Store {
|
||||
once.Do(func() {
|
||||
arrs := arr.NewStorage()
|
||||
deb := debrid.NewStorage()
|
||||
cfg := config.Get()
|
||||
qbitCfg := cfg.QBitTorrent
|
||||
|
||||
instance = &Store{
|
||||
repair: repair.New(arrs, deb),
|
||||
arr: arrs,
|
||||
debrid: deb,
|
||||
torrents: newTorrentStorage(cfg.TorrentsFile()),
|
||||
logger: logger.New("store"),
|
||||
refreshInterval: time.Duration(cmp.Or(qbitCfg.RefreshInterval, 10)) * time.Minute,
|
||||
skipPreCache: qbitCfg.SkipPreCache,
|
||||
downloadSemaphore: make(chan struct{}, cmp.Or(qbitCfg.MaxDownloads, 5)),
|
||||
}
|
||||
})
|
||||
return instance
|
||||
}
|
||||
|
||||
func Reset() {
|
||||
if instance != nil {
|
||||
if instance.debrid != nil {
|
||||
instance.debrid.Reset()
|
||||
}
|
||||
close(instance.downloadSemaphore)
|
||||
}
|
||||
once = sync.Once{}
|
||||
instance = nil
|
||||
}
|
||||
|
||||
func (s *Store) GetArr() *arr.Storage {
|
||||
return s.arr
|
||||
}
|
||||
func (s *Store) GetDebrid() *debrid.Storage {
|
||||
return s.debrid
|
||||
}
|
||||
func (s *Store) GetRepair() *repair.Repair {
|
||||
return s.repair
|
||||
}
|
||||
func (s *Store) GetTorrentStorage() *TorrentStorage {
|
||||
return s.torrents
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/sirrobot01/decypharr/internal/request"
|
||||
"github.com/sirrobot01/decypharr/internal/utils"
|
||||
debridTypes "github.com/sirrobot01/decypharr/pkg/debrid"
|
||||
"github.com/sirrobot01/decypharr/pkg/debrid/types"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (s *Store) AddTorrent(ctx context.Context, importReq *ImportRequest) error {
|
||||
torrent := createTorrentFromMagnet(importReq)
|
||||
debridTorrent, err := debridTypes.ProcessTorrent(ctx, s.debrid, importReq.Debrid, importReq.Magnet, importReq.Arr, importReq.IsSymlink, importReq.DownloadUncached)
|
||||
if err != nil || debridTorrent == nil {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("failed to process torrent")
|
||||
}
|
||||
// This error is returned immediately to the user(no need for callback)
|
||||
return err
|
||||
}
|
||||
torrent = s.UpdateTorrentMin(torrent, debridTorrent)
|
||||
s.torrents.AddOrUpdate(torrent)
|
||||
go s.processFiles(torrent, debridTorrent, importReq) // We can send async for file processing not to delay the response
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) processFiles(torrent *Torrent, debridTorrent *types.Torrent, importReq *ImportRequest) {
|
||||
client := s.debrid.GetClient(debridTorrent.Debrid)
|
||||
downloadingStatuses := client.GetDownloadingStatus()
|
||||
_arr := importReq.Arr
|
||||
for debridTorrent.Status != "downloaded" {
|
||||
s.logger.Debug().Msgf("%s <- (%s) Download Progress: %.2f%%", debridTorrent.Debrid, debridTorrent.Name, debridTorrent.Progress)
|
||||
dbT, err := client.CheckStatus(debridTorrent, importReq.IsSymlink)
|
||||
if err != nil {
|
||||
if dbT != nil && dbT.Id != "" {
|
||||
// Delete the torrent if it was not downloaded
|
||||
go func() {
|
||||
_ = client.DeleteTorrent(dbT.Id)
|
||||
}()
|
||||
}
|
||||
s.logger.Error().Msgf("Error checking status: %v", err)
|
||||
s.markTorrentAsFailed(torrent)
|
||||
go func() {
|
||||
_arr.Refresh()
|
||||
}()
|
||||
importReq.markAsFailed(err, torrent, debridTorrent)
|
||||
return
|
||||
}
|
||||
|
||||
debridTorrent = dbT
|
||||
torrent = s.UpdateTorrentMin(torrent, debridTorrent)
|
||||
|
||||
// Exit the loop for downloading statuses to prevent memory buildup
|
||||
if debridTorrent.Status == "downloaded" || !utils.Contains(downloadingStatuses, debridTorrent.Status) {
|
||||
break
|
||||
}
|
||||
if !utils.Contains(client.GetDownloadingStatus(), debridTorrent.Status) {
|
||||
break
|
||||
}
|
||||
time.Sleep(s.refreshInterval)
|
||||
}
|
||||
var torrentSymlinkPath string
|
||||
var err error
|
||||
debridTorrent.Arr = _arr
|
||||
|
||||
// Check if debrid supports webdav by checking cache
|
||||
timer := time.Now()
|
||||
if importReq.IsSymlink {
|
||||
caches := s.debrid.GetCaches()
|
||||
cache, useWebdav := caches[debridTorrent.Debrid]
|
||||
if useWebdav {
|
||||
s.logger.Info().Msgf("Using internal webdav for %s", debridTorrent.Debrid)
|
||||
|
||||
// Use webdav to download the file
|
||||
|
||||
if err := cache.Add(debridTorrent); err != nil {
|
||||
s.logger.Error().Msgf("Error adding torrent to cache: %v", err)
|
||||
s.markTorrentAsFailed(torrent)
|
||||
importReq.markAsFailed(err, torrent, debridTorrent)
|
||||
return
|
||||
}
|
||||
|
||||
rclonePath := filepath.Join(debridTorrent.MountPath, cache.GetTorrentFolder(debridTorrent)) // /mnt/remote/realdebrid/MyTVShow
|
||||
torrentFolderNoExt := utils.RemoveExtension(debridTorrent.Name)
|
||||
torrentSymlinkPath, err = s.createSymlinksWebdav(torrent, debridTorrent, rclonePath, torrentFolderNoExt) // /mnt/symlinks/{category}/MyTVShow/
|
||||
|
||||
} else {
|
||||
// User is using either zurg or debrid webdav
|
||||
torrentSymlinkPath, err = s.ProcessSymlink(torrent) // /mnt/symlinks/{category}/MyTVShow/
|
||||
}
|
||||
} else {
|
||||
torrentSymlinkPath, err = s.ProcessManualFile(torrent)
|
||||
}
|
||||
if err != nil {
|
||||
s.markTorrentAsFailed(torrent)
|
||||
go func() {
|
||||
_ = client.DeleteTorrent(debridTorrent.Id)
|
||||
}()
|
||||
s.logger.Info().Msgf("Error: %v", err)
|
||||
importReq.markAsFailed(err, torrent, debridTorrent)
|
||||
return
|
||||
}
|
||||
torrent.TorrentPath = torrentSymlinkPath
|
||||
s.UpdateTorrent(torrent, debridTorrent)
|
||||
s.logger.Info().Msgf("Adding %s took %s", debridTorrent.Name, time.Since(timer))
|
||||
|
||||
go importReq.markAsCompleted(torrent, debridTorrent) // Mark the import request as completed, send callback if needed
|
||||
go func() {
|
||||
if err := request.SendDiscordMessage("download_complete", "success", torrent.discordContext()); err != nil {
|
||||
s.logger.Error().Msgf("Error sending discord message: %v", err)
|
||||
}
|
||||
}()
|
||||
_arr.Refresh()
|
||||
}
|
||||
|
||||
func (s *Store) markTorrentAsFailed(t *Torrent) *Torrent {
|
||||
t.State = "error"
|
||||
s.torrents.AddOrUpdate(t)
|
||||
go func() {
|
||||
if err := request.SendDiscordMessage("download_failed", "error", t.discordContext()); err != nil {
|
||||
s.logger.Error().Msgf("Error sending discord message: %v", err)
|
||||
}
|
||||
}()
|
||||
return t
|
||||
}
|
||||
|
||||
func (s *Store) UpdateTorrentMin(t *Torrent, debridTorrent *types.Torrent) *Torrent {
|
||||
if debridTorrent == nil {
|
||||
return t
|
||||
}
|
||||
|
||||
addedOn, err := time.Parse(time.RFC3339, debridTorrent.Added)
|
||||
if err != nil {
|
||||
addedOn = time.Now()
|
||||
}
|
||||
totalSize := debridTorrent.Bytes
|
||||
progress := (cmp.Or(debridTorrent.Progress, 0.0)) / 100.0
|
||||
sizeCompleted := int64(float64(totalSize) * progress)
|
||||
|
||||
var speed int64
|
||||
if debridTorrent.Speed != 0 {
|
||||
speed = debridTorrent.Speed
|
||||
}
|
||||
var eta int
|
||||
if speed != 0 {
|
||||
eta = int((totalSize - sizeCompleted) / speed)
|
||||
}
|
||||
t.ID = debridTorrent.Id
|
||||
t.Name = debridTorrent.Name
|
||||
t.AddedOn = addedOn.Unix()
|
||||
t.DebridTorrent = debridTorrent
|
||||
t.Debrid = debridTorrent.Debrid
|
||||
t.Size = totalSize
|
||||
t.Completed = sizeCompleted
|
||||
t.Downloaded = sizeCompleted
|
||||
t.DownloadedSession = sizeCompleted
|
||||
t.Uploaded = sizeCompleted
|
||||
t.UploadedSession = sizeCompleted
|
||||
t.AmountLeft = totalSize - sizeCompleted
|
||||
t.Progress = progress
|
||||
t.Eta = eta
|
||||
t.Dlspeed = speed
|
||||
t.Upspeed = speed
|
||||
t.ContentPath = filepath.Join(t.SavePath, t.Name) + string(os.PathSeparator)
|
||||
return t
|
||||
}
|
||||
|
||||
func (s *Store) UpdateTorrent(t *Torrent, debridTorrent *types.Torrent) *Torrent {
|
||||
if debridTorrent == nil {
|
||||
return t
|
||||
}
|
||||
|
||||
if debridClient := s.debrid.GetClients()[debridTorrent.Debrid]; debridClient != nil {
|
||||
if debridTorrent.Status != "downloaded" {
|
||||
_ = debridClient.UpdateTorrent(debridTorrent)
|
||||
}
|
||||
}
|
||||
t = s.UpdateTorrentMin(t, debridTorrent)
|
||||
t.ContentPath = t.TorrentPath + string(os.PathSeparator)
|
||||
|
||||
if t.IsReady() {
|
||||
t.State = "pausedUP"
|
||||
s.torrents.Update(t)
|
||||
return t
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if t.IsReady() {
|
||||
t.State = "pausedUP"
|
||||
s.torrents.Update(t)
|
||||
return t
|
||||
}
|
||||
updatedT := s.UpdateTorrent(t, debridTorrent)
|
||||
t = updatedT
|
||||
|
||||
case <-time.After(10 * time.Minute): // Add a timeout
|
||||
return t
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/sirrobot01/decypharr/pkg/debrid/types"
|
||||
"os"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func keyPair(hash, category string) string {
|
||||
return fmt.Sprintf("%s|%s", hash, category)
|
||||
}
|
||||
|
||||
type Torrents = map[string]*Torrent
|
||||
|
||||
type TorrentStorage struct {
|
||||
torrents Torrents
|
||||
mu sync.RWMutex
|
||||
filename string // Added to store the filename for persistence
|
||||
}
|
||||
|
||||
func loadTorrentsFromJSON(filename string) (Torrents, error) {
|
||||
data, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
torrents := make(Torrents)
|
||||
if err := json.Unmarshal(data, &torrents); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return torrents, nil
|
||||
}
|
||||
|
||||
func newTorrentStorage(filename string) *TorrentStorage {
|
||||
// Open the JSON file and read the data
|
||||
torrents, err := loadTorrentsFromJSON(filename)
|
||||
if err != nil {
|
||||
torrents = make(Torrents)
|
||||
}
|
||||
// Create a new Storage
|
||||
return &TorrentStorage{
|
||||
torrents: torrents,
|
||||
filename: filename,
|
||||
}
|
||||
}
|
||||
|
||||
func (ts *TorrentStorage) Add(torrent *Torrent) {
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
ts.torrents[keyPair(torrent.Hash, torrent.Category)] = torrent
|
||||
go func() {
|
||||
err := ts.saveToFile()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (ts *TorrentStorage) AddOrUpdate(torrent *Torrent) {
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
ts.torrents[keyPair(torrent.Hash, torrent.Category)] = torrent
|
||||
go func() {
|
||||
err := ts.saveToFile()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (ts *TorrentStorage) Get(hash, category string) *Torrent {
|
||||
ts.mu.RLock()
|
||||
defer ts.mu.RUnlock()
|
||||
torrent, exists := ts.torrents[keyPair(hash, category)]
|
||||
if !exists && category == "" {
|
||||
// Try to find the torrent without knowing the category
|
||||
for _, t := range ts.torrents {
|
||||
if t.Hash == hash {
|
||||
return t
|
||||
}
|
||||
}
|
||||
}
|
||||
return torrent
|
||||
}
|
||||
|
||||
func (ts *TorrentStorage) GetAll(category string, filter string, hashes []string) []*Torrent {
|
||||
ts.mu.RLock()
|
||||
defer ts.mu.RUnlock()
|
||||
torrents := make([]*Torrent, 0)
|
||||
for _, torrent := range ts.torrents {
|
||||
if category != "" && torrent.Category != category {
|
||||
continue
|
||||
}
|
||||
if filter != "" && torrent.State != filter {
|
||||
continue
|
||||
}
|
||||
torrents = append(torrents, torrent)
|
||||
}
|
||||
|
||||
if len(hashes) > 0 {
|
||||
filtered := make([]*Torrent, 0)
|
||||
for _, hash := range hashes {
|
||||
for _, torrent := range torrents {
|
||||
if torrent.Hash == hash {
|
||||
filtered = append(filtered, torrent)
|
||||
}
|
||||
}
|
||||
}
|
||||
torrents = filtered
|
||||
}
|
||||
return torrents
|
||||
}
|
||||
|
||||
func (ts *TorrentStorage) GetAllSorted(category string, filter string, hashes []string, sortBy string, ascending bool) []*Torrent {
|
||||
torrents := ts.GetAll(category, filter, hashes)
|
||||
if sortBy != "" {
|
||||
sort.Slice(torrents, func(i, j int) bool {
|
||||
// If ascending is false, swap i and j to get descending order
|
||||
if !ascending {
|
||||
i, j = j, i
|
||||
}
|
||||
|
||||
switch sortBy {
|
||||
case "name":
|
||||
return torrents[i].Name < torrents[j].Name
|
||||
case "size":
|
||||
return torrents[i].Size < torrents[j].Size
|
||||
case "added_on":
|
||||
return torrents[i].AddedOn < torrents[j].AddedOn
|
||||
case "completed":
|
||||
return torrents[i].Completed < torrents[j].Completed
|
||||
case "progress":
|
||||
return torrents[i].Progress < torrents[j].Progress
|
||||
case "state":
|
||||
return torrents[i].State < torrents[j].State
|
||||
case "category":
|
||||
return torrents[i].Category < torrents[j].Category
|
||||
case "dlspeed":
|
||||
return torrents[i].Dlspeed < torrents[j].Dlspeed
|
||||
case "upspeed":
|
||||
return torrents[i].Upspeed < torrents[j].Upspeed
|
||||
case "ratio":
|
||||
return torrents[i].Ratio < torrents[j].Ratio
|
||||
default:
|
||||
// Default sort by added_on
|
||||
return torrents[i].AddedOn < torrents[j].AddedOn
|
||||
}
|
||||
})
|
||||
}
|
||||
return torrents
|
||||
}
|
||||
|
||||
func (ts *TorrentStorage) Update(torrent *Torrent) {
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
ts.torrents[keyPair(torrent.Hash, torrent.Category)] = torrent
|
||||
go func() {
|
||||
err := ts.saveToFile()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (ts *TorrentStorage) Delete(hash, category string, removeFromDebrid bool) {
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
key := keyPair(hash, category)
|
||||
torrent, exists := ts.torrents[key]
|
||||
if !exists && category == "" {
|
||||
// Remove the torrent without knowing the category
|
||||
for k, t := range ts.torrents {
|
||||
if t.Hash == hash {
|
||||
key = k
|
||||
torrent = t
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if torrent == nil {
|
||||
return
|
||||
}
|
||||
if removeFromDebrid && torrent.ID != "" && torrent.Debrid != "" {
|
||||
dbClient := GetStore().debrid.GetClient(torrent.Debrid)
|
||||
if dbClient != nil {
|
||||
_ = dbClient.DeleteTorrent(torrent.ID)
|
||||
}
|
||||
}
|
||||
|
||||
delete(ts.torrents, key)
|
||||
|
||||
// Delete the torrent folder
|
||||
if torrent.ContentPath != "" {
|
||||
err := os.RemoveAll(torrent.ContentPath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
err := ts.saveToFile()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (ts *TorrentStorage) DeleteMultiple(hashes []string, removeFromDebrid bool) {
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
toDelete := make(map[string]string)
|
||||
|
||||
for _, hash := range hashes {
|
||||
for key, torrent := range ts.torrents {
|
||||
if torrent == nil {
|
||||
continue
|
||||
}
|
||||
if torrent.Hash == hash {
|
||||
if removeFromDebrid && torrent.ID != "" && torrent.Debrid != "" {
|
||||
toDelete[torrent.ID] = torrent.Debrid
|
||||
}
|
||||
delete(ts.torrents, key)
|
||||
if torrent.ContentPath != "" {
|
||||
err := os.RemoveAll(torrent.ContentPath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
err := ts.saveToFile()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}()
|
||||
|
||||
clients := GetStore().debrid.GetClients()
|
||||
|
||||
go func() {
|
||||
for id, debrid := range toDelete {
|
||||
dbClient, ok := clients[debrid]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
err := dbClient.DeleteTorrent(id)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (ts *TorrentStorage) Save() error {
|
||||
return ts.saveToFile()
|
||||
}
|
||||
|
||||
// saveToFile is a helper function to write the current state to the JSON file
|
||||
func (ts *TorrentStorage) saveToFile() error {
|
||||
ts.mu.RLock()
|
||||
data, err := json.MarshalIndent(ts.torrents, "", " ")
|
||||
ts.mu.RUnlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(ts.filename, data, 0644)
|
||||
}
|
||||
|
||||
func (ts *TorrentStorage) Reset() {
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
ts.torrents = make(Torrents)
|
||||
}
|
||||
|
||||
type Torrent struct {
|
||||
ID string `json:"id"`
|
||||
Debrid string `json:"debrid"`
|
||||
TorrentPath string `json:"-"`
|
||||
DebridTorrent *types.Torrent `json:"-"`
|
||||
|
||||
AddedOn int64 `json:"added_on,omitempty"`
|
||||
AmountLeft int64 `json:"amount_left"`
|
||||
AutoTmm bool `json:"auto_tmm"`
|
||||
Availability float64 `json:"availability,omitempty"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Completed int64 `json:"completed"`
|
||||
CompletionOn int `json:"completion_on,omitempty"`
|
||||
ContentPath string `json:"content_path"`
|
||||
DlLimit int `json:"dl_limit"`
|
||||
Dlspeed int64 `json:"dlspeed"`
|
||||
Downloaded int64 `json:"downloaded"`
|
||||
DownloadedSession int64 `json:"downloaded_session"`
|
||||
Eta int `json:"eta"`
|
||||
FlPiecePrio bool `json:"f_l_piece_prio,omitempty"`
|
||||
ForceStart bool `json:"force_start,omitempty"`
|
||||
Hash string `json:"hash"`
|
||||
LastActivity int64 `json:"last_activity,omitempty"`
|
||||
MagnetUri string `json:"magnet_uri,omitempty"`
|
||||
MaxRatio int `json:"max_ratio,omitempty"`
|
||||
MaxSeedingTime int `json:"max_seeding_time,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
NumComplete int `json:"num_complete,omitempty"`
|
||||
NumIncomplete int `json:"num_incomplete,omitempty"`
|
||||
NumLeechs int `json:"num_leechs,omitempty"`
|
||||
NumSeeds int `json:"num_seeds,omitempty"`
|
||||
Priority int `json:"priority,omitempty"`
|
||||
Progress float64 `json:"progress"`
|
||||
Ratio int `json:"ratio,omitempty"`
|
||||
RatioLimit int `json:"ratio_limit,omitempty"`
|
||||
SavePath string `json:"save_path"`
|
||||
SeedingTimeLimit int `json:"seeding_time_limit,omitempty"`
|
||||
SeenComplete int64 `json:"seen_complete,omitempty"`
|
||||
SeqDl bool `json:"seq_dl"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
State string `json:"state,omitempty"`
|
||||
SuperSeeding bool `json:"super_seeding"`
|
||||
Tags string `json:"tags,omitempty"`
|
||||
TimeActive int `json:"time_active,omitempty"`
|
||||
TotalSize int64 `json:"total_size,omitempty"`
|
||||
Tracker string `json:"tracker,omitempty"`
|
||||
UpLimit int64 `json:"up_limit,omitempty"`
|
||||
Uploaded int64 `json:"uploaded,omitempty"`
|
||||
UploadedSession int64 `json:"uploaded_session,omitempty"`
|
||||
Upspeed int64 `json:"upspeed,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
func (t *Torrent) IsReady() bool {
|
||||
return (t.AmountLeft <= 0 || t.Progress == 1) && t.TorrentPath != ""
|
||||
}
|
||||
|
||||
func (t *Torrent) discordContext() string {
|
||||
format := `
|
||||
**Name:** %s
|
||||
**Arr:** %s
|
||||
**Hash:** %s
|
||||
**MagnetURI:** %s
|
||||
**Debrid:** %s
|
||||
`
|
||||
return fmt.Sprintf(format, t.Name, t.Category, t.Hash, t.MagnetUri, t.Debrid)
|
||||
}
|
||||
Reference in New Issue
Block a user