- Add more rclone supports

- Add rclone log viewer
- Add more stats to Stats page
- Fix some minor bugs
This commit is contained in:
Mukhtar Akere
2025-08-18 01:57:02 +01:00
parent 742d8fb088
commit 8696db42d2
40 changed files with 787 additions and 253 deletions

View File

@@ -9,13 +9,14 @@ import (
"github.com/sirrobot01/decypharr/internal/utils"
"github.com/sirrobot01/decypharr/pkg/arr"
"github.com/sirrobot01/decypharr/pkg/debrid/providers/alldebrid"
"github.com/sirrobot01/decypharr/pkg/debrid/providers/debrid_link"
"github.com/sirrobot01/decypharr/pkg/debrid/providers/debridlink"
"github.com/sirrobot01/decypharr/pkg/debrid/providers/realdebrid"
"github.com/sirrobot01/decypharr/pkg/debrid/providers/torbox"
debridStore "github.com/sirrobot01/decypharr/pkg/debrid/store"
"github.com/sirrobot01/decypharr/pkg/debrid/types"
"github.com/sirrobot01/decypharr/pkg/rclone"
"sync"
"time"
)
type Debrid struct {
@@ -69,7 +70,7 @@ func NewStorage(rcManager *rclone.Manager) *Storage {
_log := client.Logger()
if dc.UseWebDav {
if cfg.Rclone.Enabled && rcManager != nil {
mounter = rclone.NewMount(dc.Name, webdavUrl, rcManager)
mounter = rclone.NewMount(dc.Name, dc.RcloneMountPath, webdavUrl, rcManager)
}
cache = debridStore.NewDebridCache(dc, client, mounter)
_log.Info().Msg("Debrid Service started with WebDAV")
@@ -98,6 +99,47 @@ func (d *Storage) Debrid(name string) *Debrid {
return nil
}
func (d *Storage) StartWorker(ctx context.Context) error {
if ctx == nil {
ctx = context.Background()
}
// Start all debrid syncAccounts
// Runs every 1m
if err := d.syncAccounts(); err != nil {
return err
}
ticker := time.NewTicker(5 * time.Minute)
go func() {
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
_ = d.syncAccounts()
}
}
}()
return nil
}
func (d *Storage) syncAccounts() error {
d.mu.Lock()
defer d.mu.Unlock()
for name, debrid := range d.debrids {
if debrid == nil || debrid.client == nil {
continue
}
_log := debrid.client.Logger()
if err := debrid.client.SyncAccounts(); err != nil {
_log.Error().Err(err).Msgf("Failed to sync account for %s", name)
continue
}
}
return nil
}
func (d *Storage) Debrids() map[string]*Debrid {
d.mu.RLock()
defer d.mu.RUnlock()
@@ -178,7 +220,7 @@ func createDebridClient(dc config.Debrid) (types.Client, error) {
case "torbox":
return torbox.New(dc)
case "debridlink":
return debrid_link.New(dc)
return debridlink.New(dc)
case "alldebrid":
return alldebrid.New(dc)
default:

View File

@@ -25,6 +25,7 @@ type AllDebrid struct {
autoExpiresLinksAfter time.Duration
DownloadUncached bool
client *request.Client
Profile *types.Profile `json:"profile"`
MountPath string
logger zerolog.Logger
@@ -33,10 +34,6 @@ type AllDebrid struct {
minimumFreeSlot int
}
func (ad *AllDebrid) GetProfile() (*types.Profile, error) {
return nil, nil
}
func New(dc config.Debrid) (*AllDebrid, error) {
rl := request.ParseRateLimit(dc.RateLimit)
@@ -449,6 +446,58 @@ func (ad *AllDebrid) GetAvailableSlots() (int, error) {
return 0, fmt.Errorf("GetAvailableSlots not implemented for AllDebrid")
}
func (ad *AllDebrid) GetProfile() (*types.Profile, error) {
if ad.Profile != nil {
return ad.Profile, nil
}
url := fmt.Sprintf("%s/user", ad.Host)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := ad.client.MakeRequest(req)
if err != nil {
return nil, err
}
var res UserProfileResponse
err = json.Unmarshal(resp, &res)
if err != nil {
ad.logger.Error().Err(err).Msgf("Error unmarshalling user profile")
return nil, err
}
if res.Status != "success" {
message := "unknown error"
if res.Error != nil {
message = res.Error.Message
}
return nil, fmt.Errorf("error getting user profile: %s", message)
}
userData := res.Data.User
expiration := time.Unix(userData.PremiumUntil, 0)
profile := &types.Profile{
Id: 1,
Name: ad.name,
Username: userData.Username,
Email: userData.Email,
Points: userData.FidelityPoints,
Premium: userData.PremiumUntil,
Expiration: expiration,
}
if userData.IsPremium {
profile.Type = "premium"
} else if userData.IsTrial {
profile.Type = "trial"
} else {
profile.Type = "free"
}
ad.Profile = profile
return profile, nil
}
func (ad *AllDebrid) Accounts() *types.Accounts {
return ad.accounts
}
func (ad *AllDebrid) SyncAccounts() error {
return nil
}

View File

@@ -112,3 +112,22 @@ func (m *Magnets) UnmarshalJSON(data []byte) error {
}
return fmt.Errorf("magnets: unsupported JSON format")
}
type UserProfileResponse struct {
Status string `json:"status"`
Error *errorResponse `json:"error"`
Data struct {
User struct {
Username string `json:"username"`
Email string `json:"email"`
IsPremium bool `json:"isPremium"`
IsSubscribed bool `json:"isSubscribed"`
IsTrial bool `json:"isTrial"`
PremiumUntil int64 `json:"premiumUntil"`
Lang string `json:"lang"`
FidelityPoints int `json:"fidelityPoints"`
LimitedHostersQuotas map[string]int `json:"limitedHostersQuotas"`
Notifications []string `json:"notifications"`
} `json:"user"`
} `json:"data"`
}

View File

@@ -1,4 +1,4 @@
package debrid_link
package debridlink
import (
"bytes"
@@ -30,6 +30,8 @@ type DebridLink struct {
logger zerolog.Logger
checkCached bool
addSamples bool
Profile *types.Profile `json:"profile,omitempty"`
}
func New(dc config.Debrid) (*DebridLink, error) {
@@ -66,10 +68,6 @@ func New(dc config.Debrid) (*DebridLink, error) {
}, nil
}
func (dl *DebridLink) GetProfile() (*types.Profile, error) {
return nil, nil
}
func (dl *DebridLink) Name() string {
return dl.name
}
@@ -476,6 +474,55 @@ func (dl *DebridLink) GetAvailableSlots() (int, error) {
return 0, fmt.Errorf("GetAvailableSlots not implemented for DebridLink")
}
func (dl *DebridLink) GetProfile() (*types.Profile, error) {
if dl.Profile != nil {
return dl.Profile, nil
}
url := fmt.Sprintf("%s/account/infos", dl.Host)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := dl.client.MakeRequest(req)
if err != nil {
return nil, err
}
var res UserInfo
err = json.Unmarshal(resp, &res)
if err != nil {
dl.logger.Error().Err(err).Msgf("Error unmarshalling user info")
return nil, err
}
if !res.Success || res.Value == nil {
return nil, fmt.Errorf("error getting user info")
}
data := *res.Value
expiration := time.Unix(data.PremiumLeft, 0)
profile := &types.Profile{
Id: 1,
Username: data.Username,
Name: dl.name,
Email: data.Email,
Points: data.Points,
Premium: data.PremiumLeft,
Expiration: expiration,
}
if expiration.IsZero() {
profile.Expiration = time.Now().AddDate(1, 0, 0) // Default to 1 year if no expiration
}
if data.PremiumLeft > 0 {
profile.Type = "premium"
} else {
profile.Type = "free"
}
dl.Profile = profile
return profile, nil
}
func (dl *DebridLink) Accounts() *types.Accounts {
return dl.accounts
}
func (dl *DebridLink) SyncAccounts() error {
return nil
}

View File

@@ -1,4 +1,4 @@
package debrid_link
package debridlink
type APIResponse[T any] struct {
Success bool `json:"success"`
@@ -43,3 +43,12 @@ type _torrentInfo struct {
type torrentInfo APIResponse[[]_torrentInfo]
type SubmitTorrentInfo APIResponse[_torrentInfo]
type UserInfo APIResponse[struct {
Username string `json:"username"`
Email string `json:"email"`
AccountType int `json:"accountType"`
PremiumLeft int64 `json:"premiumLeft"`
Points int `json:"pts"`
Trafficshare int `json:"trafficshare"`
}]

View File

@@ -161,6 +161,23 @@ func (r *RealDebrid) getSelectedFiles(t *types.Torrent, data torrentInfo) (map[s
return files, nil
}
func (r *RealDebrid) handleRarFallback(t *types.Torrent, data torrentInfo) (map[string]types.File, error) {
files := make(map[string]types.File)
file := types.File{
TorrentId: t.Id,
Id: "0",
Name: t.Name + ".rar",
Size: data.Bytes,
IsRar: true,
ByteRange: nil,
Path: t.Name + ".rar",
Link: data.Links[0],
Generated: time.Now(),
}
files[file.Name] = file
return files, nil
}
// handleRarArchive processes RAR archives with multiple files
func (r *RealDebrid) handleRarArchive(t *types.Torrent, data torrentInfo, selectedFiles []types.File) (map[string]types.File, error) {
// This will block if 2 RAR operations are already in progress
@@ -172,21 +189,8 @@ func (r *RealDebrid) handleRarArchive(t *types.Torrent, data torrentInfo, select
files := make(map[string]types.File)
if !r.UnpackRar {
r.logger.Debug().Msgf("RAR file detected, but unpacking is disabled: %s", t.Name)
// Create a single file representing the RAR archive
file := types.File{
TorrentId: t.Id,
Id: "0",
Name: t.Name + ".rar",
Size: 0,
IsRar: true,
ByteRange: nil,
Path: t.Name + ".rar",
Link: data.Links[0],
Generated: time.Now(),
}
files[file.Name] = file
return files, nil
r.logger.Debug().Msgf("RAR file detected, but unpacking is disabled: %s. Falling back to single file representation.", t.Name)
return r.handleRarFallback(t, data)
}
r.logger.Info().Msgf("RAR file detected, unpacking: %s", t.Name)
@@ -194,20 +198,23 @@ func (r *RealDebrid) handleRarArchive(t *types.Torrent, data torrentInfo, select
downloadLinkObj, err := r.GetDownloadLink(t, linkFile)
if err != nil {
return nil, fmt.Errorf("failed to get download link for RAR file: %w", err)
r.logger.Debug().Err(err).Msgf("Error getting download link for RAR file: %s. Falling back to single file representation.", t.Name)
return r.handleRarFallback(t, data)
}
dlLink := downloadLinkObj.DownloadLink
reader, err := rar.NewReader(dlLink)
if err != nil {
return nil, fmt.Errorf("failed to create RAR reader: %w", err)
r.logger.Debug().Err(err).Msgf("Error creating RAR reader for %s. Falling back to single file representation.", t.Name)
return r.handleRarFallback(t, data)
}
rarFiles, err := reader.GetFiles()
if err != nil {
return nil, fmt.Errorf("failed to read RAR files: %w", err)
r.logger.Debug().Err(err).Msgf("Error reading RAR files for %s. Falling back to single file representation.", t.Name)
return r.handleRarFallback(t, data)
}
// Create lookup map for faster matching
@@ -232,7 +239,11 @@ func (r *RealDebrid) handleRarArchive(t *types.Torrent, data torrentInfo, select
r.logger.Warn().Msgf("RAR file %s not found in torrent files", rarFile.Name())
}
}
if len(files) == 0 {
r.logger.Warn().Msgf("No valid files found in RAR archive for torrent: %s", t.Name)
return r.handleRarFallback(t, data)
}
r.logger.Info().Msgf("Unpacked RAR archive for torrent: %s with %d files", t.Name, len(files))
return files, nil
}
@@ -700,7 +711,7 @@ func (r *RealDebrid) _getDownloadLink(file *types.File) (*types.DownloadLink, er
func (r *RealDebrid) GetDownloadLink(t *types.Torrent, file *types.File) (*types.DownloadLink, error) {
accounts := r.accounts.All()
accounts := r.accounts.Active()
for _, account := range accounts {
r.downloadClient.SetHeader("Authorization", fmt.Sprintf("Bearer %s", account.Token))
@@ -835,7 +846,7 @@ func (r *RealDebrid) GetDownloadLinks() (map[string]*types.DownloadLink, error)
offset := 0
limit := 1000
accounts := r.accounts.All()
accounts := r.accounts.Active()
if len(accounts) < 1 {
// No active download keys. It's likely that the key has reached bandwidth limit
@@ -933,6 +944,7 @@ func (r *RealDebrid) GetProfile() (*types.Profile, error) {
return nil, err
}
profile := &types.Profile{
Name: r.name,
Id: data.Id,
Username: data.Username,
Email: data.Email,
@@ -962,3 +974,71 @@ func (r *RealDebrid) GetAvailableSlots() (int, error) {
func (r *RealDebrid) Accounts() *types.Accounts {
return r.accounts
}
func (r *RealDebrid) SyncAccounts() error {
// Sync accounts with the current configuration
if len(r.accounts.Active()) == 0 {
return nil
}
for idx, account := range r.accounts.Active() {
if err := r.syncAccount(idx, account); err != nil {
r.logger.Error().Err(err).Msgf("Error syncing account %s", account.Username)
continue // Skip this account and continue with the next
}
}
return nil
}
func (r *RealDebrid) syncAccount(index int, account *types.Account) error {
if account.Token == "" {
return fmt.Errorf("account %s has no token", account.Username)
}
client := http.DefaultClient
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/user", r.Host), nil)
if err != nil {
return fmt.Errorf("error creating request for account %s: %w", account.Username, err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", account.Token))
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("error checking account %s: %w", account.Username, err)
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return fmt.Errorf("account %s is not valid, status code: %d", account.Username, resp.StatusCode)
}
defer resp.Body.Close()
var profile profileResponse
if err := json.NewDecoder(resp.Body).Decode(&profile); err != nil {
return fmt.Errorf("error decoding profile for account %s: %w", account.Username, err)
}
account.Username = profile.Username
// Get traffic usage
trafficReq, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/traffic/details", r.Host), nil)
if err != nil {
return fmt.Errorf("error creating request for traffic details for account %s: %w", account.Username, err)
}
trafficReq.Header.Set("Authorization", fmt.Sprintf("Bearer %s", account.Token))
trafficResp, err := client.Do(trafficReq)
if err != nil {
return fmt.Errorf("error checking traffic for account %s: %w", account.Username, err)
}
if trafficResp.StatusCode != http.StatusOK {
trafficResp.Body.Close()
return fmt.Errorf("error checking traffic for account %s, status code: %d", account.Username, trafficResp.StatusCode)
}
defer trafficResp.Body.Close()
var trafficData TrafficResponse
if err := json.NewDecoder(trafficResp.Body).Decode(&trafficData); err != nil {
return fmt.Errorf("error decoding traffic details for account %s: %w", account.Username, err)
}
today := time.Now().Format(time.DateOnly)
if todayData, exists := trafficData[today]; exists {
account.TrafficUsed = todayData.Bytes
}
r.accounts.Update(index, account)
return nil
}

View File

@@ -144,11 +144,11 @@ type profileResponse struct {
Id int64 `json:"id"`
Username string `json:"username"`
Email string `json:"email"`
Points int64 `json:"points"`
Points int `json:"points"`
Locale string `json:"locale"`
Avatar string `json:"avatar"`
Type string `json:"type"`
Premium int `json:"premium"`
Premium int64 `json:"premium"`
Expiration time.Time `json:"expiration"`
}
@@ -156,3 +156,10 @@ type AvailableSlotsResponse struct {
ActiveSlots int `json:"nb"`
TotalSlots int `json:"limit"`
}
type hostData struct {
Host map[string]int64 `json:"host"`
Bytes int64 `json:"bytes"`
}
type TrafficResponse map[string]hostData

View File

@@ -40,10 +40,6 @@ type Torbox struct {
addSamples bool
}
func (tb *Torbox) GetProfile() (*types.Profile, error) {
return nil, nil
}
func New(dc config.Debrid) (*Torbox, error) {
rl := request.ParseRateLimit(dc.RateLimit)
@@ -632,6 +628,14 @@ func (tb *Torbox) GetAvailableSlots() (int, error) {
return 0, fmt.Errorf("not implemented")
}
func (tb *Torbox) GetProfile() (*types.Profile, error) {
return nil, nil
}
func (tb *Torbox) Accounts() *types.Accounts {
return tb.accounts
}
func (tb *Torbox) SyncAccounts() error {
return nil
}

View File

@@ -122,9 +122,13 @@ func NewDebridCache(dc config.Debrid, client types.Client, mounter *rclone.Mount
cetSc, err := gocron.NewScheduler(gocron.WithLocation(cet))
if err != nil {
// If we can't create a CET scheduler, fallback to local time
cetSc, _ = gocron.NewScheduler(gocron.WithLocation(time.Local))
cetSc, _ = gocron.NewScheduler(gocron.WithLocation(time.Local), gocron.WithGlobalJobOptions(
gocron.WithTags("decypharr-"+dc.Name)))
}
scheduler, err := gocron.NewScheduler(gocron.WithLocation(time.Local))
scheduler, err := gocron.NewScheduler(
gocron.WithLocation(time.Local),
gocron.WithGlobalJobOptions(
gocron.WithTags("decypharr-"+dc.Name)))
if err != nil {
// If we can't create a local scheduler, fallback to CET
scheduler = cetSc
@@ -254,11 +258,6 @@ func (c *Cache) Start(ctx context.Context) error {
// initial download links
go c.refreshDownloadLinks(ctx)
if err := c.StartSchedule(ctx); err != nil {
c.logger.Error().Err(err).Msg("Failed to start cache worker")
}
c.repairChan = make(chan RepairRequest, 100) // Initialize the repair channel, max 100 requests buffered
go c.repairWorker(ctx)
@@ -708,7 +707,7 @@ func (c *Cache) ProcessTorrent(t *types.Torrent) error {
Str("torrent_id", t.Id).
Str("torrent_name", t.Name).
Int("total_files", len(t.Files)).
Msg("Torrent still not complete after refresh")
Msg("Torrent still not complete after refresh, marking as bad")
} else {
addedOn, err := time.Parse(time.RFC3339, t.Added)

View File

@@ -6,11 +6,11 @@ import (
"github.com/sirrobot01/decypharr/internal/utils"
)
func (c *Cache) StartSchedule(ctx context.Context) error {
func (c *Cache) StartWorker(ctx context.Context) error {
// For now, we just want to refresh the listing and download links
// Stop any existing jobs before starting new ones
c.scheduler.RemoveByTags("decypharr")
c.scheduler.RemoveByTags("decypharr-%s", c.GetConfig().Name)
// Schedule download link refresh job
if jd, err := utils.ConvertToJobDef(c.downloadLinksRefreshInterval); err != nil {
@@ -19,7 +19,7 @@ func (c *Cache) StartSchedule(ctx context.Context) error {
// Schedule the job
if _, err := c.scheduler.NewJob(jd, gocron.NewTask(func() {
c.refreshDownloadLinks(ctx)
}), gocron.WithContext(ctx), gocron.WithTags("decypharr")); err != nil {
}), gocron.WithContext(ctx)); err != nil {
c.logger.Error().Err(err).Msg("Failed to create download link refresh job")
} else {
c.logger.Debug().Msgf("Download link refresh job scheduled for every %s", c.downloadLinksRefreshInterval)
@@ -33,7 +33,7 @@ func (c *Cache) StartSchedule(ctx context.Context) error {
// Schedule the job
if _, err := c.scheduler.NewJob(jd, gocron.NewTask(func() {
c.refreshTorrents(ctx)
}), gocron.WithContext(ctx), gocron.WithTags("decypharr")); err != nil {
}), gocron.WithContext(ctx)); err != nil {
c.logger.Error().Err(err).Msg("Failed to create torrent refresh job")
} else {
c.logger.Debug().Msgf("Torrent refresh job scheduled for every %s", c.torrentRefreshInterval)
@@ -49,7 +49,7 @@ func (c *Cache) StartSchedule(ctx context.Context) error {
// Schedule the job
if _, err := c.cetScheduler.NewJob(jd, gocron.NewTask(func() {
c.resetInvalidLinks(ctx)
}), gocron.WithContext(ctx), gocron.WithTags("decypharr")); err != nil {
}), gocron.WithContext(ctx)); err != nil {
c.logger.Error().Err(err).Msg("Failed to create link reset job")
} else {
c.logger.Debug().Msgf("Link reset job scheduled for every midnight, CET")

View File

@@ -33,15 +33,17 @@ func NewAccounts(debridConf config.Debrid) *Accounts {
}
type Account struct {
Debrid string // e.g., "realdebrid", "torbox", etc.
Order int
Disabled bool
Token string
links map[string]*DownloadLink
mu sync.RWMutex
Debrid string // e.g., "realdebrid", "torbox", etc.
Order int
Disabled bool
Token string `json:"token"`
links map[string]*DownloadLink
mu sync.RWMutex
TrafficUsed int64 `json:"traffic_used"` // Traffic used in bytes
Username string `json:"username"` // Username for the account
}
func (a *Accounts) All() []*Account {
func (a *Accounts) Active() []*Account {
a.mu.RLock()
defer a.mu.RUnlock()
activeAccounts := make([]*Account, 0)
@@ -53,6 +55,12 @@ func (a *Accounts) All() []*Account {
return activeAccounts
}
func (a *Accounts) All() []*Account {
a.mu.RLock()
defer a.mu.RUnlock()
return a.accounts
}
func (a *Accounts) Current() *Account {
a.mu.RLock()
if a.current != nil {
@@ -177,6 +185,23 @@ func (a *Accounts) SetDownloadLinks(links map[string]*DownloadLink) {
a.Current().setLinks(links)
}
func (a *Accounts) Update(index int, account *Account) {
a.mu.Lock()
defer a.mu.Unlock()
if index < 0 || index >= len(a.accounts) {
return // Index out of bounds
}
// Update the account at the specified index
a.accounts[index] = account
// If the updated account is the current one, update the current reference
if a.current == nil || a.current.Order == index {
a.current = account
}
}
func newAccount(debridName, token string, index int) *Account {
return &Account{
Debrid: debridName,
@@ -213,7 +238,6 @@ func (a *Account) LinksCount() int {
defer a.mu.RUnlock()
return len(a.links)
}
func (a *Account) disable() {
a.Disabled = true
}

View File

@@ -25,4 +25,5 @@ type Client interface {
DeleteDownloadLink(linkId string) error
GetProfile() (*Profile, error)
GetAvailableSlots() (int, error)
SyncAccounts() error // Updates each accounts details(like traffic, username, etc.)
}

View File

@@ -114,19 +114,27 @@ type IngestData struct {
Size int64 `json:"size"`
}
type LibraryStats struct {
Total int `json:"total"`
Bad int `json:"bad"`
ActiveLinks int `json:"active_links"`
}
type Stats struct {
Profile *Profile `json:"profile"`
Library LibraryStats `json:"library"`
Accounts []map[string]any `json:"accounts"`
}
type Profile struct {
Name string `json:"name"`
Id int64 `json:"id"`
Username string `json:"username"`
Email string `json:"email"`
Points int64 `json:"points"`
Points int `json:"points"`
Type string `json:"type"`
Premium int `json:"premium"`
Premium int64 `json:"premium"`
Expiration time.Time `json:"expiration"`
LibrarySize int `json:"library_size"`
BadTorrents int `json:"bad_torrents"`
ActiveLinks int `json:"active_links"`
}
type DownloadLink struct {