diff --git a/internal/config/config.go b/internal/config/config.go index 1a0103c..d80093e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -27,6 +27,7 @@ type Debrid struct { CheckCached bool `json:"check_cached,omitempty"` RateLimit string `json:"rate_limit,omitempty"` // 200/minute or 10/second Proxy string `json:"proxy,omitempty"` + UnpackRar bool `json:"unpack_rar,omitempty"` AddSamples bool `json:"add_samples,omitempty"` UseWebDav bool `json:"use_webdav,omitempty"` diff --git a/pkg/debrid/debrid/download_link.go b/pkg/debrid/debrid/download_link.go index b7df841..9a6b152 100644 --- a/pkg/debrid/debrid/download_link.go +++ b/pkg/debrid/debrid/download_link.go @@ -3,10 +3,12 @@ package debrid import ( "errors" "fmt" - "github.com/sirrobot01/decypharr/internal/request" - "github.com/sirrobot01/decypharr/pkg/debrid/types" + "sync" "time" + + "github.com/sirrobot01/decypharr/internal/request" + "github.com/sirrobot01/decypharr/pkg/debrid/types" ) type linkCache struct { @@ -234,3 +236,12 @@ func (c *Cache) IsDownloadLinkInvalid(downloadLink string) bool { } return false } + +func (c *Cache) GetDownloadByteRange(torrentName, filename string) (*[2]int64, error) { + ct := c.GetTorrentByName(torrentName) + if ct == nil { + return nil, fmt.Errorf("torrent not found") + } + file := ct.Files[filename] + return file.ByteRange, nil +} diff --git a/pkg/debrid/realdebrid/realdebrid.go b/pkg/debrid/realdebrid/realdebrid.go index c38d1e8..da1825e 100644 --- a/pkg/debrid/realdebrid/realdebrid.go +++ b/pkg/debrid/realdebrid/realdebrid.go @@ -5,12 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/rs/zerolog" - "github.com/sirrobot01/decypharr/internal/config" - "github.com/sirrobot01/decypharr/internal/logger" - "github.com/sirrobot01/decypharr/internal/request" - "github.com/sirrobot01/decypharr/internal/utils" - "github.com/sirrobot01/decypharr/pkg/debrid/types" "io" "net/http" gourl "net/url" @@ -20,6 +14,15 @@ import ( "strings" "sync" "time" + + "github.com/rs/zerolog" + "github.com/sirrobot01/decypharr/internal/config" + "github.com/sirrobot01/decypharr/internal/logger" + "github.com/sirrobot01/decypharr/internal/request" + "github.com/sirrobot01/decypharr/internal/utils" + "github.com/sirrobot01/decypharr/pkg/debrid/types" + + "github.com/sirrobot01/decypharr/pkg/rar" ) type RealDebrid struct { @@ -35,10 +38,13 @@ type RealDebrid struct { client *request.Client downloadClient *request.Client - MountPath string - logger zerolog.Logger - checkCached bool - addSamples bool + MountPath string + logger zerolog.Logger + UnpackRar bool + + rarSemaphore chan struct{} + checkCached bool + addSamples bool } func New(dc config.Debrid) *RealDebrid { @@ -70,6 +76,7 @@ func New(dc config.Debrid) *RealDebrid { APIKey: dc.APIKey, accounts: accounts, DownloadUncached: dc.DownloadUncached, + UnpackRar: dc.UnpackRar, client: request.New( request.WithHeaders(headers), request.WithRateLimiter(rl), @@ -88,6 +95,7 @@ func New(dc config.Debrid) *RealDebrid { currentDownloadKey: currentDownloadKey, MountPath: dc.Folder, logger: logger.New(dc.Name), + rarSemaphore: make(chan struct{}, 2), checkCached: dc.CheckCached, addSamples: dc.AddSamples, } @@ -101,30 +109,127 @@ func (r *RealDebrid) GetLogger() zerolog.Logger { return r.logger } -func getSelectedFiles(t *types.Torrent, data torrentInfo) map[string]types.File { +func (r *RealDebrid) getSelectedFiles(t *types.Torrent, data torrentInfo) (map[string]types.File, error) { + files := make(map[string]types.File) selectedFiles := make([]types.File, 0) + for _, f := range data.Files { if f.Selected == 1 { - name := filepath.Base(f.Path) - file := types.File{ + selectedFiles = append(selectedFiles, types.File{ TorrentId: t.Id, - Name: name, - Path: name, + Name: filepath.Base(f.Path), + Path: filepath.Base(f.Path), Size: f.Bytes, Id: strconv.Itoa(f.ID), - } - selectedFiles = append(selectedFiles, file) + }) } } + + if len(selectedFiles) == 0 { + return files, nil + } + + // Handle RARed torrents (single link, multiple files) + if len(data.Links) == 1 && len(selectedFiles) > 1 { + return r.handleRarArchive(t, data, selectedFiles) + } + + // Standard case - map files to links + if len(selectedFiles) > len(data.Links) { + r.logger.Warn().Msgf("More files than links available: %d files, %d links for %s", len(selectedFiles), len(data.Links), t.Name) + } + + for i, f := range selectedFiles { + if i < len(data.Links) { + f.Link = data.Links[i] + files[f.Name] = f + } else { + r.logger.Warn().Str("file", f.Name).Msg("No link available for file") + } + } + + return files, nil +} + +// handleRarArchive processes RAR archives with multiple files +func (r *RealDebrid) handleRarArchive(t *types.Torrent, data torrentInfo, selectedFiles []types.File) (map[string]types.File, error) { + // This will block if 2 RAR operations are already in progress + r.rarSemaphore <- struct{}{} + defer func() { + <-r.rarSemaphore + }() + files := make(map[string]types.File) - for index, f := range selectedFiles { - if index >= len(data.Links) { - break + + if !r.UnpackRar { + r.logger.Debug().Msgf("RAR file detected, but unpacking is disabled: %s", t.Name) + // Create a single file representing the RAR archive + file := types.File{ + TorrentId: t.Id, + Id: "0", + Name: t.Name + ".rar", + Size: 0, + IsRar: true, + ByteRange: nil, + Path: t.Name + ".rar", + Link: data.Links[0], + AccountId: selectedFiles[0].AccountId, + Generated: time.Now(), } - f.Link = data.Links[index] - files[f.Name] = f + files[file.Name] = file + return files, nil } - return files + + r.logger.Info().Msgf("RAR file detected, unpacking: %s", t.Name) + linkFile := &types.File{TorrentId: t.Id, Link: data.Links[0]} + downloadLinkObj, err := r.GetDownloadLink(t, linkFile) + + if err != nil { + return nil, fmt.Errorf("failed to get download link for RAR file: %w", err) + } + + dlLink := downloadLinkObj.DownloadLink + reader, err := rar.NewReader(dlLink) + + if err != nil { + return nil, fmt.Errorf("failed to create RAR reader: %w", err) + } + + rarFiles, err := reader.GetFiles() + + if err != nil { + return nil, fmt.Errorf("failed to read RAR files: %w", err) + } + + // Create lookup map for faster matching + fileMap := make(map[string]*types.File) + for i := range selectedFiles { + // RD converts special chars to '_' for RAR file paths + // TOOD: there might be more special chars to replace + safeName := strings.NewReplacer("|", "_", "\"", "_", "\\", "_", "?", "_", "*", "_", ":", "_", "<", "_", ">", "_").Replace(selectedFiles[i].Name) + fileMap[safeName] = &selectedFiles[i] + } + + for _, rarFile := range rarFiles { + if file, exists := fileMap[rarFile.Name()]; exists { + file.IsRar = true + file.ByteRange = rarFile.ByteRange() + file.Link = data.Links[0] + file.DownloadLink = &types.DownloadLink{ + Link: data.Links[0], + DownloadLink: dlLink, + Filename: file.Name, + Size: file.Size, + Generated: time.Now(), + } + + files[file.Name] = *file + } else if !rarFile.IsDirectory { + r.logger.Warn().Msgf("RAR file %s not found in torrent files", rarFile.Name()) + } + } + + return files, nil } // getTorrentFiles returns a list of torrent files from the torrent info @@ -338,7 +443,8 @@ func (r *RealDebrid) UpdateTorrent(t *types.Torrent) error { t.MountPath = r.MountPath t.Debrid = r.Name t.Added = data.Added - t.Files = getSelectedFiles(t, data) // Get selected files + t.Files, _ = r.getSelectedFiles(t, data) // Get selected files + return nil } @@ -390,7 +496,11 @@ func (r *RealDebrid) CheckStatus(t *types.Torrent, isSymlink bool) (*types.Torre return t, fmt.Errorf("realdebrid API error: Status: %d", res.StatusCode) } } else if status == "downloaded" { - t.Files = getSelectedFiles(t, data) // Get selected files + t.Files, err = r.getSelectedFiles(t, data) // Get selected files + if err != nil { + return t, err + } + r.logger.Info().Msgf("Torrent: %s downloaded to RD", t.Name) if !isSymlink { err = r.GenerateDownloadLinks(t) diff --git a/pkg/debrid/types/torrent.go b/pkg/debrid/types/torrent.go index b072efe..ac69ac4 100644 --- a/pkg/debrid/types/torrent.go +++ b/pkg/debrid/types/torrent.go @@ -2,13 +2,14 @@ package types import ( "fmt" - "github.com/sirrobot01/decypharr/internal/logger" - "github.com/sirrobot01/decypharr/internal/utils" - "github.com/sirrobot01/decypharr/pkg/arr" "os" "path/filepath" "sync" "time" + + "github.com/sirrobot01/decypharr/internal/logger" + "github.com/sirrobot01/decypharr/internal/utils" + "github.com/sirrobot01/decypharr/pkg/arr" ) type Torrent struct { @@ -99,6 +100,8 @@ type File struct { Id string `json:"id"` Name string `json:"name"` Size int64 `json:"size"` + IsRar bool `json:"is_rar"` + ByteRange *[2]int64 `json:"byte_range,omitempty"` Path string `json:"path"` Link string `json:"link"` DownloadLink *DownloadLink `json:"-"` diff --git a/pkg/qbit/downloader.go b/pkg/qbit/downloader.go index 3881f7b..fef91bd 100644 --- a/pkg/qbit/downloader.go +++ b/pkg/qbit/downloader.go @@ -2,22 +2,30 @@ package qbit import ( "fmt" - "github.com/cavaliergopher/grab/v3" - "github.com/sirrobot01/decypharr/internal/utils" - debridTypes "github.com/sirrobot01/decypharr/pkg/debrid/types" "io" "net/http" "os" "path/filepath" "sync" "time" + + "github.com/cavaliergopher/grab/v3" + "github.com/sirrobot01/decypharr/internal/utils" + debrid "github.com/sirrobot01/decypharr/pkg/debrid/types" ) -func Download(client *grab.Client, url, filename string, progressCallback func(int64, int64)) error { +func Download(client *grab.Client, url, filename string, byterange *[2]int64, progressCallback func(int64, int64)) error { req, err := grab.NewRequest(filename, url) if err != nil { return err } + + // Set byte range if specified + if byterange != nil { + byterangeStr := fmt.Sprintf("%d-%d", byterange[0], byterange[1]) + req.HTTPRequest.Header.Set("Range", "bytes="+byterangeStr) + } + resp := client.Do(req) t := time.NewTicker(time.Second * 2) @@ -107,7 +115,7 @@ func (q *QBit) downloadFiles(torrent *Torrent, parent string) { } wg.Add(1) q.downloadSemaphore <- struct{}{} - go func(file debridTypes.File) { + go func(file debrid.File) { defer wg.Done() defer func() { <-q.downloadSemaphore }() filename := file.Name @@ -116,6 +124,7 @@ func (q *QBit) downloadFiles(torrent *Torrent, parent string) { client, file.DownloadLink.DownloadLink, filepath.Join(parent, filename), + file.ByteRange, progressCallback, ) @@ -235,7 +244,7 @@ func (q *QBit) ProcessSymlink(torrent *Torrent) (string, error) { return torrentSymlinkPath, nil } -func (q *QBit) createSymlinksWebdav(debridTorrent *debridTypes.Torrent, rclonePath, torrentFolder string) (string, error) { +func (q *QBit) createSymlinksWebdav(debridTorrent *debrid.Torrent, rclonePath, torrentFolder string) (string, error) { files := debridTorrent.Files symlinkPath := filepath.Join(q.DownloadFolder, debridTorrent.Arr.Name, torrentFolder) // /mnt/symlinks/{category}/MyTVShow/ err := os.MkdirAll(symlinkPath, os.ModePerm) @@ -243,7 +252,7 @@ func (q *QBit) createSymlinksWebdav(debridTorrent *debridTypes.Torrent, rclonePa return "", fmt.Errorf("failed to create directory: %s: %v", symlinkPath, err) } - remainingFiles := make(map[string]debridTypes.File) + remainingFiles := make(map[string]debrid.File) for _, file := range files { remainingFiles[file.Name] = file } @@ -300,7 +309,7 @@ func (q *QBit) createSymlinksWebdav(debridTorrent *debridTypes.Torrent, rclonePa return symlinkPath, nil } -func (q *QBit) createSymlinks(debridTorrent *debridTypes.Torrent, rclonePath, torrentFolder string) (string, error) { +func (q *QBit) createSymlinks(debridTorrent *debrid.Torrent, rclonePath, torrentFolder string) (string, error) { files := debridTorrent.Files symlinkPath := filepath.Join(q.DownloadFolder, debridTorrent.Arr.Name, torrentFolder) // /mnt/symlinks/{category}/MyTVShow/ err := os.MkdirAll(symlinkPath, os.ModePerm) @@ -308,7 +317,7 @@ func (q *QBit) createSymlinks(debridTorrent *debridTypes.Torrent, rclonePath, to return "", fmt.Errorf("failed to create directory: %s: %v", symlinkPath, err) } - remainingFiles := make(map[string]debridTypes.File) + remainingFiles := make(map[string]debrid.File) for _, file := range files { remainingFiles[file.Path] = file } @@ -364,7 +373,7 @@ func (q *QBit) createSymlinks(debridTorrent *debridTypes.Torrent, rclonePath, to return symlinkPath, nil } -func (q *QBit) getTorrentPath(rclonePath string, debridTorrent *debridTypes.Torrent) (string, error) { +func (q *QBit) getTorrentPath(rclonePath string, debridTorrent *debrid.Torrent) (string, error) { for { torrentPath, err := debridTorrent.GetMountFolder(rclonePath) if err == nil { diff --git a/pkg/rar/rarar.go b/pkg/rar/rarar.go new file mode 100644 index 0000000..ae9f4b5 --- /dev/null +++ b/pkg/rar/rarar.go @@ -0,0 +1,701 @@ +// Source: https://github.com/eliasbenb/RARAR.py +// Note that this code only translates the original Python for RAR3 (not RAR5) support. + +package rar + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math/rand" + "net/http" + "strings" + "time" + "unicode/utf8" +) + +// Constants from the Python code +var ( + // Chunk sizes + DefaultChunkSize = 4096 + HttpChunkSize = 32768 + MaxSearchSize = 1 << 20 // 1MB + + // RAR marker and block types + Rar3Marker = []byte{0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00} + BlockFile = byte(0x74) + BlockHeader = byte(0x73) + BlockMarker = byte(0x72) + BlockEnd = byte(0x7B) + + // Header flags + FlagDirectory = 0xE0 + FlagHasHighSize = 0x100 + FlagHasUnicodeName = 0x200 + FlagHasData = 0x8000 +) + +// Compression methods +var CompressionMethods = map[byte]string{ + 0x30: "Store", + 0x31: "Fastest", + 0x32: "Fast", + 0x33: "Normal", + 0x34: "Good", + 0x35: "Best", +} + +// Error definitions +var ( + ErrMarkerNotFound = errors.New("RAR marker not found within search limit") + ErrInvalidFormat = errors.New("invalid RAR format") + ErrNetworkError = errors.New("network error") + ErrRangeRequestsNotSupported = errors.New("server does not support range requests") + ErrCompressionNotSupported = errors.New("compression method not supported") + ErrDirectoryExtractNotSupported = errors.New("directory extract not supported") +) + +// Name returns the base filename of the file +func (f *File) Name() string { + if i := strings.LastIndexAny(f.Path, "\\/"); i >= 0 { + return f.Path[i+1:] + } + return f.Path +} + +func (f *File) ByteRange() *[2]int64 { + return &[2]int64{f.DataOffset, f.DataOffset + f.CompressedSize - 1} +} + +func NewHttpFile(url string) (*HttpFile, error) { + client := &http.Client{} + file := &HttpFile{ + URL: url, + Position: 0, + Client: client, + MaxRetries: 3, + RetryDelay: time.Second, + } + + // Get file size + size, err := file.getFileSize() + if err != nil { + return nil, fmt.Errorf("failed to get file size: %w", err) + } + file.FileSize = size + + return file, nil +} + +func (f *HttpFile) doWithRetry(operation func() (interface{}, error)) (interface{}, error) { + var lastErr error + for attempt := 0; attempt <= f.MaxRetries; attempt++ { + if attempt > 0 { + // Jitter + exponential backoff delay + delay := f.RetryDelay * time.Duration(1< 0 { + remaining := f.FileSize - off + if remaining <= 0 { + return 0, io.EOF + } + if size > remaining { + size = remaining + p = p[:size] + } + } + + result, err := f.doWithRetry(func() (interface{}, error) { + // Create HTTP request with Range header + req, err := http.NewRequest("GET", f.URL, nil) + if err != nil { + return 0, fmt.Errorf("%w: %v", ErrNetworkError, err) + } + + end := off + size - 1 + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", off, end)) + + // Make the request + resp, err := f.Client.Do(req) + if err != nil { + return 0, fmt.Errorf("%w: %v", ErrNetworkError, err) + } + defer resp.Body.Close() + + // Handle response + switch resp.StatusCode { + case http.StatusPartialContent: + // Read the content + bytesRead, err := io.ReadFull(resp.Body, p) + return bytesRead, err + case http.StatusOK: + // Some servers return the full content instead of partial + fullData, err := io.ReadAll(resp.Body) + if err != nil { + return 0, fmt.Errorf("%w: %v", ErrNetworkError, err) + } + + if int64(len(fullData)) <= off { + return 0, io.EOF + } + + end = off + size + if int64(len(fullData)) < end { + end = int64(len(fullData)) + } + + copy(p, fullData[off:end]) + return int(end - off), nil + case http.StatusRequestedRangeNotSatisfiable: + // We're at EOF + return 0, io.EOF + default: + return 0, fmt.Errorf("%w: unexpected status code: %d", ErrNetworkError, resp.StatusCode) + } + }) + + if err != nil { + return 0, err + } + + return result.(int), nil +} + +// NewReader creates a new RAR3 reader +func NewReader(url string) (*Reader, error) { + file, err := NewHttpFile(url) + if err != nil { + return nil, err + } + + reader := &Reader{ + File: file, + ChunkSize: HttpChunkSize, + Files: make([]*File, 0), + } + + // Find RAR marker + marker, err := reader.findMarker() + if err != nil { + return nil, err + } + reader.Marker = marker + pos := reader.Marker + int64(len(Rar3Marker)) // Skip marker block + + headerData, err := reader.readBytes(pos, 7) + if err != nil { + return nil, err + } + + if len(headerData) < 7 { + return nil, ErrInvalidFormat + } + + headType := headerData[2] + headSize := int(binary.LittleEndian.Uint16(headerData[5:7])) + + if headType != BlockHeader { + return nil, ErrInvalidFormat + } + + // Store the position after the archive header + reader.HeaderEndPos = pos + int64(headSize) + + return reader, nil +} + +// readBytes reads a range of bytes from the file +func (r *Reader) readBytes(start int64, length int) ([]byte, error) { + if length <= 0 { + return []byte{}, nil + } + + data := make([]byte, length) + n, err := r.File.ReadAt(data, start) + if err != nil && err != io.EOF { + return nil, err + } + + if n < length { + // Partial read, return what we got + return data[:n], nil + } + + return data, nil +} + +// findMarker finds the RAR marker in the file +func (r *Reader) findMarker() (int64, error) { + // First try to find marker in the first chunk + firstChunkSize := 8192 // 8KB + chunk, err := r.readBytes(0, firstChunkSize) + if err != nil { + return 0, err + } + + markerPos := bytes.Index(chunk, Rar3Marker) + if markerPos != -1 { + return int64(markerPos), nil + } + + // If not found, continue searching + position := int64(firstChunkSize - len(Rar3Marker) + 1) + maxSearch := int64(MaxSearchSize) + + for position < maxSearch { + chunkSize := min(r.ChunkSize, int(maxSearch-position)) + chunk, err := r.readBytes(position, chunkSize) + if err != nil || len(chunk) == 0 { + break + } + + markerPos := bytes.Index(chunk, Rar3Marker) + if markerPos != -1 { + return position + int64(markerPos), nil + } + + // Move forward by chunk size minus the marker length + position += int64(max(1, len(chunk)-len(Rar3Marker)+1)) + } + + return 0, ErrMarkerNotFound +} + +// decodeUnicode decodes RAR3 Unicode encoding +func decodeUnicode(asciiStr string, unicodeData []byte) string { + if len(unicodeData) == 0 { + return asciiStr + } + + result := []rune{} + asciiPos := 0 + dataPos := 0 + highByte := byte(0) + + for dataPos < len(unicodeData) { + flags := unicodeData[dataPos] + dataPos++ + + // Determine the number of character positions this flag byte controls + var flagBits uint + var flagCount int + var bitCount int + + if flags&0x80 != 0 { + // Extended flag - controls up to 32 characters (16 bit pairs) + flagBits = uint(flags) + bitCount = 1 + for (flagBits&(0x80>>bitCount) != 0) && dataPos < len(unicodeData) { + flagBits = ((flagBits & ((0x80 >> bitCount) - 1)) << 8) | uint(unicodeData[dataPos]) + dataPos++ + bitCount++ + } + flagCount = bitCount * 4 + } else { + // Simple flag - controls 4 characters (4 bit pairs) + flagBits = uint(flags) + flagCount = 4 + } + + // Process each 2-bit flag + for i := 0; i < flagCount; i++ { + if asciiPos >= len(asciiStr) && dataPos >= len(unicodeData) { + break + } + + flagValue := (flagBits >> (i * 2)) & 0x03 + + switch flagValue { + case 0: + // Use ASCII character + if asciiPos < len(asciiStr) { + result = append(result, rune(asciiStr[asciiPos])) + asciiPos++ + } + case 1: + // Unicode character with high byte 0 + if dataPos < len(unicodeData) { + result = append(result, rune(unicodeData[dataPos])) + dataPos++ + } + case 2: + // Unicode character with current high byte + if dataPos < len(unicodeData) { + lowByte := uint(unicodeData[dataPos]) + dataPos++ + result = append(result, rune(lowByte|(uint(highByte)<<8))) + } + case 3: + // Set new high byte + if dataPos < len(unicodeData) { + highByte = unicodeData[dataPos] + dataPos++ + } + } + } + } + + // Append any remaining ASCII characters + for asciiPos < len(asciiStr) { + result = append(result, rune(asciiStr[asciiPos])) + asciiPos++ + } + + return string(result) +} + +// readFiles reads all file entries in the archive +func (r *Reader) readFiles() error { + pos := r.Marker + pos += int64(len(Rar3Marker)) // Skip marker block + + // Read archive header + headerData, err := r.readBytes(pos, 7) + if err != nil { + return err + } + + if len(headerData) < 7 { + return ErrInvalidFormat + } + + headType := headerData[2] + headSize := int(binary.LittleEndian.Uint16(headerData[5:7])) + + if headType != BlockHeader { + return ErrInvalidFormat + } + + pos += int64(headSize) // Skip archive header + + // Track whether we've found the end marker + foundEndMarker := false + + // Process file entries + for !foundEndMarker { + headerData, err := r.readBytes(pos, 7) + if err != nil { + // Don't stop on EOF, might be temporary network error + // For definitive errors, return the error + if !errors.Is(err, io.EOF) && !errors.Is(err, ErrNetworkError) { + return fmt.Errorf("error reading block header: %w", err) + } + + // If we get EOF or network error, retry a few times + retryCount := 0 + maxRetries := 3 + retryDelay := time.Second + + for retryCount < maxRetries { + time.Sleep(retryDelay * time.Duration(1<= 7 { + break // Successfully got data + } + } + + if len(headerData) < 7 { + return fmt.Errorf("failed to read block header after retries: %w", err) + } + } + + if len(headerData) < 7 { + return fmt.Errorf("incomplete block header at position %d", pos) + } + + headType := headerData[2] + headFlags := int(binary.LittleEndian.Uint16(headerData[3:5])) + headSize := int(binary.LittleEndian.Uint16(headerData[5:7])) + + if headType == BlockEnd { + // End of archive + foundEndMarker = true + break + } + + if headType == BlockFile { + // Get complete header data + completeHeader, err := r.readBytes(pos, headSize) + if err != nil || len(completeHeader) < headSize { + // Retry logic for incomplete headers + retryCount := 0 + maxRetries := 3 + retryDelay := time.Second + + for retryCount < maxRetries && (err != nil || len(completeHeader) < headSize) { + time.Sleep(retryDelay * time.Duration(1<= headSize { + break // Successfully got data + } + } + + if len(completeHeader) < headSize { + return fmt.Errorf("failed to read complete file header after retries: %w", err) + } + } + + fileInfo, err := r.parseFileHeader(completeHeader, pos) + if err == nil && fileInfo != nil { + r.Files = append(r.Files, fileInfo) + pos = fileInfo.NextOffset + } else { + pos += int64(headSize) + } + } else { + // Skip non-file block + pos += int64(headSize) + + // Skip data if present + if headFlags&FlagHasData != 0 { + // Read data size + sizeData, err := r.readBytes(pos-4, 4) + if err != nil || len(sizeData) < 4 { + // Retry logic for data size read errors + retryCount := 0 + maxRetries := 3 + retryDelay := time.Second + + for retryCount < maxRetries && (err != nil || len(sizeData) < 4) { + time.Sleep(retryDelay * time.Duration(1<= 4 { + break // Successfully got data + } + } + + if len(sizeData) < 4 { + return fmt.Errorf("failed to read data size after retries: %w", err) + } + } + + dataSize := int64(binary.LittleEndian.Uint32(sizeData)) + pos += dataSize + } + } + } + + if !foundEndMarker { + return fmt.Errorf("end marker not found in archive") + } + + return nil +} + +// parseFileHeader parses a file header and returns file info +func (r *Reader) parseFileHeader(headerData []byte, position int64) (*File, error) { + if len(headerData) < 7 { + return nil, fmt.Errorf("header data too short") + } + + headType := headerData[2] + headFlags := int(binary.LittleEndian.Uint16(headerData[3:5])) + headSize := int(binary.LittleEndian.Uint16(headerData[5:7])) + + if headType != BlockFile { + return nil, fmt.Errorf("not a file block") + } + + // Check if we have enough data + if len(headerData) < 32 { + return nil, fmt.Errorf("file header too short") + } + + // Parse basic file header fields + packSize := binary.LittleEndian.Uint32(headerData[7:11]) + unpackSize := binary.LittleEndian.Uint32(headerData[11:15]) + // fileOS := headerData[15] + fileCRC := binary.LittleEndian.Uint32(headerData[16:20]) + // fileTime := binary.LittleEndian.Uint32(headerData[20:24]) + // unpVer := headerData[24] + method := headerData[25] + nameSize := binary.LittleEndian.Uint16(headerData[26:28]) + // fileAttr := binary.LittleEndian.Uint32(headerData[28:32]) + + // Handle high pack/unp sizes + highPackSize := uint32(0) + highUnpSize := uint32(0) + + offset := 32 // Start after basic header fields + + if headFlags&FlagHasHighSize != 0 { + if offset+8 <= len(headerData) { + highPackSize = binary.LittleEndian.Uint32(headerData[offset : offset+4]) + highUnpSize = binary.LittleEndian.Uint32(headerData[offset+4 : offset+8]) + } + offset += 8 + } + + // Calculate actual sizes + fullPackSize := int64(packSize) + (int64(highPackSize) << 32) + fullUnpSize := int64(unpackSize) + (int64(highUnpSize) << 32) + + // Read filename + var fileName string + if offset+int(nameSize) <= len(headerData) { + fileNameBytes := headerData[offset : offset+int(nameSize)] + + if headFlags&FlagHasUnicodeName != 0 { + zeroPos := bytes.IndexByte(fileNameBytes, 0) + if zeroPos != -1 { + // Try UTF-8 first + asciiPart := fileNameBytes[:zeroPos] + if utf8.Valid(asciiPart) { + fileName = string(asciiPart) + } else { + // Fall back to custom decoder + asciiStr := string(asciiPart) + unicodePart := fileNameBytes[zeroPos+1:] + fileName = decodeUnicode(asciiStr, unicodePart) + } + } else { + // No null byte + if utf8.Valid(fileNameBytes) { + fileName = string(fileNameBytes) + } else { + fileName = string(fileNameBytes) // Last resort + } + } + } else { + // Non-Unicode filename + if utf8.Valid(fileNameBytes) { + fileName = string(fileNameBytes) + } else { + fileName = string(fileNameBytes) // Fallback + } + } + } else { + fileName = fmt.Sprintf("UnknownFile%d", len(r.Files)) + } + + isDirectory := (headFlags & FlagDirectory) == FlagDirectory + + // Calculate data offsets + dataOffset := position + int64(headSize) + nextOffset := dataOffset + + // Only add data size if it's not a directory and has data + if !isDirectory && headFlags&FlagHasData != 0 { + nextOffset += fullPackSize + } + + return &File{ + Path: fileName, + Size: fullUnpSize, + CompressedSize: fullPackSize, + Method: method, + CRC: fileCRC, + IsDirectory: isDirectory, + DataOffset: dataOffset, + NextOffset: nextOffset, + }, nil +} + +// GetFiles returns all files in the archive +func (r *Reader) GetFiles() ([]*File, error) { + if len(r.Files) == 0 { + err := r.readFiles() + if err != nil { + return nil, err + } + } + + return r.Files, nil +} + +// ExtractFile extracts a file from the archive +func (r *Reader) ExtractFile(file *File) ([]byte, error) { + if file.IsDirectory { + return nil, ErrDirectoryExtractNotSupported + } + + // Only support "Store" method + if file.Method != 0x30 { // 0x30 = "Store" + return nil, ErrCompressionNotSupported + } + + return r.readBytes(file.DataOffset, int(file.CompressedSize)) +} + +// Helper functions +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/pkg/rar/types.go b/pkg/rar/types.go new file mode 100644 index 0000000..8156d6a --- /dev/null +++ b/pkg/rar/types.go @@ -0,0 +1,37 @@ +package rar + +import ( + "net/http" + "time" +) + +// File represents a file entry in a RAR archive +type File struct { + Path string + Size int64 + CompressedSize int64 + Method byte + CRC uint32 + IsDirectory bool + DataOffset int64 + NextOffset int64 +} + +// Access point for a RAR archive served through HTTP +type HttpFile struct { + URL string + Position int64 + Client *http.Client + FileSize int64 + MaxRetries int + RetryDelay time.Duration +} + +// Reader reads RAR3 format archives +type Reader struct { + File *HttpFile + ChunkSize int + Marker int64 + HeaderEndPos int64 // Position after the archive header + Files []*File +} diff --git a/pkg/web/templates/config.html b/pkg/web/templates/config.html index b044acb..5b009b4 100644 --- a/pkg/web/templates/config.html +++ b/pkg/web/templates/config.html @@ -363,11 +363,18 @@ Add samples, extras etc when adding torrent to debrid(disabled by default)
+
+ + +
+ Preprocess RARed torrents to allow reading the files inside +
+
- Create an internal webdav for this debrid + Create an internal webdav for this debrid
@@ -1092,6 +1099,7 @@ rate_limit: document.querySelector(`[name="debrid[${i}].rate_limit"]`).value, download_uncached: document.querySelector(`[name="debrid[${i}].download_uncached"]`).checked, check_cached: document.querySelector(`[name="debrid[${i}].check_cached"]`).checked, + unpack_rar: document.querySelector(`[name="debrid[${i}].unpack_rar"]`).checked, add_samples: document.querySelector(`[name="debrid[${i}].add_samples"]`).checked, use_webdav: document.querySelector(`[name="debrid[${i}].use_webdav"]`).checked }; diff --git a/pkg/webdav/file.go b/pkg/webdav/file.go index 528ae7f..babe36a 100644 --- a/pkg/webdav/file.go +++ b/pkg/webdav/file.go @@ -3,12 +3,13 @@ package webdav import ( "crypto/tls" "fmt" - "github.com/sirrobot01/decypharr/pkg/debrid/debrid" "io" "net/http" "os" "strings" "time" + + "github.com/sirrobot01/decypharr/pkg/debrid/debrid" ) var sharedClient = &http.Client{ @@ -76,104 +77,143 @@ func (f *File) getDownloadLink() (string, error) { return "", os.ErrNotExist } -func (f *File) stream() (*http.Response, error) { - client := sharedClient // Might be replaced with the custom client - _log := f.cache.GetLogger() - var ( - err error - downloadLink string - ) - - downloadLink, err = f.getDownloadLink() +func (f *File) getDownloadByteRange() (*[2]int64, error) { + byteRange, err := f.cache.GetDownloadByteRange(f.torrentName, f.name) if err != nil { - - _log.Trace().Msgf("Failed to get download link for %s. %s", f.name, err) - return nil, io.EOF + return nil, err } + return byteRange, nil +} + +func (f *File) stream() (*http.Response, error) { + client := sharedClient + _log := f.cache.GetLogger() + + downloadLink, err := f.getDownloadLink() + if err != nil { + _log.Trace().Msgf("Failed to get download link for %s: %v", f.name, err) + return nil, err + } + if downloadLink == "" { _log.Trace().Msgf("Failed to get download link for %s. Empty download link", f.name) - return nil, io.EOF + return nil, fmt.Errorf("empty download link") + } + + byteRange, err := f.getDownloadByteRange() + if err != nil { + _log.Trace().Msgf("Failed to get download byte range for %s: %v", f.name, err) + return nil, err } req, err := http.NewRequest("GET", downloadLink, nil) if err != nil { - _log.Trace().Msgf("Failed to create HTTP request: %s", err) - return nil, io.EOF + _log.Trace().Msgf("Failed to create HTTP request: %v", err) + return nil, err } - if f.offset > 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", f.offset)) + if byteRange == nil { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", max(0, f.offset))) + } else { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", byteRange[0]+max(0, f.offset))) } + // Make the request resp, err := client.Do(req) if err != nil { - return resp, io.EOF + _log.Trace().Msgf("HTTP request failed: %v", err) + return nil, err } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { f.downloadLink = "" - closeResp := func() { - _, _ = io.Copy(io.Discard, resp.Body) - resp.Body.Close() + + cleanupResp := func() { + if resp.Body != nil { + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + } } - if resp.StatusCode == http.StatusServiceUnavailable { - b, _ := io.ReadAll(resp.Body) - err := resp.Body.Close() - if err != nil { - _log.Trace().Msgf("Failed to close response body: %s", err) - return nil, io.EOF + switch resp.StatusCode { + case http.StatusServiceUnavailable: + // Read the body to check for specific error messages + body, readErr := io.ReadAll(resp.Body) + resp.Body.Close() + + if readErr != nil { + _log.Trace().Msgf("Failed to read response body: %v", readErr) + return nil, fmt.Errorf("failed to read error response: %w", readErr) } - if strings.Contains(string(b), "You can not download this file because you have exceeded your traffic on this hoster") { + + bodyStr := string(body) + if strings.Contains(bodyStr, "You can not download this file because you have exceeded your traffic on this hoster") { _log.Trace().Msgf("Bandwidth exceeded for %s. Download token will be disabled if you have more than one", f.name) f.cache.MarkDownloadLinkAsInvalid(f.link, downloadLink, "bandwidth_exceeded") // Retry with a different API key if it's available return f.stream() - } else { - _log.Trace().Msgf("Failed to get download link for %s. %s", f.name, string(b)) - return resp, io.EOF } - } else if resp.StatusCode == http.StatusNotFound { - closeResp() + return nil, fmt.Errorf("service unavailable: %s", bodyStr) + + case http.StatusNotFound: + cleanupResp() // Mark download link as not found // Regenerate a new download link + _log.Trace().Msgf("File not found (404) for %s. Marking link as invalid and regenerating", f.name) f.cache.MarkDownloadLinkAsInvalid(f.link, downloadLink, "link_not_found") // Generate a new download link - downloadLink, err = f.getDownloadLink() + downloadLink, err := f.getDownloadLink() if err != nil { _log.Trace().Msgf("Failed to get download link for %s. %s", f.name, err) - return nil, io.EOF + return nil, err } + if downloadLink == "" { _log.Trace().Msgf("Failed to get download link for %s", f.name) - return nil, io.EOF - } - req, err = http.NewRequest("GET", downloadLink, nil) - if err != nil { - return nil, io.EOF - } - if f.offset > 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", f.offset)) + return nil, fmt.Errorf("failed to regenerate download link") } - resp, err = client.Do(req) + req, err := http.NewRequest("GET", downloadLink, nil) if err != nil { - return resp, fmt.Errorf("HTTP request error: %w", err) + return nil, err } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - closeResp() - // Read the body to consume the response + + // Set the range header again + if byteRange == nil { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", max(0, f.offset))) + } else { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", byteRange[0]+max(0, f.offset))) + } + + newResp, err := client.Do(req) + if err != nil { + return nil, err + } + + if newResp.StatusCode != http.StatusOK && newResp.StatusCode != http.StatusPartialContent { + cleanupBody := func() { + if newResp.Body != nil { + io.Copy(io.Discard, newResp.Body) + newResp.Body.Close() + } + } + + cleanupBody() + _log.Trace().Msgf("Regenerated link also failed with status %d", newResp.StatusCode) f.cache.MarkDownloadLinkAsInvalid(f.link, downloadLink, "link_not_found") - return resp, io.EOF + return nil, fmt.Errorf("failed with status code %d even after link regeneration", newResp.StatusCode) } - return resp, nil - } else { - closeResp() - return resp, io.EOF + return newResp, nil + + default: + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + _log.Trace().Msgf("Unexpected status code %d for %s: %s", resp.StatusCode, f.name, string(body)) + return nil, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body)) } - } return resp, nil } @@ -196,7 +236,7 @@ func (f *File) Read(p []byte) (n int, err error) { // If we haven't started streaming the file yet or need to reposition if f.reader == nil || f.seekPending { - if f.reader != nil && f.seekPending { + if f.reader != nil { f.reader.Close() f.reader = nil } @@ -207,7 +247,7 @@ func (f *File) Read(p []byte) (n int, err error) { return 0, err } if resp == nil { - return 0, io.EOF + return 0, fmt.Errorf("stream returned nil response") } f.reader = resp.Body