diff --git a/cmd/decypharr/main.go b/cmd/decypharr/main.go index 9aaaea0..ac42c6a 100644 --- a/cmd/decypharr/main.go +++ b/cmd/decypharr/main.go @@ -11,6 +11,7 @@ import ( "github.com/sirrobot01/debrid-blackhole/pkg/service" "github.com/sirrobot01/debrid-blackhole/pkg/version" "github.com/sirrobot01/debrid-blackhole/pkg/web" + "github.com/sirrobot01/debrid-blackhole/pkg/webdav" "github.com/sirrobot01/debrid-blackhole/pkg/worker" "runtime/debug" "sync" @@ -30,12 +31,16 @@ func Start(ctx context.Context) error { svc := service.New() _qbit := qbit.New() srv := server.New() - webRoutes := web.New(_qbit).Routes() + _webdav := webdav.New() + + ui := web.New(_qbit).Routes() + webdavRoutes := _webdav.Routes() qbitRoutes := _qbit.Routes() // Register routes - srv.Mount("/", webRoutes) + srv.Mount("/", ui) srv.Mount("/api/v2", qbitRoutes) + srv.Mount("/webdav", webdavRoutes) safeGo := func(f func() error) { wg.Add(1) @@ -66,6 +71,10 @@ func Start(ctx context.Context) error { }) } + safeGo(func() error { + return _webdav.Start(ctx) + }) + safeGo(func() error { return srv.Start(ctx) }) diff --git a/go.mod b/go.mod index 8398fe2..9f7940b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/rs/zerolog v1.33.0 github.com/valyala/fastjson v1.6.4 golang.org/x/crypto v0.33.0 - golang.org/x/net v0.33.0 + golang.org/x/net v0.35.0 golang.org/x/sync v0.11.0 golang.org/x/time v0.8.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -25,15 +25,29 @@ require ( github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo/v2 v2.7.3 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgraph-io/badger/v4 v4.6.0 // indirect + github.com/dgraph-io/ristretto/v2 v2.1.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/flatbuffers v25.2.10+incompatible // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/huandu/xstrings v1.3.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/stretchr/testify v1.10.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect golang.org/x/sys v0.30.0 // indirect golang.org/x/text v0.22.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect ) diff --git a/go.sum b/go.sum index 59ecc6b..1699181 100644 --- a/go.sum +++ b/go.sum @@ -46,16 +46,25 @@ github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaq github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v4 v4.6.0 h1:acOwfOOZ4p1dPRnYzvkVm7rUk2Y21TgPVepCy5dJdFQ= +github.com/dgraph-io/badger/v4 v4.6.0/go.mod h1:KSJ5VTuZNC3Sd+YhvVjk2nYua9UZnnTr/SkXvdtiPgI= +github.com/dgraph-io/ristretto/v2 v2.1.0 h1:59LjpOJLNDULHh8MC4UaegN52lC4JnO2dITsie/Pa8I= +github.com/dgraph-io/ristretto/v2 v2.1.0/go.mod h1:uejeqfYXpUomfse0+lO+13ATz4TypQYLJZzBSAemuB4= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -78,6 +87,11 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -98,6 +112,8 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -132,6 +148,8 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -163,6 +181,7 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -187,6 +206,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= @@ -215,6 +235,14 @@ github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPy go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= @@ -236,6 +264,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -288,6 +318,8 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/debrid/alldebrid/alldebrid.go b/pkg/debrid/alldebrid/alldebrid.go index 190b5cb..fd509c2 100644 --- a/pkg/debrid/alldebrid/alldebrid.go +++ b/pkg/debrid/alldebrid/alldebrid.go @@ -273,7 +273,7 @@ func (ad *AllDebrid) GetCheckCached() bool { } func (ad *AllDebrid) GetTorrents() ([]*torrent.Torrent, error) { - return nil, fmt.Errorf("not implemented") + return nil, nil } func (ad *AllDebrid) GetDownloadingStatus() []string { diff --git a/pkg/debrid/cache/cache.go b/pkg/debrid/cache/cache.go index 348a421..849a173 100644 --- a/pkg/debrid/cache/cache.go +++ b/pkg/debrid/cache/cache.go @@ -1,15 +1,17 @@ package cache import ( - "bufio" + "context" "encoding/json" "fmt" + "github.com/dgraph-io/badger/v4" "github.com/rs/zerolog" "github.com/sirrobot01/debrid-blackhole/internal/logger" "os" "path/filepath" "runtime" "sync" + "sync/atomic" "time" "github.com/sirrobot01/debrid-blackhole/internal/config" @@ -35,17 +37,45 @@ var ( func getLogger() zerolog.Logger { once.Do(func() { - _logInstance = logger.NewLogger("cache", "info", os.Stdout) + cfg := config.GetConfig() + _logInstance = logger.NewLogger("cache", cfg.LogLevel, os.Stdout) }) return _logInstance } type Cache struct { - dir string - client engine.Service - torrents *sync.Map // key: torrent.Id, value: *CachedTorrent - torrentsNames *sync.Map // key: torrent.Name, value: torrent.Id - LastUpdated time.Time `json:"last_updated"` + dir string + client engine.Service + db *badger.DB + torrents map[string]*CachedTorrent // key: torrent.Id, value: *CachedTorrent + torrentsMutex sync.RWMutex + torrentsNames map[string]*CachedTorrent // key: torrent.Name, value: torrent + torrentNamesMutex sync.RWMutex + LastUpdated time.Time `json:"last_updated"` +} + +func (c *Cache) SetTorrent(t *CachedTorrent) { + c.torrentsMutex.Lock() + defer c.torrentsMutex.Unlock() + c.torrents[t.Id] = t +} + +func (c *Cache) SetTorrentName(name string, t *CachedTorrent) { + c.torrentNamesMutex.Lock() + defer c.torrentNamesMutex.Unlock() + c.torrentsNames[name] = t +} + +func (c *Cache) GetTorrents() map[string]*CachedTorrent { + c.torrentsMutex.RLock() + defer c.torrentsMutex.RUnlock() + return c.torrents +} + +func (c *Cache) GetTorrentNames() map[string]*CachedTorrent { + c.torrentNamesMutex.RLock() + defer c.torrentNamesMutex.RUnlock() + return c.torrentsNames } type Manager struct { @@ -73,10 +103,11 @@ func (m *Manager) GetCache(debridName string) *Cache { } func New(debridService engine.Service, basePath string) *Cache { + dbPath := filepath.Join(basePath, "cache", debridService.GetName(), "db") return &Cache{ - dir: filepath.Join(basePath, "cache", debridService.GetName(), "torrents"), - torrents: &sync.Map{}, - torrentsNames: &sync.Map{}, + dir: dbPath, + torrents: make(map[string]*CachedTorrent), + torrentsNames: make(map[string]*CachedTorrent), client: debridService, } } @@ -84,93 +115,117 @@ func New(debridService engine.Service, basePath string) *Cache { func (c *Cache) Start() error { _logger := getLogger() _logger.Info().Msg("Starting cache for: " + c.client.GetName()) + + // Make sure the directory exists + if err := os.MkdirAll(c.dir, 0755); err != nil { + return fmt.Errorf("failed to create cache directory: %w", err) + } + + // Open BadgerDB + opts := badger.DefaultOptions(c.dir) + opts.Logger = nil // Disable Badger's internal logger + + var err error + c.db, err = badger.Open(opts) + if err != nil { + return fmt.Errorf("failed to open BadgerDB: %w", err) + } + if err := c.Load(); err != nil { return fmt.Errorf("failed to load cache: %v", err) } + if err := c.Sync(); err != nil { return fmt.Errorf("failed to sync cache: %v", err) } + + return nil +} + +func (c *Cache) Close() error { + if c.db != nil { + return c.db.Close() + } return nil } func (c *Cache) Load() error { _logger := getLogger() - if err := os.MkdirAll(c.dir, 0755); err != nil { - return fmt.Errorf("failed to create cache directory: %w", err) - } + err := c.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + it := txn.NewIterator(opts) + defer it.Close() - files, err := os.ReadDir(c.dir) - if err != nil { - return fmt.Errorf("failed to read cache directory: %w", err) - } + prefix := []byte("torrent:") + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + item := it.Item() - for _, file := range files { - if file.IsDir() || filepath.Ext(file.Name()) != ".json" { - continue + err := item.Value(func(val []byte) error { + var ct CachedTorrent + if err := json.Unmarshal(val, &ct); err != nil { + _logger.Debug().Err(err).Msgf("Failed to unmarshal torrent") + return nil // Continue to next item + } + + if len(ct.Files) > 0 { + c.SetTorrent(&ct) + c.SetTorrentName(ct.Name, &ct) + } + return nil + }) + + if err != nil { + _logger.Debug().Err(err).Msg("Error reading torrent value") + } } + return nil + }) - filePath := filepath.Join(c.dir, file.Name()) - data, err := os.ReadFile(filePath) - if err != nil { - _logger.Debug().Err(err).Msgf("Failed to read file: %s", filePath) - continue - } - - var ct CachedTorrent - if err := json.Unmarshal(data, &ct); err != nil { - _logger.Debug().Err(err).Msgf("Failed to unmarshal file: %s", filePath) - continue - } - if len(ct.Files) > 0 { - c.torrents.Store(ct.Torrent.Id, &ct) - c.torrentsNames.Store(ct.Torrent.Name, ct.Torrent.Id) - } - } - - return nil + return err } func (c *Cache) GetTorrent(id string) *CachedTorrent { - if value, ok := c.torrents.Load(id); ok { - return value.(*CachedTorrent) + if t, ok := c.GetTorrents()[id]; ok { + return t } return nil } func (c *Cache) GetTorrentByName(name string) *CachedTorrent { - if id, ok := c.torrentsNames.Load(name); ok { - return c.GetTorrent(id.(string)) + if t, ok := c.GetTorrentNames()[name]; ok { + return t } return nil } func (c *Cache) SaveTorrent(ct *CachedTorrent) error { - data, err := json.MarshalIndent(ct, "", " ") + data, err := json.Marshal(ct) if err != nil { return fmt.Errorf("failed to marshal torrent: %w", err) } - fileName := ct.Torrent.Id + ".json" - filePath := filepath.Join(c.dir, fileName) - tmpFile := filePath + ".tmp" + key := []byte(fmt.Sprintf("torrent:%s", ct.Torrent.Id)) + + err = c.db.Update(func(txn *badger.Txn) error { + return txn.Set(key, data) + }) - f, err := os.Create(tmpFile) if err != nil { - return fmt.Errorf("failed to create temp file: %w", err) - } - defer f.Close() - - w := bufio.NewWriter(f) - if _, err := w.Write(data); err != nil { - return fmt.Errorf("failed to write data: %w", err) + return fmt.Errorf("failed to save torrent to BadgerDB: %w", err) } - if err := w.Flush(); err != nil { - return fmt.Errorf("failed to flush data: %w", err) + // Also create an index by name for quick lookups + nameKey := []byte(fmt.Sprintf("name:%s", ct.Torrent.Name)) + err = c.db.Update(func(txn *badger.Txn) error { + return txn.Set(nameKey, []byte(ct.Torrent.Id)) + }) + + if err != nil { + return fmt.Errorf("failed to save torrent name index: %w", err) } - return os.Rename(tmpFile, filePath) + return nil } func (c *Cache) SaveAll() error { @@ -192,14 +247,23 @@ func (c *Cache) SaveAll() error { }() } - c.torrents.Range(func(_, value interface{}) bool { - tasks <- value.(*CachedTorrent) - return true - }) + for _, value := range c.GetTorrents() { + tasks <- value + } close(tasks) wg.Wait() c.LastUpdated = time.Now() + + // Run value log garbage collection when appropriate + // This helps reclaim space from deleted/updated values + go func() { + err := c.db.RunValueLogGC(0.5) // Run GC if 50% of the value log can be discarded + if err != nil && err != badger.ErrNoRewrite { + _logger.Debug().Err(err).Msg("BadgerDB value log GC") + } + }() + return nil } @@ -209,44 +273,76 @@ func (c *Cache) Sync() error { if err != nil { return fmt.Errorf("failed to sync torrents: %v", err) } + _logger.Info().Msgf("Syncing %d torrents", len(torrents)) - workers := runtime.NumCPU() * 200 - workChan := make(chan *torrent.Torrent, len(torrents)) - errChan := make(chan error, len(torrents)) + // Calculate optimal workers - balance between CPU and IO + workers := runtime.NumCPU() * 4 // A more balanced multiplier for BadgerDB + // Create channels with appropriate buffering + workChan := make(chan *torrent.Torrent, workers*2) + + // Use an atomic counter for progress tracking + var processed int64 + var errorCount int64 + + // Create a context with cancellation in case of critical errors + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Create a wait group for workers var wg sync.WaitGroup + // Start workers for i := 0; i < workers; i++ { wg.Add(1) go func() { defer wg.Done() - for t := range workChan { - if err := c.processTorrent(t); err != nil { - errChan <- err + for { + select { + case t, ok := <-workChan: + if !ok { + return // Channel closed, exit goroutine + } + + if err := c.processTorrent(t); err != nil { + _logger.Error().Err(err).Str("torrent", t.Name).Msg("sync error") + atomic.AddInt64(&errorCount, 1) + } + + count := atomic.AddInt64(&processed, 1) + if count%1000 == 0 { + _logger.Info().Msgf("Progress: %d/%d torrents processed", count, len(torrents)) + } + + case <-ctx.Done(): + return // Context cancelled, exit goroutine } } }() } + // Feed work to workers for _, t := range torrents { - workChan <- t + select { + case workChan <- t: + // Work sent successfully + case <-ctx.Done(): + break // Context cancelled + } } + + // Signal workers that no more work is coming close(workChan) + // Wait for all workers to complete wg.Wait() - close(errChan) - for err := range errChan { - _logger.Error().Err(err).Msg("sync error") - } - - _logger.Info().Msgf("Synced %d torrents", len(torrents)) + _logger.Info().Msgf("Sync complete: %d torrents processed, %d errors", len(torrents), errorCount) return nil } func (c *Cache) processTorrent(t *torrent.Torrent) error { - if existing, ok := c.torrents.Load(t.Id); ok { - ct := existing.(*CachedTorrent) + if ct := c.GetTorrent(t.Id); ct != nil { if ct.IsComplete { return nil } @@ -259,7 +355,7 @@ func (c *Cache) AddTorrent(t *torrent.Torrent) { _logger := getLogger() if len(t.Files) == 0 { - tNew, err := c.client.GetTorrent(t.Id) + tNew, err := c.client.GetTorrent(t) _logger.Debug().Msgf("Getting torrent files for %s", t.Id) if err != nil { _logger.Debug().Msgf("Failed to get torrent files for %s: %v", t.Id, err) @@ -280,8 +376,8 @@ func (c *Cache) AddTorrent(t *torrent.Torrent) { DownloadLinks: make(map[string]DownloadLinkCache), } - c.torrents.Store(t.Id, ct) - c.torrentsNames.Store(t.Name, t.Id) + c.SetTorrent(ct) + c.SetTorrentName(t.Name, ct) go func() { if err := c.SaveTorrent(ct); err != nil { @@ -290,12 +386,12 @@ func (c *Cache) AddTorrent(t *torrent.Torrent) { }() } -func (c *Cache) RefreshTorrent(torrentId string) *CachedTorrent { +func (c *Cache) RefreshTorrent(torrent *CachedTorrent) *CachedTorrent { _logger := getLogger() - t, err := c.client.GetTorrent(torrentId) + t, err := c.client.GetTorrent(torrent.Torrent) if err != nil { - _logger.Debug().Msgf("Failed to get torrent files for %s: %v", torrentId, err) + _logger.Debug().Msgf("Failed to get torrent files for %s: %v", torrent.Id, err) return nil } if len(t.Files) == 0 { @@ -309,8 +405,8 @@ func (c *Cache) RefreshTorrent(torrentId string) *CachedTorrent { DownloadLinks: make(map[string]DownloadLinkCache), } - c.torrents.Store(t.Id, ct) - c.torrentsNames.Store(t.Name, t.Id) + c.SetTorrent(ct) + c.SetTorrentName(t.Name, ct) go func() { if err := c.SaveTorrent(ct); err != nil { @@ -329,7 +425,7 @@ func (c *Cache) GetFileDownloadLink(t *CachedTorrent, file *torrent.File) (strin } if file.Link == "" { - t = c.RefreshTorrent(t.Id) + t = c.RefreshTorrent(t) if t == nil { return "", fmt.Errorf("torrent not found") } @@ -354,7 +450,3 @@ func (c *Cache) GetFileDownloadLink(t *CachedTorrent, file *torrent.File) (strin return link.DownloadLink, nil } - -func (c *Cache) GetTorrents() *sync.Map { - return c.torrents -} diff --git a/pkg/debrid/debrid_link/debrid_link.go b/pkg/debrid/debrid_link/debrid_link.go index 059eaf5..5cbd447 100644 --- a/pkg/debrid/debrid_link/debrid_link.go +++ b/pkg/debrid/debrid_link/debrid_link.go @@ -292,5 +292,5 @@ func New(dc config.Debrid, cache *cache.Cache) *DebridLink { } func (dl *DebridLink) GetTorrents() ([]*torrent.Torrent, error) { - return nil, fmt.Errorf("not implemented") + return nil, nil } diff --git a/pkg/debrid/realdebrid/realdebrid.go b/pkg/debrid/realdebrid/realdebrid.go index 7a64ff7..8d25bf8 100644 --- a/pkg/debrid/realdebrid/realdebrid.go +++ b/pkg/debrid/realdebrid/realdebrid.go @@ -345,8 +345,11 @@ func (r *RealDebrid) getTorrents(offset int, limit int) ([]*torrent.Torrent, err return nil, err } torrents := make([]*torrent.Torrent, 0) + filenames := map[string]bool{} for _, t := range data { - + if _, exists := filenames[t.Filename]; exists { + continue + } torrents = append(torrents, &torrent.Torrent{ Id: t.Id, Name: t.Filename, @@ -364,18 +367,10 @@ func (r *RealDebrid) getTorrents(offset int, limit int) ([]*torrent.Torrent, err func (r *RealDebrid) GetTorrents() ([]*torrent.Torrent, error) { torrents := make([]*torrent.Torrent, 0) offset := 0 - limit := 5000 - for { - ts, err := r.getTorrents(offset, limit) - if err != nil { - break - } - if len(ts) == 0 { - break - } - torrents = append(torrents, ts...) - offset = len(torrents) - } + limit := 1000 + ts, _ := r.getTorrents(offset, limit) + torrents = append(torrents, ts...) + offset = len(torrents) return torrents, nil } diff --git a/pkg/debrid/torbox/torbox.go b/pkg/debrid/torbox/torbox.go index d95c7f2..b9346ed 100644 --- a/pkg/debrid/torbox/torbox.go +++ b/pkg/debrid/torbox/torbox.go @@ -329,7 +329,7 @@ func (tb *Torbox) GetCheckCached() bool { } func (tb *Torbox) GetTorrents() ([]*torrent.Torrent, error) { - return nil, fmt.Errorf("not implemented") + return nil, nil } func New(dc config.Debrid, cache *cache.Cache) *Torbox { diff --git a/pkg/service/service.go b/pkg/service/service.go index 104a7c6..1c9e216 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -3,15 +3,17 @@ package service import ( "github.com/sirrobot01/debrid-blackhole/pkg/arr" "github.com/sirrobot01/debrid-blackhole/pkg/debrid" + "github.com/sirrobot01/debrid-blackhole/pkg/debrid/cache" "github.com/sirrobot01/debrid-blackhole/pkg/debrid/engine" "github.com/sirrobot01/debrid-blackhole/pkg/repair" "sync" ) type Service struct { - Repair *repair.Repair - Arr *arr.Storage - Debrid *engine.Engine + Repair *repair.Repair + Arr *arr.Storage + Debrid *engine.Engine + DebridCache *cache.Manager } var ( @@ -24,9 +26,10 @@ func New() *Service { arrs := arr.NewStorage() deb := debrid.New() instance = &Service{ - Repair: repair.New(arrs), - Arr: arrs, - Debrid: deb, + Repair: repair.New(arrs), + Arr: arrs, + Debrid: deb, + DebridCache: cache.NewManager(deb), } }) return instance diff --git a/pkg/webdav/handler.go b/pkg/webdav/handler.go index 33c18a1..170f7dc 100644 --- a/pkg/webdav/handler.go +++ b/pkg/webdav/handler.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "path" + "sort" "strings" "sync" "sync/atomic" @@ -60,19 +61,23 @@ func (h *Handler) refreshRootListing() { return } - var files []os.FileInfo - h.cache.GetTorrents().Range(func(key, value interface{}) bool { - cachedTorrent := value.(*cache.CachedTorrent) + torrents := h.cache.GetTorrentNames() + files := make([]os.FileInfo, 0, len(torrents)) + + for name, cachedTorrent := range torrents { if cachedTorrent != nil && cachedTorrent.Torrent != nil { files = append(files, &FileInfo{ - name: cachedTorrent.Torrent.Name, + name: name, size: 0, mode: 0755 | os.ModeDir, modTime: time.Now(), isDir: true, }) } - return true + } + + sort.Slice(files, func(i, j int) bool { + return files[i].Name() < files[j].Name() }) h.rootListing.Store(files) diff --git a/pkg/webdav/webdav.go b/pkg/webdav/webdav.go index a14e639..6457974 100644 --- a/pkg/webdav/webdav.go +++ b/pkg/webdav/webdav.go @@ -4,8 +4,12 @@ import ( "context" "fmt" "github.com/go-chi/chi/v5" + "github.com/sirrobot01/debrid-blackhole/internal/config" + "github.com/sirrobot01/debrid-blackhole/internal/logger" + "github.com/sirrobot01/debrid-blackhole/pkg/service" "html/template" "net/http" + "os" "sync" ) @@ -14,15 +18,15 @@ type WebDav struct { } func New() *WebDav { - //svc := service.GetService() - //cfg := config.GetConfig() + svc := service.GetService() + cfg := config.GetConfig() w := &WebDav{ Handlers: make([]*Handler, 0), } - //for name, c := range svc.DebridCache.GetCaches() { - // h := NewHandler(name, c, logger.NewLogger(fmt.Sprintf("%s-webdav", name), cfg.LogLevel, os.Stdout)) - // w.Handlers = append(w.Handlers, h) - //} + for name, c := range svc.DebridCache.GetCaches() { + h := NewHandler(name, c, logger.NewLogger(fmt.Sprintf("%s-webdav", name), cfg.LogLevel, os.Stdout)) + w.Handlers = append(w.Handlers, h) + } return w }