experimental

This commit is contained in:
Mukhtar Akere
2025-03-15 21:08:15 +01:00
parent b4e4db27fb
commit 2d29996d2c
11 changed files with 281 additions and 127 deletions

View File

@@ -11,6 +11,7 @@ import (
"github.com/sirrobot01/debrid-blackhole/pkg/service"
"github.com/sirrobot01/debrid-blackhole/pkg/version"
"github.com/sirrobot01/debrid-blackhole/pkg/web"
"github.com/sirrobot01/debrid-blackhole/pkg/webdav"
"github.com/sirrobot01/debrid-blackhole/pkg/worker"
"runtime/debug"
"sync"
@@ -30,12 +31,16 @@ func Start(ctx context.Context) error {
svc := service.New()
_qbit := qbit.New()
srv := server.New()
webRoutes := web.New(_qbit).Routes()
_webdav := webdav.New()
ui := web.New(_qbit).Routes()
webdavRoutes := _webdav.Routes()
qbitRoutes := _qbit.Routes()
// Register routes
srv.Mount("/", webRoutes)
srv.Mount("/", ui)
srv.Mount("/api/v2", qbitRoutes)
srv.Mount("/webdav", webdavRoutes)
safeGo := func(f func() error) {
wg.Add(1)
@@ -66,6 +71,10 @@ func Start(ctx context.Context) error {
})
}
safeGo(func() error {
return _webdav.Start(ctx)
})
safeGo(func() error {
return srv.Start(ctx)
})

18
go.mod
View File

@@ -15,7 +15,7 @@ require (
github.com/rs/zerolog v1.33.0
github.com/valyala/fastjson v1.6.4
golang.org/x/crypto v0.33.0
golang.org/x/net v0.33.0
golang.org/x/net v0.35.0
golang.org/x/sync v0.11.0
golang.org/x/time v0.8.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
@@ -25,15 +25,29 @@ require (
github.com/anacrolix/missinggo v1.3.0 // indirect
github.com/anacrolix/missinggo/v2 v2.7.3 // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgraph-io/badger/v4 v4.6.0 // indirect
github.com/dgraph-io/ristretto/v2 v2.1.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/rogpeppe/go-internal v1.13.1 // indirect
github.com/stretchr/testify v1.10.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel v1.34.0 // indirect
go.opentelemetry.io/otel/metric v1.34.0 // indirect
go.opentelemetry.io/otel/trace v1.34.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/text v0.22.0 // indirect
google.golang.org/protobuf v1.36.5 // indirect
)

32
go.sum
View File

@@ -46,16 +46,25 @@ github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaq
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4=
github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger/v4 v4.6.0 h1:acOwfOOZ4p1dPRnYzvkVm7rUk2Y21TgPVepCy5dJdFQ=
github.com/dgraph-io/badger/v4 v4.6.0/go.mod h1:KSJ5VTuZNC3Sd+YhvVjk2nYua9UZnnTr/SkXvdtiPgI=
github.com/dgraph-io/ristretto/v2 v2.1.0 h1:59LjpOJLNDULHh8MC4UaegN52lC4JnO2dITsie/Pa8I=
github.com/dgraph-io/ristretto/v2 v2.1.0/go.mod h1:uejeqfYXpUomfse0+lO+13ATz4TypQYLJZzBSAemuB4=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
@@ -78,6 +87,11 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -98,6 +112,8 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -132,6 +148,8 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -163,6 +181,7 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
@@ -187,6 +206,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
@@ -215,6 +235,14 @@ github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPy
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=
go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
@@ -236,6 +264,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -288,6 +318,8 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@@ -273,7 +273,7 @@ func (ad *AllDebrid) GetCheckCached() bool {
}
func (ad *AllDebrid) GetTorrents() ([]*torrent.Torrent, error) {
return nil, fmt.Errorf("not implemented")
return nil, nil
}
func (ad *AllDebrid) GetDownloadingStatus() []string {

View File

@@ -1,15 +1,17 @@
package cache
import (
"bufio"
"context"
"encoding/json"
"fmt"
"github.com/dgraph-io/badger/v4"
"github.com/rs/zerolog"
"github.com/sirrobot01/debrid-blackhole/internal/logger"
"os"
"path/filepath"
"runtime"
"sync"
"sync/atomic"
"time"
"github.com/sirrobot01/debrid-blackhole/internal/config"
@@ -35,17 +37,45 @@ var (
func getLogger() zerolog.Logger {
once.Do(func() {
_logInstance = logger.NewLogger("cache", "info", os.Stdout)
cfg := config.GetConfig()
_logInstance = logger.NewLogger("cache", cfg.LogLevel, os.Stdout)
})
return _logInstance
}
type Cache struct {
dir string
client engine.Service
torrents *sync.Map // key: torrent.Id, value: *CachedTorrent
torrentsNames *sync.Map // key: torrent.Name, value: torrent.Id
LastUpdated time.Time `json:"last_updated"`
dir string
client engine.Service
db *badger.DB
torrents map[string]*CachedTorrent // key: torrent.Id, value: *CachedTorrent
torrentsMutex sync.RWMutex
torrentsNames map[string]*CachedTorrent // key: torrent.Name, value: torrent
torrentNamesMutex sync.RWMutex
LastUpdated time.Time `json:"last_updated"`
}
func (c *Cache) SetTorrent(t *CachedTorrent) {
c.torrentsMutex.Lock()
defer c.torrentsMutex.Unlock()
c.torrents[t.Id] = t
}
func (c *Cache) SetTorrentName(name string, t *CachedTorrent) {
c.torrentNamesMutex.Lock()
defer c.torrentNamesMutex.Unlock()
c.torrentsNames[name] = t
}
func (c *Cache) GetTorrents() map[string]*CachedTorrent {
c.torrentsMutex.RLock()
defer c.torrentsMutex.RUnlock()
return c.torrents
}
func (c *Cache) GetTorrentNames() map[string]*CachedTorrent {
c.torrentNamesMutex.RLock()
defer c.torrentNamesMutex.RUnlock()
return c.torrentsNames
}
type Manager struct {
@@ -73,10 +103,11 @@ func (m *Manager) GetCache(debridName string) *Cache {
}
func New(debridService engine.Service, basePath string) *Cache {
dbPath := filepath.Join(basePath, "cache", debridService.GetName(), "db")
return &Cache{
dir: filepath.Join(basePath, "cache", debridService.GetName(), "torrents"),
torrents: &sync.Map{},
torrentsNames: &sync.Map{},
dir: dbPath,
torrents: make(map[string]*CachedTorrent),
torrentsNames: make(map[string]*CachedTorrent),
client: debridService,
}
}
@@ -84,93 +115,117 @@ func New(debridService engine.Service, basePath string) *Cache {
func (c *Cache) Start() error {
_logger := getLogger()
_logger.Info().Msg("Starting cache for: " + c.client.GetName())
// Make sure the directory exists
if err := os.MkdirAll(c.dir, 0755); err != nil {
return fmt.Errorf("failed to create cache directory: %w", err)
}
// Open BadgerDB
opts := badger.DefaultOptions(c.dir)
opts.Logger = nil // Disable Badger's internal logger
var err error
c.db, err = badger.Open(opts)
if err != nil {
return fmt.Errorf("failed to open BadgerDB: %w", err)
}
if err := c.Load(); err != nil {
return fmt.Errorf("failed to load cache: %v", err)
}
if err := c.Sync(); err != nil {
return fmt.Errorf("failed to sync cache: %v", err)
}
return nil
}
func (c *Cache) Close() error {
if c.db != nil {
return c.db.Close()
}
return nil
}
func (c *Cache) Load() error {
_logger := getLogger()
if err := os.MkdirAll(c.dir, 0755); err != nil {
return fmt.Errorf("failed to create cache directory: %w", err)
}
err := c.db.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions
it := txn.NewIterator(opts)
defer it.Close()
files, err := os.ReadDir(c.dir)
if err != nil {
return fmt.Errorf("failed to read cache directory: %w", err)
}
prefix := []byte("torrent:")
for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
item := it.Item()
for _, file := range files {
if file.IsDir() || filepath.Ext(file.Name()) != ".json" {
continue
err := item.Value(func(val []byte) error {
var ct CachedTorrent
if err := json.Unmarshal(val, &ct); err != nil {
_logger.Debug().Err(err).Msgf("Failed to unmarshal torrent")
return nil // Continue to next item
}
if len(ct.Files) > 0 {
c.SetTorrent(&ct)
c.SetTorrentName(ct.Name, &ct)
}
return nil
})
if err != nil {
_logger.Debug().Err(err).Msg("Error reading torrent value")
}
}
return nil
})
filePath := filepath.Join(c.dir, file.Name())
data, err := os.ReadFile(filePath)
if err != nil {
_logger.Debug().Err(err).Msgf("Failed to read file: %s", filePath)
continue
}
var ct CachedTorrent
if err := json.Unmarshal(data, &ct); err != nil {
_logger.Debug().Err(err).Msgf("Failed to unmarshal file: %s", filePath)
continue
}
if len(ct.Files) > 0 {
c.torrents.Store(ct.Torrent.Id, &ct)
c.torrentsNames.Store(ct.Torrent.Name, ct.Torrent.Id)
}
}
return nil
return err
}
func (c *Cache) GetTorrent(id string) *CachedTorrent {
if value, ok := c.torrents.Load(id); ok {
return value.(*CachedTorrent)
if t, ok := c.GetTorrents()[id]; ok {
return t
}
return nil
}
func (c *Cache) GetTorrentByName(name string) *CachedTorrent {
if id, ok := c.torrentsNames.Load(name); ok {
return c.GetTorrent(id.(string))
if t, ok := c.GetTorrentNames()[name]; ok {
return t
}
return nil
}
func (c *Cache) SaveTorrent(ct *CachedTorrent) error {
data, err := json.MarshalIndent(ct, "", " ")
data, err := json.Marshal(ct)
if err != nil {
return fmt.Errorf("failed to marshal torrent: %w", err)
}
fileName := ct.Torrent.Id + ".json"
filePath := filepath.Join(c.dir, fileName)
tmpFile := filePath + ".tmp"
key := []byte(fmt.Sprintf("torrent:%s", ct.Torrent.Id))
err = c.db.Update(func(txn *badger.Txn) error {
return txn.Set(key, data)
})
f, err := os.Create(tmpFile)
if err != nil {
return fmt.Errorf("failed to create temp file: %w", err)
}
defer f.Close()
w := bufio.NewWriter(f)
if _, err := w.Write(data); err != nil {
return fmt.Errorf("failed to write data: %w", err)
return fmt.Errorf("failed to save torrent to BadgerDB: %w", err)
}
if err := w.Flush(); err != nil {
return fmt.Errorf("failed to flush data: %w", err)
// Also create an index by name for quick lookups
nameKey := []byte(fmt.Sprintf("name:%s", ct.Torrent.Name))
err = c.db.Update(func(txn *badger.Txn) error {
return txn.Set(nameKey, []byte(ct.Torrent.Id))
})
if err != nil {
return fmt.Errorf("failed to save torrent name index: %w", err)
}
return os.Rename(tmpFile, filePath)
return nil
}
func (c *Cache) SaveAll() error {
@@ -192,14 +247,23 @@ func (c *Cache) SaveAll() error {
}()
}
c.torrents.Range(func(_, value interface{}) bool {
tasks <- value.(*CachedTorrent)
return true
})
for _, value := range c.GetTorrents() {
tasks <- value
}
close(tasks)
wg.Wait()
c.LastUpdated = time.Now()
// Run value log garbage collection when appropriate
// This helps reclaim space from deleted/updated values
go func() {
err := c.db.RunValueLogGC(0.5) // Run GC if 50% of the value log can be discarded
if err != nil && err != badger.ErrNoRewrite {
_logger.Debug().Err(err).Msg("BadgerDB value log GC")
}
}()
return nil
}
@@ -209,44 +273,76 @@ func (c *Cache) Sync() error {
if err != nil {
return fmt.Errorf("failed to sync torrents: %v", err)
}
_logger.Info().Msgf("Syncing %d torrents", len(torrents))
workers := runtime.NumCPU() * 200
workChan := make(chan *torrent.Torrent, len(torrents))
errChan := make(chan error, len(torrents))
// Calculate optimal workers - balance between CPU and IO
workers := runtime.NumCPU() * 4 // A more balanced multiplier for BadgerDB
// Create channels with appropriate buffering
workChan := make(chan *torrent.Torrent, workers*2)
// Use an atomic counter for progress tracking
var processed int64
var errorCount int64
// Create a context with cancellation in case of critical errors
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Create a wait group for workers
var wg sync.WaitGroup
// Start workers
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for t := range workChan {
if err := c.processTorrent(t); err != nil {
errChan <- err
for {
select {
case t, ok := <-workChan:
if !ok {
return // Channel closed, exit goroutine
}
if err := c.processTorrent(t); err != nil {
_logger.Error().Err(err).Str("torrent", t.Name).Msg("sync error")
atomic.AddInt64(&errorCount, 1)
}
count := atomic.AddInt64(&processed, 1)
if count%1000 == 0 {
_logger.Info().Msgf("Progress: %d/%d torrents processed", count, len(torrents))
}
case <-ctx.Done():
return // Context cancelled, exit goroutine
}
}
}()
}
// Feed work to workers
for _, t := range torrents {
workChan <- t
select {
case workChan <- t:
// Work sent successfully
case <-ctx.Done():
break // Context cancelled
}
}
// Signal workers that no more work is coming
close(workChan)
// Wait for all workers to complete
wg.Wait()
close(errChan)
for err := range errChan {
_logger.Error().Err(err).Msg("sync error")
}
_logger.Info().Msgf("Synced %d torrents", len(torrents))
_logger.Info().Msgf("Sync complete: %d torrents processed, %d errors", len(torrents), errorCount)
return nil
}
func (c *Cache) processTorrent(t *torrent.Torrent) error {
if existing, ok := c.torrents.Load(t.Id); ok {
ct := existing.(*CachedTorrent)
if ct := c.GetTorrent(t.Id); ct != nil {
if ct.IsComplete {
return nil
}
@@ -259,7 +355,7 @@ func (c *Cache) AddTorrent(t *torrent.Torrent) {
_logger := getLogger()
if len(t.Files) == 0 {
tNew, err := c.client.GetTorrent(t.Id)
tNew, err := c.client.GetTorrent(t)
_logger.Debug().Msgf("Getting torrent files for %s", t.Id)
if err != nil {
_logger.Debug().Msgf("Failed to get torrent files for %s: %v", t.Id, err)
@@ -280,8 +376,8 @@ func (c *Cache) AddTorrent(t *torrent.Torrent) {
DownloadLinks: make(map[string]DownloadLinkCache),
}
c.torrents.Store(t.Id, ct)
c.torrentsNames.Store(t.Name, t.Id)
c.SetTorrent(ct)
c.SetTorrentName(t.Name, ct)
go func() {
if err := c.SaveTorrent(ct); err != nil {
@@ -290,12 +386,12 @@ func (c *Cache) AddTorrent(t *torrent.Torrent) {
}()
}
func (c *Cache) RefreshTorrent(torrentId string) *CachedTorrent {
func (c *Cache) RefreshTorrent(torrent *CachedTorrent) *CachedTorrent {
_logger := getLogger()
t, err := c.client.GetTorrent(torrentId)
t, err := c.client.GetTorrent(torrent.Torrent)
if err != nil {
_logger.Debug().Msgf("Failed to get torrent files for %s: %v", torrentId, err)
_logger.Debug().Msgf("Failed to get torrent files for %s: %v", torrent.Id, err)
return nil
}
if len(t.Files) == 0 {
@@ -309,8 +405,8 @@ func (c *Cache) RefreshTorrent(torrentId string) *CachedTorrent {
DownloadLinks: make(map[string]DownloadLinkCache),
}
c.torrents.Store(t.Id, ct)
c.torrentsNames.Store(t.Name, t.Id)
c.SetTorrent(ct)
c.SetTorrentName(t.Name, ct)
go func() {
if err := c.SaveTorrent(ct); err != nil {
@@ -329,7 +425,7 @@ func (c *Cache) GetFileDownloadLink(t *CachedTorrent, file *torrent.File) (strin
}
if file.Link == "" {
t = c.RefreshTorrent(t.Id)
t = c.RefreshTorrent(t)
if t == nil {
return "", fmt.Errorf("torrent not found")
}
@@ -354,7 +450,3 @@ func (c *Cache) GetFileDownloadLink(t *CachedTorrent, file *torrent.File) (strin
return link.DownloadLink, nil
}
func (c *Cache) GetTorrents() *sync.Map {
return c.torrents
}

View File

@@ -292,5 +292,5 @@ func New(dc config.Debrid, cache *cache.Cache) *DebridLink {
}
func (dl *DebridLink) GetTorrents() ([]*torrent.Torrent, error) {
return nil, fmt.Errorf("not implemented")
return nil, nil
}

View File

@@ -345,8 +345,11 @@ func (r *RealDebrid) getTorrents(offset int, limit int) ([]*torrent.Torrent, err
return nil, err
}
torrents := make([]*torrent.Torrent, 0)
filenames := map[string]bool{}
for _, t := range data {
if _, exists := filenames[t.Filename]; exists {
continue
}
torrents = append(torrents, &torrent.Torrent{
Id: t.Id,
Name: t.Filename,
@@ -364,18 +367,10 @@ func (r *RealDebrid) getTorrents(offset int, limit int) ([]*torrent.Torrent, err
func (r *RealDebrid) GetTorrents() ([]*torrent.Torrent, error) {
torrents := make([]*torrent.Torrent, 0)
offset := 0
limit := 5000
for {
ts, err := r.getTorrents(offset, limit)
if err != nil {
break
}
if len(ts) == 0 {
break
}
torrents = append(torrents, ts...)
offset = len(torrents)
}
limit := 1000
ts, _ := r.getTorrents(offset, limit)
torrents = append(torrents, ts...)
offset = len(torrents)
return torrents, nil
}

View File

@@ -329,7 +329,7 @@ func (tb *Torbox) GetCheckCached() bool {
}
func (tb *Torbox) GetTorrents() ([]*torrent.Torrent, error) {
return nil, fmt.Errorf("not implemented")
return nil, nil
}
func New(dc config.Debrid, cache *cache.Cache) *Torbox {

View File

@@ -3,15 +3,17 @@ package service
import (
"github.com/sirrobot01/debrid-blackhole/pkg/arr"
"github.com/sirrobot01/debrid-blackhole/pkg/debrid"
"github.com/sirrobot01/debrid-blackhole/pkg/debrid/cache"
"github.com/sirrobot01/debrid-blackhole/pkg/debrid/engine"
"github.com/sirrobot01/debrid-blackhole/pkg/repair"
"sync"
)
type Service struct {
Repair *repair.Repair
Arr *arr.Storage
Debrid *engine.Engine
Repair *repair.Repair
Arr *arr.Storage
Debrid *engine.Engine
DebridCache *cache.Manager
}
var (
@@ -24,9 +26,10 @@ func New() *Service {
arrs := arr.NewStorage()
deb := debrid.New()
instance = &Service{
Repair: repair.New(arrs),
Arr: arrs,
Debrid: deb,
Repair: repair.New(arrs),
Arr: arrs,
Debrid: deb,
DebridCache: cache.NewManager(deb),
}
})
return instance

View File

@@ -11,6 +11,7 @@ import (
"net/http"
"os"
"path"
"sort"
"strings"
"sync"
"sync/atomic"
@@ -60,19 +61,23 @@ func (h *Handler) refreshRootListing() {
return
}
var files []os.FileInfo
h.cache.GetTorrents().Range(func(key, value interface{}) bool {
cachedTorrent := value.(*cache.CachedTorrent)
torrents := h.cache.GetTorrentNames()
files := make([]os.FileInfo, 0, len(torrents))
for name, cachedTorrent := range torrents {
if cachedTorrent != nil && cachedTorrent.Torrent != nil {
files = append(files, &FileInfo{
name: cachedTorrent.Torrent.Name,
name: name,
size: 0,
mode: 0755 | os.ModeDir,
modTime: time.Now(),
isDir: true,
})
}
return true
}
sort.Slice(files, func(i, j int) bool {
return files[i].Name() < files[j].Name()
})
h.rootListing.Store(files)

View File

@@ -4,8 +4,12 @@ import (
"context"
"fmt"
"github.com/go-chi/chi/v5"
"github.com/sirrobot01/debrid-blackhole/internal/config"
"github.com/sirrobot01/debrid-blackhole/internal/logger"
"github.com/sirrobot01/debrid-blackhole/pkg/service"
"html/template"
"net/http"
"os"
"sync"
)
@@ -14,15 +18,15 @@ type WebDav struct {
}
func New() *WebDav {
//svc := service.GetService()
//cfg := config.GetConfig()
svc := service.GetService()
cfg := config.GetConfig()
w := &WebDav{
Handlers: make([]*Handler, 0),
}
//for name, c := range svc.DebridCache.GetCaches() {
// h := NewHandler(name, c, logger.NewLogger(fmt.Sprintf("%s-webdav", name), cfg.LogLevel, os.Stdout))
// w.Handlers = append(w.Handlers, h)
//}
for name, c := range svc.DebridCache.GetCaches() {
h := NewHandler(name, c, logger.NewLogger(fmt.Sprintf("%s-webdav", name), cfg.LogLevel, os.Stdout))
w.Handlers = append(w.Handlers, h)
}
return w
}