Skip to content

Commit 3676fb4

Browse files
authored
Merge pull request #384 from g0ldyy/development
refactor: better organization + yggtorrent scraper
2 parents b53aac6 + 5646055 commit 3676fb4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+2819
-2440
lines changed

‎.env-sample‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ BACKGROUND_SCRAPER_MAX_MOVIES_PER_RUN=100 # Maximum number of movies to scrape p
5858
BACKGROUND_SCRAPER_MAX_SERIES_PER_RUN=100 # Maximum number of series to scrape per run
5959

6060
# ============================== #
61-
# Debrid Proxy Configuration #
61+
# Proxy Configuration #
6262
# ============================== #
63-
DEBRID_PROXY_URL=http://warp:1080 # Bypass Debrid Services and Torrentio IP blacklist
63+
BYPASS_PROXY_URL=http://warp:1080 # To bypass scraper IP blacklists
6464

6565
# ============================== #
6666
# Indexer Manager Settings #
@@ -122,6 +122,8 @@ STREMTHRU_SCRAPE_URL=https://stremthru.13377001.xyz
122122

123123
SCRAPE_BITMAGNET=False
124124
BITMAGNET_URL=https://bitmagnetfortheweebs.midnightignite.me
125+
BITMAGNET_MAX_CONCURRENT_PAGES=5
126+
BITMAGNET_MAX_OFFSET=15000 # Maximum number of entries to scrape
125127
# Multi-instance example:
126128
# BITMAGNET_URL='["https://bitmagnet1.example.com", "https://bitmagnet2.example.com"]'
127129

@@ -161,6 +163,11 @@ DEBRIDIO_PROVIDER_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
161163
SCRAPE_TORBOX=False
162164
TORBOX_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
163165

166+
SCRAPE_YGGTORRENT=False
167+
YGGTORRENT_USERNAME=username
168+
YGGTORRENT_PASSWORD=password
169+
YGGTORRENT_MAX_CONCURRENT_PAGES=5
170+
164171
# ============================== #
165172
# Debrid Stream Proxy Settings #
166173
# ============================== #

‎.github/workflows/docker-build-push.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ on:
44
push:
55
branches:
66
- main
7+
- development
78
workflow_dispatch:
89

910
jobs:
@@ -49,6 +50,8 @@ jobs:
4950
run: |
5051
if [ "${{ steps.branch-name.outputs.branch }}" = "main" ]; then
5152
echo "tags=ghcr.io/g0ldyy/comet:latest,docker.io/g0ldyy/comet:latest" >> $GITHUB_OUTPUT
53+
elif [ "${{ steps.branch-name.outputs.branch }}" = "development" ]; then
54+
echo "tags=ghcr.io/g0ldyy/comet:dev,docker.io/g0ldyy/comet:dev" >> $GITHUB_OUTPUT
5255
else
5356
echo "tags=ghcr.io/g0ldyy/comet:${{ steps.branch-name.outputs.branch }},docker.io/g0ldyy/comet:${{ steps.branch-name.outputs.branch }}" >> $GITHUB_OUTPUT
5457
fi

‎comet/api/app.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import asyncio
2+
import time
3+
from contextlib import asynccontextmanager
4+
5+
import aiohttp
6+
from fastapi import FastAPI
7+
from fastapi.middleware.cors import CORSMiddleware
8+
from fastapi.staticfiles import StaticFiles
9+
from starlette.middleware.base import BaseHTTPMiddleware
10+
from starlette.requests import Request
11+
12+
from comet.api.endpoints import admin, base, config, manifest, playback
13+
from comet.api.endpoints import stream as streams_router
14+
from comet.background_scraper.worker import background_scraper
15+
from comet.core.database import (cleanup_expired_locks,
16+
cleanup_expired_sessions, setup_database,
17+
teardown_database)
18+
from comet.core.logger import logger
19+
from comet.core.models import settings
20+
from comet.services.anime import anime_mapper
21+
from comet.services.bandwidth import bandwidth_monitor
22+
from comet.services.trackers import download_best_trackers
23+
24+
25+
class LoguruMiddleware(BaseHTTPMiddleware):
26+
async def dispatch(self, request: Request, call_next):
27+
start_time = time.time()
28+
try:
29+
response = await call_next(request)
30+
except Exception as e:
31+
logger.exception(f"Exception during request processing: {e}")
32+
raise
33+
finally:
34+
process_time = time.time() - start_time
35+
logger.log(
36+
"API",
37+
f"{request.method} {request.url.path} - {response.status_code if 'response' in locals() else '500'} - {process_time:.2f}s",
38+
)
39+
return response
40+
41+
42+
@asynccontextmanager
43+
async def lifespan(app: FastAPI):
44+
# loop = asyncio.get_running_loop()
45+
# loop.set_debug(True)
46+
47+
await setup_database()
48+
await download_best_trackers()
49+
50+
# Load anime ID mapping for enhanced metadata and anime detection
51+
async with aiohttp.ClientSession() as session:
52+
await anime_mapper.load_anime_mapping(session)
53+
54+
# Initialize bandwidth monitoring system
55+
if settings.PROXY_DEBRID_STREAM:
56+
await bandwidth_monitor.initialize()
57+
58+
# Start background cleanup tasks
59+
cleanup_locks_task = asyncio.create_task(cleanup_expired_locks())
60+
cleanup_sessions_task = asyncio.create_task(cleanup_expired_sessions())
61+
62+
# Start background scraper if enabled
63+
background_scraper_task = None
64+
if settings.BACKGROUND_SCRAPER_ENABLED:
65+
background_scraper_task = asyncio.create_task(background_scraper.start())
66+
67+
try:
68+
yield
69+
finally:
70+
if background_scraper_task:
71+
await background_scraper.stop()
72+
background_scraper_task.cancel()
73+
try:
74+
await background_scraper_task
75+
except asyncio.CancelledError:
76+
pass
77+
78+
cleanup_locks_task.cancel()
79+
cleanup_sessions_task.cancel()
80+
try:
81+
await cleanup_locks_task
82+
except asyncio.CancelledError:
83+
pass
84+
try:
85+
await cleanup_sessions_task
86+
except asyncio.CancelledError:
87+
pass
88+
89+
if settings.PROXY_DEBRID_STREAM:
90+
await bandwidth_monitor.shutdown()
91+
92+
await teardown_database()
93+
94+
95+
app = FastAPI(
96+
title="Comet",
97+
summary="Stremio's fastest torrent/debrid search add-on.",
98+
lifespan=lifespan,
99+
redoc_url=None,
100+
)
101+
102+
103+
app.add_middleware(LoguruMiddleware)
104+
app.add_middleware(
105+
CORSMiddleware,
106+
allow_origins=["*"],
107+
allow_credentials=True,
108+
allow_methods=["*"],
109+
allow_headers=["*"],
110+
)
111+
112+
app.mount("/static", StaticFiles(directory="comet/templates"), name="static")
113+
114+
app.include_router(base.router)
115+
app.include_router(config.router)
116+
app.include_router(manifest.router)
117+
app.include_router(admin.router)
118+
app.include_router(playback.router)
119+
app.include_router(streams_router.streams)

0 commit comments

Comments
 (0)