aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorComradeCrow <comradecrow@vivaldi.net>2025-03-20 00:02:56 -0700
committerComradeCrow <comradecrow@vivaldi.net>2025-03-20 00:02:56 -0700
commitedd4c39c2b4cdaa8bfac2a91dcb50d77475c3589 (patch)
tree3aecdd3a8110235f5679f5be7216ad044af4158e
parentdb8f8cc660f8871265ad7633855c0ccbb890bd91 (diff)
downloadpodweb-edd4c39c2b4cdaa8bfac2a91dcb50d77475c3589.tar.gz
receiving data
syncs work and download_episode is getting audio data from the internet. Considering removing the podcasts table, just use podcast_url in episodes and downloads
-rwxr-xr-xPodWeb.py194
-rw-r--r--requirements.txt1
2 files changed, 160 insertions, 35 deletions
diff --git a/PodWeb.py b/PodWeb.py
index 5c2d9e0..a177a8d 100755
--- a/PodWeb.py
+++ b/PodWeb.py
@@ -10,6 +10,7 @@ import urllib.request
import xml.etree.ElementTree as xmlet
import click
+import fake_useragent
import podcastparser
from ruamel.yaml import YAML
@@ -110,6 +111,7 @@ class PodWeb:
if not os.path.exists(self.options["downloadlocation"]):
os.makedirs(options["downloadlocation"])
self._load_serverlist()
+ self._sync_podcasts_table()
def __del__(self):
self._close_db()
@@ -127,32 +129,65 @@ class PodWeb:
def _create_tables(self) -> None:
self.data.execute(
- """CREATE TABLE IF NOT EXISTS "episodes" (
- "guid" TEXT NOT NULL UNIQUE,
- "title" TEXT,
- "description" TEXT,
- "img" TEXT,
- "url" TEXT,
- "website" TEXT,
- "season" INTEGER,
- PRIMARY KEY("guid")
- )"""
+ """
+ CREATE TABLE IF NOT EXISTS "episodes" (
+ "guid" TEXT NOT NULL UNIQUE,
+ "podcast_url" TEXT NOT NULL,
+ "title" TEXT,
+ "description" TEXT,
+ "img" TEXT,
+ "url" TEXT,
+ "website" TEXT,
+ "season" INTEGER,
+ "deleted" BOOL NOT NULL CHECK("deleted" in (0,1)),
+ PRIMARY KEY("guid","podcast_url")
+ )
+ """
)
self.data.execute(
- """CREATE TABLE IF NOT EXISTS "downloads" (
- "guid" TEXT NOT NULL UNIQUE,
- "filepath" TEXT NOT NULL UNIQUE,
- PRIMARY KEY("guid"),
- FOREIGN KEY("guid") REFERENCES "episodes"("guid")
- )"""
+ """
+ CREATE TABLE IF NOT EXISTS "downloads" (
+ "guid" TEXT NOT NULL UNIQUE,
+ "filepath" TEXT NOT NULL UNIQUE,
+ PRIMARY KEY("guid"),
+ FOREIGN KEY("guid") REFERENCES "episodes"("guid")
+ )
+ """
)
self.data.execute(
"""CREATE TABLE IF NOT EXISTS "podcasts" (
"url" TEXT NOT NULL UNIQUE,
"episodes" TEXT,
- PRIMARY KEY("url)
+ PRIMARY KEY("url")
)"""
)
+ self.data.execute(
+ """
+ CREATE UNIQUE INDEX
+ IF NOT EXISTS
+ "downloadGuid" ON "downloads" (
+ "guid"
+ );
+ """
+ )
+ self.data.execute(
+ """
+ CREATE UNIQUE INDEX
+ IF NOT EXISTS
+ "episodeGuid" ON "episodes" (
+ "guid"
+ );
+ """
+ )
+ self.data.execute(
+ """
+ CREATE UNIQUE INDEX
+ IF NOT EXISTS
+ "podcastUrl" ON "podcasts" (
+ "url"
+ );
+ """
+ )
def _load_config(self) -> None:
"""Loads current config"""
@@ -188,7 +223,7 @@ class PodWeb:
return content
if content:
for i in content:
- content["url"] = podcastparser.normalize_feed_url(content["url"])
+ i["url"] = podcastparser.normalize_feed_url(i["url"])
self.servers = content
def _create_serverlist(self) -> None:
@@ -206,9 +241,55 @@ class PodWeb:
f.write(self.DEFAULT_SERVERLIST_HEADING)
yaml.dump(self.servers, f)
+ def _sync_podcasts_table(self) -> None:
+ """Syncs the podcasts table with self.serverlist"""
+ server_urls = []
+ for i in self.servers:
+ self.data.execute(
+ """
+ SELECT COUNT(1)
+ FROM \"podcasts\"
+ WHERE \"url\" = :URL;
+ """,
+ {"URL": i["url"]},
+ )
+ resp = self.data.fetchone()[0]
+ if resp == 0:
+ self.data.execute(
+ """
+ INSERT INTO \"podcasts\"
+ VALUES (
+ :URL,
+ \"\"
+ );
+ """,
+ {"URL": i["url"]},
+ )
+ server_urls.append(i["url"])
+
+ self.data.execute(
+ """
+ SELECT \"url\"
+ FROM \"podcasts\";
+ """
+ )
+
+ db_server_urls = [i[0] for i in self.data.fetchall()]
+ for i in db_server_urls:
+ if i not in server_urls:
+ self.data.execute(
+ """
+ DELETE FROM \"podcasts\"
+ WHERE \"url\" = :URL;
+ """,
+ {"URL": i},
+ )
+ self.con.commit()
+
def add_podcast(
self, feedurl: str, name=None, category=None, site=None, img=None
) -> None:
+ """Adds a new podcast to the serverlist"""
feedurl = podcastparser.normalize_feed_url(feedurl)
parsed = podcastparser.parse(feedurl, urllib.request.urlopen(feedurl))
if parsed.get("newLocation") != None:
@@ -236,7 +317,8 @@ class PodWeb:
self.servers.append(new_feed)
self._update_serverlist()
- def sync_episodes(self, feedurl: str) -> None:
+ def _sync_episodes(self, feedurl: str, min_size: bool = True) -> None:
+ """syncs the available episodes for download for the given feedurl"""
feedurl = podcastparser.normalize_feed_url(feedurl)
parsed = podcastparser.parse(feedurl, urllib.request.urlopen(feedurl))
if parsed.get("newLocation") != None:
@@ -244,11 +326,22 @@ class PodWeb:
for i in self.servers:
if i["url"] == feedurl:
i["url"] = new_feedurl
- feedurl = new_feedurl
self._update_serverlist()
+ self.data.execute(
+ """
+ UPDATE "podcasts"
+ SET "url" = :NEWURL
+ WHERE "url" = :URL
+ """,
+ {"URL": feedurl, "NEWURL": new_feedurl},
+ )
+ feedurl = new_feedurl
break
+ guid_list = []
for i in parsed["episodes"]:
- enclosure_list = sorted(i["enclosures"], key=lambda d: d["file_size"])
+ enclosure_list = sorted(
+ i["enclosures"], key=lambda d: d["file_size"], reverse=min_size
+ )
mime = True
j = 0
size = len(enclosure_list)
@@ -257,44 +350,75 @@ class PodWeb:
if enclosure_list[j]["mime_type"] == "audio/mpeg":
mime = False
else:
- ++j
+ j += 1
self.data.execute(
- """INSERT OR REPLACE INTO "episodes" VALUES (
+ """
+ INSERT OR REPLACE INTO "episodes" VALUES (
:GUID,
+ :PODCAST_URL,
:TITLE,
:DESCRIPTION,
:IMG,
:URL,
:WEBSITE,
- :SEASON
- );""",
+ :SEASON,
+ :DELETED
+ );
+ """,
{
"GUID": i["guid"],
+ "PODCAST_URL": feedurl,
"TITLE": i["title"],
"DESCRIPTION": i["description"],
"IMG": i["episode_art_url"],
"URL": episode_url,
"WEBSITE": i["link"],
- "SEASON": i["number"],
+ "SEASON": i.get("number", -1),
+ "DELETED": False,
},
)
- self.data.execute(
- 'UPDATE "podcasts" WHERE "url" = :URL SET "episodes" += :EPSTRING',
- {"EPSTRING": f";{i['guid']}"},
- )
self.con.commit()
+ guid_list.append(i["guid"])
+ self.data.execute(
+ """
+ SELECT \"guid\"
+ FROM \"episodes\";
+ """
+ )
+ db_guid_list = [i[0] for i in self.data.fetchall()]
+ for i in db_guid_list:
+ if i not in guid_list:
+ self.data.execute(
+ """
+ UPDATE \"episodes\"
+ SET \"deleted\" = 1
+ WHERE \"guid\" = :GUID;
+ """,
+ {"GUID": i},
+ )
def download_episode(self, guid):
response = self.data.execute(
- 'SELECT "url" FROM "episodes" WHERE guid = :GUID;', {"GUID": guid}
+ """
+ SELECT "url","podcast_url"
+ FROM "episodes"
+ WHERE guid = :GUID;
+ """,
+ {"GUID": guid},
)
- url = response.fetchone()
+ url, feedurl = response.fetchone()
if url == None:
click.echo("No URL found!", err=True)
return 1
- url = url[0]
- audio_data = urllib.request.urlopen(url)
- click.echo(audio_data.read())
+ ua = fake_useragent.UserAgent()
+ request_obj = urllib.request.Request(
+ url, data=None, headers={"User-Agent": ua.ff}
+ )
+ try:
+ audio_data = urllib.request.urlopen(request_obj)
+ except urllib.error.HTTPError:
+ self._sync_episodes(feedurl)
+ audio_data = urllib.request.urlopen(request_obj)
def import_opml(self, opml_path: str) -> None:
body = xmlet.parse(source=opml_path).getroot().find("body")
diff --git a/requirements.txt b/requirements.txt
index 9142357..c7f4542 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,4 +5,5 @@ pytest >= 4.6
pytest-cov
coverage
ruamel.yaml
+fake-useragent