diff options
| -rwxr-xr-x | PodWeb.py | 194 | ||||
| -rw-r--r-- | requirements.txt | 1 |
2 files changed, 160 insertions, 35 deletions
diff --git a/PodWeb.py b/PodWeb.py index 5c2d9e0..a177a8d 100755 --- a/PodWeb.py +++ b/PodWeb.py @@ -10,6 +10,7 @@ import urllib.request import xml.etree.ElementTree as xmlet import click +import fake_useragent import podcastparser from ruamel.yaml import YAML @@ -110,6 +111,7 @@ class PodWeb: if not os.path.exists(self.options["downloadlocation"]): os.makedirs(options["downloadlocation"]) self._load_serverlist() + self._sync_podcasts_table() def __del__(self): self._close_db() @@ -127,32 +129,65 @@ class PodWeb: def _create_tables(self) -> None: self.data.execute( - """CREATE TABLE IF NOT EXISTS "episodes" ( - "guid" TEXT NOT NULL UNIQUE, - "title" TEXT, - "description" TEXT, - "img" TEXT, - "url" TEXT, - "website" TEXT, - "season" INTEGER, - PRIMARY KEY("guid") - )""" + """ + CREATE TABLE IF NOT EXISTS "episodes" ( + "guid" TEXT NOT NULL UNIQUE, + "podcast_url" TEXT NOT NULL, + "title" TEXT, + "description" TEXT, + "img" TEXT, + "url" TEXT, + "website" TEXT, + "season" INTEGER, + "deleted" BOOL NOT NULL CHECK("deleted" in (0,1)), + PRIMARY KEY("guid","podcast_url") + ) + """ ) self.data.execute( - """CREATE TABLE IF NOT EXISTS "downloads" ( - "guid" TEXT NOT NULL UNIQUE, - "filepath" TEXT NOT NULL UNIQUE, - PRIMARY KEY("guid"), - FOREIGN KEY("guid") REFERENCES "episodes"("guid") - )""" + """ + CREATE TABLE IF NOT EXISTS "downloads" ( + "guid" TEXT NOT NULL UNIQUE, + "filepath" TEXT NOT NULL UNIQUE, + PRIMARY KEY("guid"), + FOREIGN KEY("guid") REFERENCES "episodes"("guid") + ) + """ ) self.data.execute( """CREATE TABLE IF NOT EXISTS "podcasts" ( "url" TEXT NOT NULL UNIQUE, "episodes" TEXT, - PRIMARY KEY("url) + PRIMARY KEY("url") )""" ) + self.data.execute( + """ + CREATE UNIQUE INDEX + IF NOT EXISTS + "downloadGuid" ON "downloads" ( + "guid" + ); + """ + ) + self.data.execute( + """ + CREATE UNIQUE INDEX + IF NOT EXISTS + "episodeGuid" ON "episodes" ( + "guid" + ); + """ + ) + self.data.execute( + """ + CREATE UNIQUE INDEX + IF NOT EXISTS + "podcastUrl" ON "podcasts" ( + "url" + ); + """ + ) def _load_config(self) -> None: """Loads current config""" @@ -188,7 +223,7 @@ class PodWeb: return content if content: for i in content: - content["url"] = podcastparser.normalize_feed_url(content["url"]) + i["url"] = podcastparser.normalize_feed_url(i["url"]) self.servers = content def _create_serverlist(self) -> None: @@ -206,9 +241,55 @@ class PodWeb: f.write(self.DEFAULT_SERVERLIST_HEADING) yaml.dump(self.servers, f) + def _sync_podcasts_table(self) -> None: + """Syncs the podcasts table with self.serverlist""" + server_urls = [] + for i in self.servers: + self.data.execute( + """ + SELECT COUNT(1) + FROM \"podcasts\" + WHERE \"url\" = :URL; + """, + {"URL": i["url"]}, + ) + resp = self.data.fetchone()[0] + if resp == 0: + self.data.execute( + """ + INSERT INTO \"podcasts\" + VALUES ( + :URL, + \"\" + ); + """, + {"URL": i["url"]}, + ) + server_urls.append(i["url"]) + + self.data.execute( + """ + SELECT \"url\" + FROM \"podcasts\"; + """ + ) + + db_server_urls = [i[0] for i in self.data.fetchall()] + for i in db_server_urls: + if i not in server_urls: + self.data.execute( + """ + DELETE FROM \"podcasts\" + WHERE \"url\" = :URL; + """, + {"URL": i}, + ) + self.con.commit() + def add_podcast( self, feedurl: str, name=None, category=None, site=None, img=None ) -> None: + """Adds a new podcast to the serverlist""" feedurl = podcastparser.normalize_feed_url(feedurl) parsed = podcastparser.parse(feedurl, urllib.request.urlopen(feedurl)) if parsed.get("newLocation") != None: @@ -236,7 +317,8 @@ class PodWeb: self.servers.append(new_feed) self._update_serverlist() - def sync_episodes(self, feedurl: str) -> None: + def _sync_episodes(self, feedurl: str, min_size: bool = True) -> None: + """syncs the available episodes for download for the given feedurl""" feedurl = podcastparser.normalize_feed_url(feedurl) parsed = podcastparser.parse(feedurl, urllib.request.urlopen(feedurl)) if parsed.get("newLocation") != None: @@ -244,11 +326,22 @@ class PodWeb: for i in self.servers: if i["url"] == feedurl: i["url"] = new_feedurl - feedurl = new_feedurl self._update_serverlist() + self.data.execute( + """ + UPDATE "podcasts" + SET "url" = :NEWURL + WHERE "url" = :URL + """, + {"URL": feedurl, "NEWURL": new_feedurl}, + ) + feedurl = new_feedurl break + guid_list = [] for i in parsed["episodes"]: - enclosure_list = sorted(i["enclosures"], key=lambda d: d["file_size"]) + enclosure_list = sorted( + i["enclosures"], key=lambda d: d["file_size"], reverse=min_size + ) mime = True j = 0 size = len(enclosure_list) @@ -257,44 +350,75 @@ class PodWeb: if enclosure_list[j]["mime_type"] == "audio/mpeg": mime = False else: - ++j + j += 1 self.data.execute( - """INSERT OR REPLACE INTO "episodes" VALUES ( + """ + INSERT OR REPLACE INTO "episodes" VALUES ( :GUID, + :PODCAST_URL, :TITLE, :DESCRIPTION, :IMG, :URL, :WEBSITE, - :SEASON - );""", + :SEASON, + :DELETED + ); + """, { "GUID": i["guid"], + "PODCAST_URL": feedurl, "TITLE": i["title"], "DESCRIPTION": i["description"], "IMG": i["episode_art_url"], "URL": episode_url, "WEBSITE": i["link"], - "SEASON": i["number"], + "SEASON": i.get("number", -1), + "DELETED": False, }, ) - self.data.execute( - 'UPDATE "podcasts" WHERE "url" = :URL SET "episodes" += :EPSTRING', - {"EPSTRING": f";{i['guid']}"}, - ) self.con.commit() + guid_list.append(i["guid"]) + self.data.execute( + """ + SELECT \"guid\" + FROM \"episodes\"; + """ + ) + db_guid_list = [i[0] for i in self.data.fetchall()] + for i in db_guid_list: + if i not in guid_list: + self.data.execute( + """ + UPDATE \"episodes\" + SET \"deleted\" = 1 + WHERE \"guid\" = :GUID; + """, + {"GUID": i}, + ) def download_episode(self, guid): response = self.data.execute( - 'SELECT "url" FROM "episodes" WHERE guid = :GUID;', {"GUID": guid} + """ + SELECT "url","podcast_url" + FROM "episodes" + WHERE guid = :GUID; + """, + {"GUID": guid}, ) - url = response.fetchone() + url, feedurl = response.fetchone() if url == None: click.echo("No URL found!", err=True) return 1 - url = url[0] - audio_data = urllib.request.urlopen(url) - click.echo(audio_data.read()) + ua = fake_useragent.UserAgent() + request_obj = urllib.request.Request( + url, data=None, headers={"User-Agent": ua.ff} + ) + try: + audio_data = urllib.request.urlopen(request_obj) + except urllib.error.HTTPError: + self._sync_episodes(feedurl) + audio_data = urllib.request.urlopen(request_obj) def import_opml(self, opml_path: str) -> None: body = xmlet.parse(source=opml_path).getroot().find("body") diff --git a/requirements.txt b/requirements.txt index 9142357..c7f4542 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ pytest >= 4.6 pytest-cov coverage ruamel.yaml +fake-useragent |