From 6cf90981bda927832ed0836d24016c5e2b8ff9c7 Mon Sep 17 00:00:00 2001 From: Thilo Kogge Date: Tue, 6 Dec 2022 23:36:41 +0100 Subject: [PATCH 1/4] #80 initial draft --- python/podcast/podcast.py | 35 +------------ python/podcast/podpost.py | 25 +++------ python/podcast/util.py | 73 ++++++++++++++++++++------- test/test_podcast.py | 12 +++-- translations/harbour-podqast-de.ts | 2 +- translations/harbour-podqast-es.ts | 2 +- translations/harbour-podqast-fr.ts | 2 +- translations/harbour-podqast-sv.ts | 2 +- translations/harbour-podqast-zh_CN.ts | 2 +- translations/harbour-podqast.ts | 2 +- 10 files changed, 76 insertions(+), 81 deletions(-) diff --git a/python/podcast/podcast.py b/python/podcast/podcast.py index ae573fd..a7c946a 100644 --- a/python/podcast/podcast.py +++ b/python/podcast/podcast.py @@ -9,7 +9,7 @@ from feedparser import FeedParserDict from peewee import CharField, TextField, BooleanField, IntegerField, ModelSelect, DoesNotExist, FloatField, \ IntegrityError from podcast.persistent_log import persist_log, LogType -from podcast.util import movePost +from podcast.util import movePost, download_image_into_storage sys.path.append("/usr/share/podqast/python") @@ -132,9 +132,7 @@ class Podcast(BaseModel): self.logo_url = feed.feed.image.href logger.debug("podcast new logo_url: %s", self.logo_url) old_path = self.logo_path - if self.logo_path: - self.logo_path = None - self.__download_icon() + self.logo_path = download_image_into_storage(self.logo_url, Constants().iconpath) or None if old_path: util.delete_file(old_path) except: @@ -291,35 +289,6 @@ class Podcast(BaseModel): else: return None - def __download_icon(self): - """ - Download icon - """ - - iconpath = Constants().iconpath - rdot = self.logo_url.rfind(".") - extexcl = self.logo_url.rfind("?") - if extexcl > 0: - ext = self.logo_url[rdot + 1: extexcl] - else: - ext = self.logo_url[rdot + 1:] - if ext == "jpg" or ext == "jpeg" or ext == "png" or ext == "gif": - name = ( - hashlib.sha256(self.logo_url.encode()).hexdigest() + "." + ext - ) - - logo_path = os.path.join(iconpath, name) - try: - util.dl_from_url(self.logo_url, logo_path) - except: - logo_path = "" - logger.info( - "could not load icon: %s", str(sys.exc_info()[0]) - ) - self.logo_path = logo_path - else: - self.logo_path = "" - def refresh(self, moveto, limit=0, full_refresh=False) -> Iterator[Tuple[int, 'Podpost']]: """ Refresh podcast and return list of new entries diff --git a/python/podcast/podpost.py b/python/podcast/podpost.py index cc47b9f..a684e61 100644 --- a/python/podcast/podpost.py +++ b/python/podcast/podpost.py @@ -324,28 +324,15 @@ class Podpost(BaseModel): """ Download icon """ - + if not self.logo_url: + return if self.logo_path: if os.path.exists(self.logo_path): return - - iconpath = Constants().iconpath - rdot = self.logo_url.rfind(".") - ext = self.logo_url[rdot + 1:] - if ext == "jpg" or ext == "jpeg" or ext == "png" or ext == "gif": - name = ( - hashlib.sha256(self.logo_url.encode()).hexdigest() + "." + ext - ) - - logo_path = os.path.join(iconpath, name) - try: - util.dl_from_url(self.logo_url, logo_path) - except: - logo_path = "" - - self.logo_path = logo_path - else: - self.logo_path = "" + try: + self.logo_path = util.download_image_into_storage(self.logo_url, Constants().iconpath) or None + except: + self.logo_path = None def play(self) -> Dict[str,object]: """ diff --git a/python/podcast/util.py b/python/podcast/util.py index da74726..be26d90 100644 --- a/python/podcast/util.py +++ b/python/podcast/util.py @@ -1,23 +1,29 @@ """ some utilities """ +import base64 +import datetime +import hashlib import logging import os -import urllib.request -import urllib.error +import shutil +import subprocess +import sys +import tempfile import time -import datetime +import urllib.error +import urllib.request from collections import deque from enum import unique, IntEnum from itertools import count -from typing import List, Iterator, Tuple +from typing import List, Iterator, Tuple, Optional +from xml.dom import minidom from xml.etree import ElementTree as ET from xml.etree.ElementTree import Element, SubElement, Comment, tostring -from xml.dom import minidom + import pyotherside + import mutagen -import hashlib -import base64 from mutagen.flac import Picture, error as FLACError logger = logging.getLogger(__name__) @@ -114,20 +120,51 @@ def s_to_year(secs): return time.strftime("%Y-%m-%d", time.gmtime(secs)) -def dl_from_url(url, path): +def download_image_into_storage(url: str, iconpath: str) -> Optional[str]: + name = hashlib.sha256(url.encode()).hexdigest() + save_path = os.path.join(iconpath, name) + try: + download_image(url, save_path) + return save_path + except Exception as e: + logger.exception("could not load icon") + return None + + +def download_image(url, path, max_file_size_in_mb=20): """ - Download a file from url and save to path + downloads the image from url to path. checks if it is actually an image + :param url: url of the image + :param path: path to save to + :param max_file_size_in_mb: raises an exception if the file is larger than that + :return: nothing """ + req = urllib.request.Request(url, data=None, headers={"User-Agent": __get_http_agent(url)}) + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + with urllib.request.urlopen(req) as response: + shutil.copyfileobj(response, tmp_file) + + filesize = get_file_length(tmp_file.name) + if filesize > 1024 * 1024 * max_file_size_in_mb: # panic if file is larger than 20 mb + raise ValueError("the downloaded file was had size %d bytes which is larger than %d bytes" % ( + filesize, max_file_size_in_mb * 1024 * 1024)) + __assert_is_image_type(tmp_file.name, url) + shutil.move(tmp_file.name, path) + + +def __assert_is_image_type(path: str, url: str): + result = subprocess.check_output(['file', '--mime-type', path], text=True) + mime_type = result.split(maxsplit=1)[1] + if not mime_type.startswith("image/"): + raise ValueError("expected an image from '%s' on path %s but detected mimetype '%s'" % (url, path, mime_type)) + +def __get_http_agent(url): if url.find("tumblr.com") >= 0: agent = user_agent2 else: agent = user_agent - - rsp = urllib.request.Request(url, data=None, headers={"User-Agent": agent}) - response = urllib.request.urlopen(rsp) - with open(path, "wb") as fhandle: - fhandle.write(response.read()) + return agent def dl_from_url_progress(url, path): @@ -139,10 +176,7 @@ def dl_from_url_progress(url, path): count = 0 p1 = 0 - if url.find("tumblr.com") >= 0: - agent = user_agent2 - else: - agent = user_agent + agent = __get_http_agent(url) req = urllib.request.Request(url, data=None, headers={"User-Agent": agent}) try: @@ -415,6 +449,7 @@ def down_audio_icon(afile, downpath): return filepath + @unique class MovePostToPage(IntEnum): INBOX = 0 @@ -423,7 +458,7 @@ class MovePostToPage(IntEnum): HISTORY = 3 def get_humanized_name(self): - return ["Inbox","Queue","Queue","History"][self] + return ["Inbox", "Queue", "Queue", "History"][self] def movePost(move: int, postid): diff --git a/test/test_podcast.py b/test/test_podcast.py index 9531750..7d9b55b 100644 --- a/test/test_podcast.py +++ b/test/test_podcast.py @@ -21,6 +21,11 @@ from test import xml_headers, read_testdata logger = logging.getLogger(__name__) +def request_callback(request, uri, response_headers): + image = b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00}\x00}\x00\x00\xff\xdb\x00C\x00\x08\x06\x06\x07\x06\x05\x08\x07\x07\x00\xf7\xfa(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80?\xff\xd9' + + return [200, response_headers, image] + @httpretty.activate(allow_net_connect=False) def test_feed_entry(): """ @@ -35,15 +40,14 @@ def test_feed_entry(): url_fs = 'https://freakshow.fm/feed/opus/' HTTPretty.register_uri(HTTPretty.GET, url_fs, body=read_testdata('testdata/freakshow.rss'), adding_headers=xml_headers) - image = b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00}\x00}\x00\x00\xff\xdb\x00C\x00\x08\x06\x06\x07\x06\x05\x08\x07\x07\x00\xf7\xfa(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80?\xff\xd9' HTTPretty.register_uri(HTTPretty.GET, 'https://meta.metaebene.me/media/mm/freakshow-logo-1.0.jpg', - body=image) + body=request_callback) HTTPretty.register_uri(HTTPretty.GET, 'https://elroy.twit.tv/sites/default/files/styles/twit_album_art_144x144/public/images/shows/security_now/album_art/audio/sn_albumart_mask.jpg?itok=VEh3JGKV', - body=image) + body=request_callback) HTTPretty.register_uri(HTTPretty.GET, 'https://elroy.twit.tv/sites/default/files/styles/twit_album_art_2048x2048/public/images/shows/security_now/album_art/audio/sn_albumart_mask.jpg?itok=scC8c-TL', - body=image) + body=request_callback) seen_postids = {} diff --git a/translations/harbour-podqast-de.ts b/translations/harbour-podqast-de.ts index c0e4996..dfc3338 100644 --- a/translations/harbour-podqast-de.ts +++ b/translations/harbour-podqast-de.ts @@ -128,7 +128,7 @@ - <b>Whats New?</b><ul><li>Some of you lost episodes in the last migration. Because we found this pretty sad, we're trying to recover them now.</li><li>If you are missing old episodes of feeds in general and this didn't help, try the force refresh button in the settings (We added support for Pagination).</li><li>Podcasts are now sorted alphabetically, we hope you like that!</li><li>Gpodder should work again.</li><li>Check the new Log Page next to the settings.</li><li>If you're streaming an episode and the download finishes, we're now switching to the downloaded file automatically.</li><li>Also more speed, bugfixes and code cleanups.</li></ul><br>If you want to contribute to podQast, you can help translating the app or report issues on GitLab. + whatsnew text whatsnew section of the migration diff --git a/translations/harbour-podqast-es.ts b/translations/harbour-podqast-es.ts index c80c633..176a293 100644 --- a/translations/harbour-podqast-es.ts +++ b/translations/harbour-podqast-es.ts @@ -128,7 +128,7 @@ - <b>Whats New?</b><ul><li>Some of you lost episodes in the last migration. Because we found this pretty sad, we're trying to recover them now.</li><li>If you are missing old episodes of feeds in general and this didn't help, try the force refresh button in the settings (We added support for Pagination).</li><li>Podcasts are now sorted alphabetically, we hope you like that!</li><li>Gpodder should work again.</li><li>Check the new Log Page next to the settings.</li><li>If you're streaming an episode and the download finishes, we're now switching to the downloaded file automatically.</li><li>Also more speed, bugfixes and code cleanups.</li></ul><br>If you want to contribute to podQast, you can help translating the app or report issues on GitLab. + whatsnew text whatsnew section of the migration diff --git a/translations/harbour-podqast-fr.ts b/translations/harbour-podqast-fr.ts index 0f66804..d19d845 100644 --- a/translations/harbour-podqast-fr.ts +++ b/translations/harbour-podqast-fr.ts @@ -128,7 +128,7 @@ - <b>Whats New?</b><ul><li>Some of you lost episodes in the last migration. Because we found this pretty sad, we're trying to recover them now.</li><li>If you are missing old episodes of feeds in general and this didn't help, try the force refresh button in the settings (We added support for Pagination).</li><li>Podcasts are now sorted alphabetically, we hope you like that!</li><li>Gpodder should work again.</li><li>Check the new Log Page next to the settings.</li><li>If you're streaming an episode and the download finishes, we're now switching to the downloaded file automatically.</li><li>Also more speed, bugfixes and code cleanups.</li></ul><br>If you want to contribute to podQast, you can help translating the app or report issues on GitLab. + whatsnew text whatsnew section of the migration diff --git a/translations/harbour-podqast-sv.ts b/translations/harbour-podqast-sv.ts index 7b652c3..6569337 100644 --- a/translations/harbour-podqast-sv.ts +++ b/translations/harbour-podqast-sv.ts @@ -128,7 +128,7 @@ - <b>Whats New?</b><ul><li>Some of you lost episodes in the last migration. Because we found this pretty sad, we're trying to recover them now.</li><li>If you are missing old episodes of feeds in general and this didn't help, try the force refresh button in the settings (We added support for Pagination).</li><li>Podcasts are now sorted alphabetically, we hope you like that!</li><li>Gpodder should work again.</li><li>Check the new Log Page next to the settings.</li><li>If you're streaming an episode and the download finishes, we're now switching to the downloaded file automatically.</li><li>Also more speed, bugfixes and code cleanups.</li></ul><br>If you want to contribute to podQast, you can help translating the app or report issues on GitLab. + whatsnew text whatsnew section of the migration diff --git a/translations/harbour-podqast-zh_CN.ts b/translations/harbour-podqast-zh_CN.ts index 5b494c2..64e1404 100644 --- a/translations/harbour-podqast-zh_CN.ts +++ b/translations/harbour-podqast-zh_CN.ts @@ -128,7 +128,7 @@ - <b>Whats New?</b><ul><li>Some of you lost episodes in the last migration. Because we found this pretty sad, we're trying to recover them now.</li><li>If you are missing old episodes of feeds in general and this didn't help, try the force refresh button in the settings (We added support for Pagination).</li><li>Podcasts are now sorted alphabetically, we hope you like that!</li><li>Gpodder should work again.</li><li>Check the new Log Page next to the settings.</li><li>If you're streaming an episode and the download finishes, we're now switching to the downloaded file automatically.</li><li>Also more speed, bugfixes and code cleanups.</li></ul><br>If you want to contribute to podQast, you can help translating the app or report issues on GitLab. + whatsnew text whatsnew section of the migration diff --git a/translations/harbour-podqast.ts b/translations/harbour-podqast.ts index 90be444..125c00f 100644 --- a/translations/harbour-podqast.ts +++ b/translations/harbour-podqast.ts @@ -128,7 +128,7 @@ - <b>Whats New?</b><ul><li>Some of you lost episodes in the last migration. Because we found this pretty sad, we're trying to recover them now.</li><li>If you are missing old episodes of feeds in general and this didn't help, try the force refresh button in the settings (We added support for Pagination).</li><li>Podcasts are now sorted alphabetically, we hope you like that!</li><li>Gpodder should work again.</li><li>Check the new Log Page next to the settings.</li><li>If you're streaming an episode and the download finishes, we're now switching to the downloaded file automatically.</li><li>Also more speed, bugfixes and code cleanups.</li></ul><br>If you want to contribute to podQast, you can help translating the app or report issues on GitLab. + whatsnew text whatsnew section of the migration -- GitLab From 8e396c4031281dabbcb02928bab8a9f57c2ba0e6 Mon Sep 17 00:00:00 2001 From: Thilo Kogge Date: Wed, 21 Dec 2022 23:15:03 +0100 Subject: [PATCH 2/4] #80 flushing tempfile beforedetecting mime type --- python/podcast/podcast.py | 1 - python/podcast/util.py | 4 ++-- test/test_podcast.py | 10 +++++----- test/testdata/empty_image.png | Bin 0 -> 153 bytes 4 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 test/testdata/empty_image.png diff --git a/python/podcast/podcast.py b/python/podcast/podcast.py index a7c946a..f208d0a 100644 --- a/python/podcast/podcast.py +++ b/python/podcast/podcast.py @@ -28,7 +28,6 @@ from podcast import util, feedutils from email.utils import mktime_tz, parsedate_tz from calendar import timegm -import hashlib import os import logging import pyotherside diff --git a/python/podcast/util.py b/python/podcast/util.py index be26d90..3480834 100644 --- a/python/podcast/util.py +++ b/python/podcast/util.py @@ -8,7 +8,6 @@ import logging import os import shutil import subprocess -import sys import tempfile import time import urllib.error @@ -126,7 +125,7 @@ def download_image_into_storage(url: str, iconpath: str) -> Optional[str]: try: download_image(url, save_path) return save_path - except Exception as e: + except Exception: logger.exception("could not load icon") return None @@ -143,6 +142,7 @@ def download_image(url, path, max_file_size_in_mb=20): with tempfile.NamedTemporaryFile(delete=False) as tmp_file: with urllib.request.urlopen(req) as response: shutil.copyfileobj(response, tmp_file) + tmp_file.flush() filesize = get_file_length(tmp_file.name) if filesize > 1024 * 1024 * max_file_size_in_mb: # panic if file is larger than 20 mb diff --git a/test/test_podcast.py b/test/test_podcast.py index 7d9b55b..b581dfb 100644 --- a/test/test_podcast.py +++ b/test/test_podcast.py @@ -2,6 +2,8 @@ Test Podcast functions """ import logging +import os +from io import StringIO from typing import Tuple, List import httpretty @@ -22,9 +24,9 @@ logger = logging.getLogger(__name__) def request_callback(request, uri, response_headers): - image = b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00}\x00}\x00\x00\xff\xdb\x00C\x00\x08\x06\x06\x07\x06\x05\x08\x07\x07\x00\xf7\xfa(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80\n(\xa2\x80?\xff\xd9' + with open(os.path.join(os.path.dirname(__file__), "testdata/empty_image.png"), "rb") as file: + return [200, response_headers, file.read()] - return [200, response_headers, image] @httpretty.activate(allow_net_connect=False) def test_feed_entry(): @@ -211,6 +213,7 @@ def test_podcastlist_refresh(refreshable_podcast_fixture): assert plist.get_podcast_count() == 1 assert list(plist.refresh(0)) == [(2, p.title, 'Hello, I am a new fake episode for testing!!öäü', 0)] + @pytest.mark.parametrize("refreshable_podcast_fixture", ["testdata/fakefeed"], indirect=True) def test_move(refreshable_podcast_fixture): p, episodes = refreshable_podcast_fixture @@ -226,7 +229,6 @@ def test_move(refreshable_podcast_fixture): assert len(QueueData.get_queue()) == 1 - @pytest.fixture def refreshable_podcast_fixture(request) -> Tuple[Podcast, List[Podpost]]: """ @@ -251,8 +253,6 @@ def refreshable_podcast_fixture(request) -> Tuple[Podcast, List[Podpost]]: assert invoked > 0 - - @httpretty.activate(allow_net_connect=False) def test_pagination_stops(): url_f1 = "https://fakefeed.com/" diff --git a/test/testdata/empty_image.png b/test/testdata/empty_image.png new file mode 100644 index 0000000000000000000000000000000000000000..f11f13440cc18625138ff5ddb4a5d9937ac56f64 GIT binary patch literal 153 zcmeAS@N?(olHy`uVBq!ia0y~yV9;k^V9@1YV_;ygdvCOyfq{Xuz$3DlfkD_3gcu+V0d8vbHyfi=c3RHw#J+7N=qKNRxaHW6sNg1T; literal 0 HcmV?d00001 -- GitLab From 8ea9c271a5223b08a27838a7979a446b73f7b623 Mon Sep 17 00:00:00 2001 From: Thilo Kogge Date: Fri, 30 Dec 2022 21:45:39 +0100 Subject: [PATCH 3/4] #80 disabled mime type check --- python/podcast/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/podcast/util.py b/python/podcast/util.py index 3480834..c850a59 100644 --- a/python/podcast/util.py +++ b/python/podcast/util.py @@ -153,6 +153,7 @@ def download_image(url, path, max_file_size_in_mb=20): def __assert_is_image_type(path: str, url: str): + return True # file util is not allowed with sailjail unfortunatley... result = subprocess.check_output(['file', '--mime-type', path], text=True) mime_type = result.split(maxsplit=1)[1] if not mime_type.startswith("image/"): -- GitLab From 83f522d2a5e7692755a912bc1d5b8929967576cf Mon Sep 17 00:00:00 2001 From: Thilo Kogge Date: Sat, 24 Jun 2023 12:07:48 +0200 Subject: [PATCH 4/4] fixed hints --- qml/components/EpisodeImage.qml | 3 ++- qml/components/hints/HintEpisodeImage.qml | 5 ++--- qml/components/hints/PlayerHint.qml | 4 ++-- qml/lib/hints/SlideShowHint.qml | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/qml/components/EpisodeImage.qml b/qml/components/EpisodeImage.qml index f24ce90..591de93 100644 --- a/qml/components/EpisodeImage.qml +++ b/qml/components/EpisodeImage.qml @@ -52,7 +52,8 @@ Thumbnail { height: Theme.iconSizeLarge sourceSize.width: Theme.iconSizeLarge sourceSize.height: Theme.iconSizeLarge - source: model.podcast_logo === "" ? "../../images/podcast.png" : model.podcast_logo + source: (model.podcast_logo === "" || model === undefined + || model.podcast_logo === undefined) ? "../../images/podcast.png" : model.podcast_logo Rectangle { id: dlstatus visible: showDownladingState diff --git a/qml/components/hints/HintEpisodeImage.qml b/qml/components/hints/HintEpisodeImage.qml index 496c782..9dfd643 100644 --- a/qml/components/hints/HintEpisodeImage.qml +++ b/qml/components/hints/HintEpisodeImage.qml @@ -11,7 +11,6 @@ EpisodeImage { showIsListened: relativeIndex === 2 percentage: relativeIndex === 0 ? 100 : 0 anchors.fill: centeredComponent - property string logo_url: "../../images/podcast.png" - width: Theme.iconSizeExtraLarge - height: Theme.iconSizeExtraLarge + width: Theme.itemSizeHuge + height: Theme.itemSizeHuge } diff --git a/qml/components/hints/PlayerHint.qml b/qml/components/hints/PlayerHint.qml index d500d4f..78e8539 100644 --- a/qml/components/hints/PlayerHint.qml +++ b/qml/components/hints/PlayerHint.qml @@ -8,12 +8,12 @@ SlideShowHint { "text": qsTr("Select if the next item of the playlist should be played automatically"), "direction": TouchInteraction.Down, "interactionMode": TouchInteraction.EdgeSwipe, - "loops": 3 + "loops": 0 }, { "text": qsTr("Click the image to adjust the playrate"), "direction": 0, "interactionMode": 0, "interactionHidden": true, - "loops": 3 + "loops": 0 }] } diff --git a/qml/lib/hints/SlideShowHint.qml b/qml/lib/hints/SlideShowHint.qml index c121957..ce48eff 100644 --- a/qml/lib/hints/SlideShowHint.qml +++ b/qml/lib/hints/SlideShowHint.qml @@ -27,7 +27,8 @@ Item { width: Theme.iconSizeExtraLarge height: Theme.iconSizeExtraLarge property int index: slideNr - source: !slideloaded ? "" : slideconfig.centeredComponent + source: (!slideloaded || slideconfig.centeredComponent + === undefined) ? "" : slideconfig.centeredComponent visible: !slideloaded ? false : slideconfig.centeredComponent !== undefined active: !slideloaded ? false : slideconfig.centeredComponent !== undefined onActiveChanged: console.info("loading component " + source) -- GitLab