From 22c05baf2c4b70a658305c43ac64ab0d8bcc558f Mon Sep 17 00:00:00 2001 From: Zviger Date: Sun, 10 Nov 2019 16:57:10 +0300 Subject: [PATCH 01/21] The first version of the application is added. --- .gitignore | 1 + args_parser.py | 10 ++++++++++ format_converter.py | 27 +++++++++++++++++++++++++++ frbz.py | 19 +++++++++++++++++++ rss_parser.py | 41 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 98 insertions(+) create mode 100644 args_parser.py create mode 100644 format_converter.py create mode 100644 frbz.py create mode 100644 rss_parser.py diff --git a/.gitignore b/.gitignore index 894a44c..af7597f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ wheels/ *.egg-info/ .installed.cfg *.egg +.idea MANIFEST # PyInstaller diff --git a/args_parser.py b/args_parser.py new file mode 100644 index 0000000..fe13dac --- /dev/null +++ b/args_parser.py @@ -0,0 +1,10 @@ +import argparse + + +def get_args(): + parser = argparse.ArgumentParser(description="frbz(free reader by Zviger) - python command-line rss reader") + parser.add_argument("source", help="RSS URL") + parser.add_argument('--version', action='version', version='%(prog)s 0.1') + parser.parse_args() + args = parser.parse_args() + return args diff --git a/format_converter.py b/format_converter.py new file mode 100644 index 0000000..2746d32 --- /dev/null +++ b/format_converter.py @@ -0,0 +1,27 @@ +import textwrap +import functools + + +class Converter: + def __init__(self, items): + self.__items = items + + def to_console_format(self): + str_len = 80 + strings = [] + for item in self.__items: + strings.append("-" * str_len) + strings.append(f"Feed: {item.get('feed_title')}") + strings.append(f"Author: {item.get('item_author')}") + strings.append(f"Date: {item.get('item_date')}") + strings.append("\n") + strings.append(f"Title: {item.get('item_title')}") + strings.append(f"Description: {item.get('item_description')}") + strings.append("-" * str_len) + strings.append("\n") + strings = map(lambda s: textwrap.fill(s, width=str_len) + "\n", strings) + + result_string = "-" * str_len + "\n" + result_string += functools.reduce(lambda a, b: a + b, strings) + + return result_string diff --git a/frbz.py b/frbz.py new file mode 100644 index 0000000..4a4730c --- /dev/null +++ b/frbz.py @@ -0,0 +1,19 @@ +import args_parser +import rss_parser +import format_converter + + +class Reader: + + @staticmethod + def exec_console_args(): + _args = args_parser.get_args() + parser = rss_parser.Parser(_args.source) + items = parser.parse_feed() + converter = format_converter.Converter(items) + print(converter.to_console_format()) + + +if __name__ == "__main__": + reader = Reader() + reader.exec_console_args() diff --git a/rss_parser.py b/rss_parser.py new file mode 100644 index 0000000..53b3eff --- /dev/null +++ b/rss_parser.py @@ -0,0 +1,41 @@ +import feedparser + +FEED_FIELD_MAPPING = {"title": "feed_title", + "link": "feed_link"} + +ITEM_FIELD_MAPPING = {"title": "item_title", + "link": "item_link", + "author": "item_author", + "description": "item_description", + "published": "item_date"} + + +class Parser: + + def __init__(self, url): + self.url = url + + def parse_feed(self, items=-1): + d = feedparser.parse(self.url) + feed = d.get("feed", default={}) + feed_data = Parser.__apply_field_mapping(FEED_FIELD_MAPPING, feed) + result_items = [] + for item in d.get("entries")[:items]: + item_data = Parser.__apply_field_mapping(ITEM_FIELD_MAPPING, item) + result_item = {} + result_item.update(feed_data) + result_item.update(item_data) + result_items.append(result_item) + return result_items + + @staticmethod + def __apply_field_mapping(field_mapping, source): + data = {} + for key in field_mapping: + data[field_mapping[key]] = source.get(key) + return data + + +if __name__ == "__main__": + parser = Parser("https://news.tut.by/rss/economics.rss") + print(parser.parse_feed(2)) From d503f4244b46ae37220a71ba5dc371efb501f75f Mon Sep 17 00:00:00 2001 From: Zviger Date: Tue, 12 Nov 2019 13:50:23 +0300 Subject: [PATCH 02/21] Readme file is added --- README.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.txt diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..0e2c410 --- /dev/null +++ b/README.txt @@ -0,0 +1 @@ +It is a one-shot command-line RSS reader by Zviger. From c591b63ebc885b4e1ebeb5645293bc8c15a11b8b Mon Sep 17 00:00:00 2001 From: Zviger Date: Tue, 12 Nov 2019 13:53:59 +0300 Subject: [PATCH 03/21] Some fetures are added The structure of the data received by the RSS parser has been redone. Improved parser mechanism. Added parameters "limit" and "length". Added some validation. --- args_parser.py | 11 ++++++++++- format_converter.py | 34 ++++++++++++++++++++-------------- frbz.py | 17 ++++++++++++++--- rss_parser.py | 29 ++++++++++++++++++----------- 4 files changed, 62 insertions(+), 29 deletions(-) diff --git a/args_parser.py b/args_parser.py index fe13dac..7d06fbb 100644 --- a/args_parser.py +++ b/args_parser.py @@ -1,10 +1,19 @@ +""" +This module is a parser of console arguments for this project. +""" import argparse def get_args(): + """ + Function, that parse console args. + Return an object that provides the values ​​of parsed arguments. + """ parser = argparse.ArgumentParser(description="frbz(free reader by Zviger) - python command-line rss reader") parser.add_argument("source", help="RSS URL") - parser.add_argument('--version', action='version', version='%(prog)s 0.1') + parser.add_argument("--version", action="version", version="%(prog)s 0.1", help="Print version info") + parser.add_argument("-l", "--limit", type=int, help="Limit news topics if this parameter provided", default=-1) + parser.add_argument("--length", type=int, help="Sets the length of each line of news output", default=120) parser.parse_args() args = parser.parse_args() return args diff --git a/format_converter.py b/format_converter.py index 2746d32..0c80f1c 100644 --- a/format_converter.py +++ b/format_converter.py @@ -1,24 +1,30 @@ import textwrap import functools +import json class Converter: - def __init__(self, items): - self.__items = items + def __init__(self, feeds): + self.__feeds = feeds - def to_console_format(self): - str_len = 80 + def to_console_format(self, str_len=80): strings = [] - for item in self.__items: - strings.append("-" * str_len) - strings.append(f"Feed: {item.get('feed_title')}") - strings.append(f"Author: {item.get('item_author')}") - strings.append(f"Date: {item.get('item_date')}") - strings.append("\n") - strings.append(f"Title: {item.get('item_title')}") - strings.append(f"Description: {item.get('item_description')}") - strings.append("-" * str_len) - strings.append("\n") + for feed in self.__feeds: + strings.append(f"Feed: {feed.get('feed_title')}") + for item in feed.get("items", []): + strings.append("-" * str_len) + strings.append(f"Author: {item.get('item_author')}") + strings.append(f"Date: {item.get('item_date')}") + strings.append("\n") + strings.append(f"Title: {item.get('item_title')}") + strings.append(f"Description: {item.get('item_description')}") + strings.append("\n") + strings.append(f"Link: {item.get('item_link')}") + strings.append(f"Image link: {item.get('item_img_link')}") + strings.append("-" * str_len) + strings.append("\n") + strings.pop() + strings = map(lambda s: textwrap.fill(s, width=str_len) + "\n", strings) result_string = "-" * str_len + "\n" diff --git a/frbz.py b/frbz.py index 4a4730c..b269cd3 100644 --- a/frbz.py +++ b/frbz.py @@ -9,9 +9,20 @@ class Reader: def exec_console_args(): _args = args_parser.get_args() parser = rss_parser.Parser(_args.source) - items = parser.parse_feed() - converter = format_converter.Converter(items) - print(converter.to_console_format()) + limit = _args.limit + len_each_line = _args.length + if len_each_line < 60: + print("The length must be greater than 60") + return + if limit < 1: + print("The limit must be greater than 0") + return + feeds = [parser.parse_feed(limit)] + if feeds[0] is None: + print("Invalid url.") + return + converter = format_converter.Converter(feeds) + print(converter.to_console_format(str_len=len_each_line)) if __name__ == "__main__": diff --git a/rss_parser.py b/rss_parser.py index 53b3eff..201ebe8 100644 --- a/rss_parser.py +++ b/rss_parser.py @@ -1,4 +1,5 @@ import feedparser +from bs4 import BeautifulSoup FEED_FIELD_MAPPING = {"title": "feed_title", "link": "feed_link"} @@ -15,18 +16,24 @@ class Parser: def __init__(self, url): self.url = url - def parse_feed(self, items=-1): + def parse_feed(self, items_count=-1): d = feedparser.parse(self.url) - feed = d.get("feed", default={}) - feed_data = Parser.__apply_field_mapping(FEED_FIELD_MAPPING, feed) - result_items = [] - for item in d.get("entries")[:items]: - item_data = Parser.__apply_field_mapping(ITEM_FIELD_MAPPING, item) - result_item = {} - result_item.update(feed_data) - result_item.update(item_data) - result_items.append(result_item) - return result_items + if d.status != 200: + return None + feed = d.get("feed", {}) + result_data = Parser.__apply_field_mapping(FEED_FIELD_MAPPING, feed) + items = [Parser.__apply_field_mapping(ITEM_FIELD_MAPPING, item) + for item in d.get("entries", [])[:items_count]] + for item in items: + soup = BeautifulSoup(item["item_description"], 'html.parser') + item_img_link = soup.find("img").get("src") + if not item_img_link: + item_img_link = None + item["item_img_link"] = item_img_link + item["item_description"] = soup.text + + result_data["items"] = items + return result_data @staticmethod def __apply_field_mapping(field_mapping, source): From b3215e521d452ad162cca5233856a09d9385a670 Mon Sep 17 00:00:00 2001 From: Zviger Date: Tue, 12 Nov 2019 14:57:24 +0300 Subject: [PATCH 04/21] Added conversion to json format and reworked getting img links --- args_parser.py | 1 + format_converter.py | 7 ++++++- frbz.py | 20 +++++++++++++------- rss_parser.py | 7 ++----- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/args_parser.py b/args_parser.py index 7d06fbb..882ac7e 100644 --- a/args_parser.py +++ b/args_parser.py @@ -13,6 +13,7 @@ def get_args(): parser.add_argument("source", help="RSS URL") parser.add_argument("--version", action="version", version="%(prog)s 0.1", help="Print version info") parser.add_argument("-l", "--limit", type=int, help="Limit news topics if this parameter provided", default=-1) + parser.add_argument("--json", action="store_true", help="Print result as JSON in stdout", default=False) parser.add_argument("--length", type=int, help="Sets the length of each line of news output", default=120) parser.parse_args() args = parser.parse_args() diff --git a/format_converter.py b/format_converter.py index 0c80f1c..9becf11 100644 --- a/format_converter.py +++ b/format_converter.py @@ -20,7 +20,9 @@ def to_console_format(self, str_len=80): strings.append(f"Description: {item.get('item_description')}") strings.append("\n") strings.append(f"Link: {item.get('item_link')}") - strings.append(f"Image link: {item.get('item_img_link')}") + strings.append("Image links:") + for img_link in item.get('item_img_links'): + strings.append(f"{img_link}") strings.append("-" * str_len) strings.append("\n") strings.pop() @@ -31,3 +33,6 @@ def to_console_format(self, str_len=80): result_string += functools.reduce(lambda a, b: a + b, strings) return result_string + + def to_json_format(self, str_len=80): + return textwrap.fill(json.dumps(self.__feeds, ensure_ascii=False), width=str_len) diff --git a/frbz.py b/frbz.py index b269cd3..df3db66 100644 --- a/frbz.py +++ b/frbz.py @@ -10,19 +10,25 @@ def exec_console_args(): _args = args_parser.get_args() parser = rss_parser.Parser(_args.source) limit = _args.limit - len_each_line = _args.length - if len_each_line < 60: - print("The length must be greater than 60") - return - if limit < 1: - print("The limit must be greater than 0") + to_json = _args.json + + if limit < 1 and limit != -1: + print("The limit must be -1 or greater than 0") return feeds = [parser.parse_feed(limit)] if feeds[0] is None: print("Invalid url.") return + + len_each_line = _args.length + if len_each_line < 60: + print("The length must be greater than 60") + return converter = format_converter.Converter(feeds) - print(converter.to_console_format(str_len=len_each_line)) + if to_json: + print(converter.to_json_format(str_len=len_each_line)) + else: + print(converter.to_console_format(str_len=len_each_line)) if __name__ == "__main__": diff --git a/rss_parser.py b/rss_parser.py index 201ebe8..43d67b4 100644 --- a/rss_parser.py +++ b/rss_parser.py @@ -18,7 +18,7 @@ def __init__(self, url): def parse_feed(self, items_count=-1): d = feedparser.parse(self.url) - if d.status != 200: + if d.bozo != 0 or d.status != 200: return None feed = d.get("feed", {}) result_data = Parser.__apply_field_mapping(FEED_FIELD_MAPPING, feed) @@ -26,10 +26,7 @@ def parse_feed(self, items_count=-1): for item in d.get("entries", [])[:items_count]] for item in items: soup = BeautifulSoup(item["item_description"], 'html.parser') - item_img_link = soup.find("img").get("src") - if not item_img_link: - item_img_link = None - item["item_img_link"] = item_img_link + item["item_img_links"] = [link.get("src") for link in soup.find_all("img") if link.get("src")] item["item_description"] = soup.text result_data["items"] = items From 446dab7f76be558556a212ca1cb236cc0974fba6 Mon Sep 17 00:00:00 2001 From: Zviger Date: Tue, 12 Nov 2019 19:26:26 +0300 Subject: [PATCH 05/21] Some fetures are added Added parametr "verbose" and some docstrings. --- README.md | 40 +++++++++++++++++++ README.txt | 1 - frbz.py | 28 ++++++++++--- support_files/__init__.py | 0 .../args_parser.py | 5 ++- .../format_converter.py | 19 +++++++++ rss_parser.py => support_files/rss_parser.py | 25 +++++++++--- 7 files changed, 104 insertions(+), 14 deletions(-) create mode 100644 README.md delete mode 100644 README.txt create mode 100644 support_files/__init__.py rename args_parser.py => support_files/args_parser.py (70%) rename format_converter.py => support_files/format_converter.py (73%) rename rss_parser.py => support_files/rss_parser.py (69%) diff --git a/README.md b/README.md new file mode 100644 index 0000000..4d238c2 --- /dev/null +++ b/README.md @@ -0,0 +1,40 @@ +##It is a one-shot command-line RSS reader by Zviger. +###User interface +```text +usage: frbz.py [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] source + +positional arguments: + source RSS URL + +optional arguments: + -h, --help show this help message and exit + --version Print version info + -l LIMIT, --limit LIMIT + Limit news topics if this parameter provided + --verbose Print result as JSON in stdout + --json Outputs verbose status messages + --length LENGTH Sets the length of each line of news output +``` + +###Json structure +```json +[ + { + "feed_title": "Yahoo News - Latest News & Headlines", + "feed_link": "https://www.yahoo.com/news", + "items": + [ + { + "item_title": "Top House Armed Services Republican: Trump's Ukraine call was 'inappropriate' but not impeachable", + "item_link": "https://news.yahoo.com/trump-impeachment-mac-thornberry-abc-this-week-165743982.html", + "item_author": null, "item_description": "Rep. Mac Thornberry, R-Texas, said President Trump's call with Ukraine'spresident was \"inappropriate\" — but it did not warrant his impeachment.", + "item_date": "Sun, 10 Nov 2019 11:57:43-0500", "item_img_links": + [ + "http://l1.yimg.com/uu/api/res/1.2/nZ9ESccFgs8cyvX3b2LOUA--/YXBwaWQ9eXRhY2h5b247aD04Njt3PTEzMDs-/https://media-mbst-pub-ue1.s3.amazonaws.com/creatr-uploaded-images/2019-11/922d39a0-03db-11ea-bf1e-9fbc638c65a1" + ] + } + ] + } +] +``` + diff --git a/README.txt b/README.txt deleted file mode 100644 index 0e2c410..0000000 --- a/README.txt +++ /dev/null @@ -1 +0,0 @@ -It is a one-shot command-line RSS reader by Zviger. diff --git a/frbz.py b/frbz.py index df3db66..73f42c5 100644 --- a/frbz.py +++ b/frbz.py @@ -1,24 +1,40 @@ -import args_parser -import rss_parser -import format_converter +import logging +import sys +from support_files import args_parser, format_converter, rss_parser class Reader: @staticmethod def exec_console_args(): + logger = logging.getLogger("console_app") + logger.setLevel(logging.INFO) + # create the logging file handler + fh = logging.StreamHandler(sys.stdout) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(message)s') + fh.setFormatter(formatter) + # add handler to logger object + logger.addHandler(fh) + _args = args_parser.get_args() + logger.disabled = not _args.verbose + logger.info("Program started") + + logger.info(f"Parsing {_args.source} started") parser = rss_parser.Parser(_args.source) + limit = _args.limit to_json = _args.json if limit < 1 and limit != -1: print("The limit must be -1 or greater than 0") return + feeds = [parser.parse_feed(limit)] if feeds[0] is None: print("Invalid url.") return + logger.info(f"Parsing {_args.source} finished") len_each_line = _args.length if len_each_line < 60: @@ -26,11 +42,13 @@ def exec_console_args(): return converter = format_converter.Converter(feeds) if to_json: + logger.info("Data is converted to json format and printing is started") print(converter.to_json_format(str_len=len_each_line)) else: + logger.info("Data is converted to console format and printing is started") print(converter.to_console_format(str_len=len_each_line)) + logger.info("Printing is stoped") if __name__ == "__main__": - reader = Reader() - reader.exec_console_args() + Reader.exec_console_args() diff --git a/support_files/__init__.py b/support_files/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/args_parser.py b/support_files/args_parser.py similarity index 70% rename from args_parser.py rename to support_files/args_parser.py index 882ac7e..23e8874 100644 --- a/args_parser.py +++ b/support_files/args_parser.py @@ -7,13 +7,14 @@ def get_args(): """ Function, that parse console args. - Return an object that provides the values ​​of parsed arguments. + :return: An object that provides the values ​​of parsed arguments. """ parser = argparse.ArgumentParser(description="frbz(free reader by Zviger) - python command-line rss reader") parser.add_argument("source", help="RSS URL") parser.add_argument("--version", action="version", version="%(prog)s 0.1", help="Print version info") parser.add_argument("-l", "--limit", type=int, help="Limit news topics if this parameter provided", default=-1) - parser.add_argument("--json", action="store_true", help="Print result as JSON in stdout", default=False) + parser.add_argument("--verbose", action="store_true", help="Print result as JSON in stdout", default=False) + parser.add_argument("--json", action="store_true", help="Outputs verbose status messages", default=False) parser.add_argument("--length", type=int, help="Sets the length of each line of news output", default=120) parser.parse_args() args = parser.parse_args() diff --git a/format_converter.py b/support_files/format_converter.py similarity index 73% rename from format_converter.py rename to support_files/format_converter.py index 9becf11..e1afd5c 100644 --- a/format_converter.py +++ b/support_files/format_converter.py @@ -1,13 +1,27 @@ +""" +This module contains class for converting parsed data from RSS. +""" import textwrap import functools import json class Converter: + """ + This class represents format converter for parsed data from RSS. + """ def __init__(self, feeds): + """ + :param feeds: Parsed data from RSS. + """ self.__feeds = feeds def to_console_format(self, str_len=80): + """ + Convert data to console format. + :param str_len: Length of output strings. + :return: Converted data. + """ strings = [] for feed in self.__feeds: strings.append(f"Feed: {feed.get('feed_title')}") @@ -35,4 +49,9 @@ def to_console_format(self, str_len=80): return result_string def to_json_format(self, str_len=80): + """ + Convert data to json format. + :param str_len: Length of output strings. + :return: Converted data. + """ return textwrap.fill(json.dumps(self.__feeds, ensure_ascii=False), width=str_len) diff --git a/rss_parser.py b/support_files/rss_parser.py similarity index 69% rename from rss_parser.py rename to support_files/rss_parser.py index 43d67b4..e0b0e7e 100644 --- a/rss_parser.py +++ b/support_files/rss_parser.py @@ -1,3 +1,6 @@ +""" +This module contains class for parsing RSS. +""" import feedparser from bs4 import BeautifulSoup @@ -12,18 +15,28 @@ class Parser: - + """ + This class provides methods to parse RSS. + """ def __init__(self, url): + """ + :param url: Url of RSS. + """ self.url = url - def parse_feed(self, items_count=-1): - d = feedparser.parse(self.url) - if d.bozo != 0 or d.status != 200: + def parse_feed(self, items_limit=-1): + """ + Parse the RSS file. + :param items_limit: Limit count of returned items + :return: Dict with parsed data. + """ + data = feedparser.parse(self.url) + if data.bozo != 0 or data.status != 200: return None - feed = d.get("feed", {}) + feed = data.get("feed", {}) result_data = Parser.__apply_field_mapping(FEED_FIELD_MAPPING, feed) items = [Parser.__apply_field_mapping(ITEM_FIELD_MAPPING, item) - for item in d.get("entries", [])[:items_count]] + for item in data.get("entries", [])[:items_limit]] for item in items: soup = BeautifulSoup(item["item_description"], 'html.parser') item["item_img_links"] = [link.get("src") for link in soup.find_all("img") if link.get("src")] From 028983d3bcaaf22d2d6422dfd4c16b08ba583997 Mon Sep 17 00:00:00 2001 From: Zviger Date: Thu, 14 Nov 2019 02:09:37 +0300 Subject: [PATCH 06/21] Version of the app is changed --- support_files/args_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support_files/args_parser.py b/support_files/args_parser.py index 23e8874..2e981c6 100644 --- a/support_files/args_parser.py +++ b/support_files/args_parser.py @@ -11,7 +11,7 @@ def get_args(): """ parser = argparse.ArgumentParser(description="frbz(free reader by Zviger) - python command-line rss reader") parser.add_argument("source", help="RSS URL") - parser.add_argument("--version", action="version", version="%(prog)s 0.1", help="Print version info") + parser.add_argument("--version", action="version", version="%(prog)s 1.0", help="Print version info") parser.add_argument("-l", "--limit", type=int, help="Limit news topics if this parameter provided", default=-1) parser.add_argument("--verbose", action="store_true", help="Print result as JSON in stdout", default=False) parser.add_argument("--json", action="store_true", help="Outputs verbose status messages", default=False) From cb17e243102ee295b556478e16b35ba8cfdb7492 Mon Sep 17 00:00:00 2001 From: Zviger Date: Thu, 14 Nov 2019 02:29:12 +0300 Subject: [PATCH 07/21] Readme file is fixed --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4d238c2..ed05c82 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -##It is a one-shot command-line RSS reader by Zviger. -###User interface +## It is a one-shot command-line RSS reader by Zviger. +### User interface ```text usage: frbz.py [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] source @@ -16,7 +16,7 @@ optional arguments: --length LENGTH Sets the length of each line of news output ``` -###Json structure +### Json structure ```json [ { From 534b5476c6b351b27bd81177b7f409403910dce2 Mon Sep 17 00:00:00 2001 From: Zviger Date: Thu, 14 Nov 2019 04:42:52 +0300 Subject: [PATCH 08/21] New version of the app Project structure is reworked. Setup tool is added. --- README.md | 4 ++++ app/__init__.py | 1 + app/core.py | 5 +++++ .../support_files}/__init__.py | 0 .../support_files}/args_parser.py | 5 +++-- .../support_files}/format_converter.py | 0 .../support_files}/rss_parser.py | 0 frbz.py => app/support_files/rss_reader.py | 2 +- requirements.txt | 2 ++ setup.py | 21 +++++++++++++++++++ 10 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 app/__init__.py create mode 100644 app/core.py rename {support_files => app/support_files}/__init__.py (100%) rename {support_files => app/support_files}/args_parser.py (78%) rename {support_files => app/support_files}/format_converter.py (100%) rename {support_files => app/support_files}/rss_parser.py (100%) rename frbz.py => app/support_files/rss_reader.py (96%) create mode 100644 requirements.txt create mode 100644 setup.py diff --git a/README.md b/README.md index ed05c82..f65b1f8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,8 @@ ## It is a one-shot command-line RSS reader by Zviger. +### Installation +```text +Clone this repository and run setup.py file with parameters "install --user" +``` ### User interface ```text usage: frbz.py [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] source diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..f2dc0e4 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +__version__ = "2.0" diff --git a/app/core.py b/app/core.py new file mode 100644 index 0000000..5856fd4 --- /dev/null +++ b/app/core.py @@ -0,0 +1,5 @@ +from app.support_files.rss_reader import Reader + + +def main(): + Reader.exec_console_args() diff --git a/support_files/__init__.py b/app/support_files/__init__.py similarity index 100% rename from support_files/__init__.py rename to app/support_files/__init__.py diff --git a/support_files/args_parser.py b/app/support_files/args_parser.py similarity index 78% rename from support_files/args_parser.py rename to app/support_files/args_parser.py index 2e981c6..77ac039 100644 --- a/support_files/args_parser.py +++ b/app/support_files/args_parser.py @@ -2,6 +2,7 @@ This module is a parser of console arguments for this project. """ import argparse +import app def get_args(): @@ -9,9 +10,9 @@ def get_args(): Function, that parse console args. :return: An object that provides the values ​​of parsed arguments. """ - parser = argparse.ArgumentParser(description="frbz(free reader by Zviger) - python command-line rss reader") + parser = argparse.ArgumentParser(description="It is a python command-line rss reader") parser.add_argument("source", help="RSS URL") - parser.add_argument("--version", action="version", version="%(prog)s 1.0", help="Print version info") + parser.add_argument("--version", action="version", version=f"%(prog)s {app.__version__}", help="Print version info") parser.add_argument("-l", "--limit", type=int, help="Limit news topics if this parameter provided", default=-1) parser.add_argument("--verbose", action="store_true", help="Print result as JSON in stdout", default=False) parser.add_argument("--json", action="store_true", help="Outputs verbose status messages", default=False) diff --git a/support_files/format_converter.py b/app/support_files/format_converter.py similarity index 100% rename from support_files/format_converter.py rename to app/support_files/format_converter.py diff --git a/support_files/rss_parser.py b/app/support_files/rss_parser.py similarity index 100% rename from support_files/rss_parser.py rename to app/support_files/rss_parser.py diff --git a/frbz.py b/app/support_files/rss_reader.py similarity index 96% rename from frbz.py rename to app/support_files/rss_reader.py index 73f42c5..a76d7fb 100644 --- a/frbz.py +++ b/app/support_files/rss_reader.py @@ -1,6 +1,6 @@ import logging import sys -from support_files import args_parser, format_converter, rss_parser +from . import rss_parser, args_parser, format_converter class Reader: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..03d85c6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +beautifulsoup4 +feedparser \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7f1dde8 --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +import setuptools +import os +import app + +with open('requirements.txt') as fp: + install_requires = fp.read() + +setuptools.setup( + name='rss_reader', + version=app.__version__, + author='Budzich Maxim', + author_email='131119999@gmail.com', + long_description=open(os.path.join(os.path.dirname(__file__), 'README.md')).read(), + url='https://github.com/Zviger/PythonHomework/tree/final_proj', + packages=setuptools.find_packages(), + python_requires='>=3.8', + install_requires=install_requires, + entry_points={ + 'console_scripts': ['rss_reader=app.core:main'], + } +) From e1fb15a3ff547fbde3930c5cfa89e7ddad18b8d8 Mon Sep 17 00:00:00 2001 From: Zviger Date: Thu, 14 Nov 2019 04:58:11 +0300 Subject: [PATCH 09/21] Single quotes replaced --- setup.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 7f1dde8..4e18362 100644 --- a/setup.py +++ b/setup.py @@ -2,20 +2,20 @@ import os import app -with open('requirements.txt') as fp: +with open("requirements.txt") as fp: install_requires = fp.read() setuptools.setup( - name='rss_reader', + name="rss_reader", version=app.__version__, - author='Budzich Maxim', - author_email='131119999@gmail.com', - long_description=open(os.path.join(os.path.dirname(__file__), 'README.md')).read(), - url='https://github.com/Zviger/PythonHomework/tree/final_proj', + author="Budzich Maxim", + author_email="131119999@gmail.com", + long_description=open(os.path.join(os.path.dirname(__file__), "README.md")).read(), + url="https://github.com/Zviger/PythonHomework/tree/final_proj", packages=setuptools.find_packages(), - python_requires='>=3.8', + python_requires=">=3.8", install_requires=install_requires, entry_points={ - 'console_scripts': ['rss_reader=app.core:main'], + "console_scripts": ["rss_reader=app.core:main"], } ) From cf4fe4c8c4f2aa4927bba79f07b94b9b6bf2e497 Mon Sep 17 00:00:00 2001 From: Zviger Date: Thu, 14 Nov 2019 18:46:21 +0300 Subject: [PATCH 10/21] Reworked some things The structure of parsed data has been redone. Typehinting is added. Etc... --- README.md | 28 +++++++------- app/core.py | 4 ++ app/support_files/app_logger.py | 21 ++++++++++ app/support_files/dtos.py | 22 +++++++++++ app/support_files/format_converter.py | 38 ++++++++++-------- app/support_files/rss_parser.py | 55 +++++++++++++-------------- app/support_files/rss_reader.py | 41 ++++++++++---------- 7 files changed, 130 insertions(+), 79 deletions(-) create mode 100644 app/support_files/app_logger.py create mode 100644 app/support_files/dtos.py diff --git a/README.md b/README.md index f65b1f8..3796f6f 100644 --- a/README.md +++ b/README.md @@ -23,21 +23,21 @@ optional arguments: ### Json structure ```json [ - { - "feed_title": "Yahoo News - Latest News & Headlines", - "feed_link": "https://www.yahoo.com/news", + {"title": "Yahoo News - Latest News & Headlines", + "link": "https://www.yahoo.com/news", "items": - [ - { - "item_title": "Top House Armed Services Republican: Trump's Ukraine call was 'inappropriate' but not impeachable", - "item_link": "https://news.yahoo.com/trump-impeachment-mac-thornberry-abc-this-week-165743982.html", - "item_author": null, "item_description": "Rep. Mac Thornberry, R-Texas, said President Trump's call with Ukraine'spresident was \"inappropriate\" — but it did not warrant his impeachment.", - "item_date": "Sun, 10 Nov 2019 11:57:43-0500", "item_img_links": - [ - "http://l1.yimg.com/uu/api/res/1.2/nZ9ESccFgs8cyvX3b2LOUA--/YXBwaWQ9eXRhY2h5b247aD04Njt3PTEzMDs-/https://media-mbst-pub-ue1.s3.amazonaws.com/creatr-uploaded-images/2019-11/922d39a0-03db-11ea-bf1e-9fbc638c65a1" - ] - } - ] + [ + {"title": "CouldPresident Trump be impeached and removed from office \u2014 but still reelected?", + "link": "https://news.yahoo.com/could-president-trump-be-impeached-and-removed-from-office-but-still-reelected-184643831.html", + "author": "no author", + "published": "Tue, 12 Nov 2019 13:46:43 -0500", + "description": "What happens when a presidentialimpeachment inquiry runs into a presidential election year? The United States in uncharted territory.", + "img_links": + [ + "http://l2.yimg.com/uu/api/res/1.2/7LKu1VqFsBWR.ZGGf.U.zQ--/YXBwaWQ9eXRhY2h5b247aD04Njt3PTEzMDs-/https://s.yimg.com/os/creatr-images/2019-11/fae1bf00-0581-11ea-93ff-f43ed8c16284" + ] + } + ] } ] ``` diff --git a/app/core.py b/app/core.py index 5856fd4..2bb95bf 100644 --- a/app/core.py +++ b/app/core.py @@ -3,3 +3,7 @@ def main(): Reader.exec_console_args() + + +if __name__ == "__main__": + main() diff --git a/app/support_files/app_logger.py b/app/support_files/app_logger.py new file mode 100644 index 0000000..f48dd3f --- /dev/null +++ b/app/support_files/app_logger.py @@ -0,0 +1,21 @@ +""" +This module provides functions to work with logging. +""" +import logging +import sys + + +def init_logger(name: str) -> logging.Logger: + """ + Initialize and return logger object. + :param name: Name of the logger object. + """ + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + # create the logging file handler + stream_handler = logging.StreamHandler(sys.stdout) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(message)s') + stream_handler.setFormatter(formatter) + # add handler to logger object + logger.addHandler(stream_handler) + return logger diff --git a/app/support_files/dtos.py b/app/support_files/dtos.py new file mode 100644 index 0000000..ebc4e79 --- /dev/null +++ b/app/support_files/dtos.py @@ -0,0 +1,22 @@ +""" +This module contains data classes to work with feeds. +""" +from dataclasses import dataclass, field +from typing import List + + +@dataclass +class Item: + title: str = "no title" + link: str = "no link" + author: str = "no author" + published: str = "no date" + description: str = "description" + img_links: List[str] = field(default_factory=list) + + +@dataclass +class Feed: + title: str = "no title" + link: str = "no link" + items: List[Item] = field(default_factory=list) diff --git a/app/support_files/format_converter.py b/app/support_files/format_converter.py index e1afd5c..cfcfff4 100644 --- a/app/support_files/format_converter.py +++ b/app/support_files/format_converter.py @@ -4,54 +4,60 @@ import textwrap import functools import json +import dataclasses +from typing import List +from app.support_files.dtos import Feed class Converter: """ This class represents format converter for parsed data from RSS. """ - def __init__(self, feeds): + + def __init__(self, feeds: List[Feed]): """ :param feeds: Parsed data from RSS. """ self.__feeds = feeds - def to_console_format(self, str_len=80): + def to_console_format(self, str_len: int = 80) -> str: """ Convert data to console format. :param str_len: Length of output strings. :return: Converted data. """ strings = [] + separator = "-" * str_len for feed in self.__feeds: - strings.append(f"Feed: {feed.get('feed_title')}") - for item in feed.get("items", []): - strings.append("-" * str_len) - strings.append(f"Author: {item.get('item_author')}") - strings.append(f"Date: {item.get('item_date')}") + strings.append(separator) + strings.append(f"Feed: {feed.title}") + for item in feed.items: + strings.append(separator) + strings.append(f"Author: {item.author}") + strings.append(f"Published: {item.published}") strings.append("\n") - strings.append(f"Title: {item.get('item_title')}") - strings.append(f"Description: {item.get('item_description')}") + strings.append(f"Title: {item.title}") + strings.append(f"Description: {item.description}") strings.append("\n") - strings.append(f"Link: {item.get('item_link')}") + strings.append(f"Link: {item.link}") strings.append("Image links:") - for img_link in item.get('item_img_links'): + for img_link in item.img_links: strings.append(f"{img_link}") - strings.append("-" * str_len) + strings.append(separator) strings.append("\n") strings.pop() strings = map(lambda s: textwrap.fill(s, width=str_len) + "\n", strings) - result_string = "-" * str_len + "\n" - result_string += functools.reduce(lambda a, b: a + b, strings) + result_string = functools.reduce(lambda a, b: a + b, strings) return result_string - def to_json_format(self, str_len=80): + def to_json_format(self, str_len: int = 80) -> str: """ Convert data to json format. :param str_len: Length of output strings. :return: Converted data. """ - return textwrap.fill(json.dumps(self.__feeds, ensure_ascii=False), width=str_len) + dicts_of_feeds = list(map(dataclasses.asdict, self.__feeds)) + return textwrap.fill(json.dumps(dicts_of_feeds), width=str_len) diff --git a/app/support_files/rss_parser.py b/app/support_files/rss_parser.py index e0b0e7e..e8f08bc 100644 --- a/app/support_files/rss_parser.py +++ b/app/support_files/rss_parser.py @@ -3,54 +3,53 @@ """ import feedparser from bs4 import BeautifulSoup +from app.support_files.dtos import Item, Feed -FEED_FIELD_MAPPING = {"title": "feed_title", - "link": "feed_link"} +FEED_FIELD_MAPPING = {"title": "title", + "link": "link"} -ITEM_FIELD_MAPPING = {"title": "item_title", - "link": "item_link", - "author": "item_author", - "description": "item_description", - "published": "item_date"} +ITEM_FIELD_MAPPING = {"title": "title", + "link": "link", + "author": "author", + "description": "description", + "published": "published"} + + +def apply_field_mapping(field_mapping: dict, source: dict) -> dict: + return {v: source.get(k) for k, v in field_mapping.items() if source.get(k)} class Parser: """ This class provides methods to parse RSS. """ - def __init__(self, url): + + def __init__(self, url: str): """ :param url: Url of RSS. """ self.url = url - def parse_feed(self, items_limit=-1): + def parse_feed(self, items_limit: int = -1) -> Feed: """ Parse the RSS file. :param items_limit: Limit count of returned items - :return: Dict with parsed data. """ data = feedparser.parse(self.url) if data.bozo != 0 or data.status != 200: - return None + raise ConnectionError("Invalid url") feed = data.get("feed", {}) - result_data = Parser.__apply_field_mapping(FEED_FIELD_MAPPING, feed) - items = [Parser.__apply_field_mapping(ITEM_FIELD_MAPPING, item) - for item in data.get("entries", [])[:items_limit]] - for item in items: - soup = BeautifulSoup(item["item_description"], 'html.parser') - item["item_img_links"] = [link.get("src") for link in soup.find_all("img") if link.get("src")] - item["item_description"] = soup.text - - result_data["items"] = items - return result_data - - @staticmethod - def __apply_field_mapping(field_mapping, source): - data = {} - for key in field_mapping: - data[field_mapping[key]] = source.get(key) - return data + feed_data = apply_field_mapping(FEED_FIELD_MAPPING, feed) + feed = Feed(**feed_data) + items_data = [apply_field_mapping(ITEM_FIELD_MAPPING, item) + for item in data.get("entries", [])[:items_limit]] + for item_data in items_data: + soup = BeautifulSoup(item_data["description"], 'html.parser') + item_data["img_links"] = [link.get("src") for link in soup.find_all("img") if link.get("src")] + item_data["description"] = soup.text + + feed.items = [Item(**item_data) for item_data in items_data] + return feed if __name__ == "__main__": diff --git a/app/support_files/rss_reader.py b/app/support_files/rss_reader.py index a76d7fb..7c5347b 100644 --- a/app/support_files/rss_reader.py +++ b/app/support_files/rss_reader.py @@ -1,25 +1,23 @@ -import logging -import sys -from . import rss_parser, args_parser, format_converter +""" +This module contains class for fork with RSS. +""" +from app.support_files import rss_parser, args_parser, format_converter, app_logger class Reader: + """ + Class for fork with RSS. + """ @staticmethod def exec_console_args(): - logger = logging.getLogger("console_app") - logger.setLevel(logging.INFO) - # create the logging file handler - fh = logging.StreamHandler(sys.stdout) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(message)s') - fh.setFormatter(formatter) - # add handler to logger object - logger.addHandler(fh) - + """ + Execute console commands. + """ + logger = app_logger.init_logger("console_app") _args = args_parser.get_args() - logger.disabled = not _args.verbose + logger.disabled = not _args.verbose logger.info("Program started") - logger.info(f"Parsing {_args.source} started") parser = rss_parser.Parser(_args.source) @@ -28,19 +26,20 @@ def exec_console_args(): if limit < 1 and limit != -1: print("The limit must be -1 or greater than 0") - return + return None - feeds = [parser.parse_feed(limit)] - if feeds[0] is None: - print("Invalid url.") - return + try: + feed = parser.parse_feed(limit) + except ConnectionError as err: + print(err) + return None logger.info(f"Parsing {_args.source} finished") len_each_line = _args.length if len_each_line < 60: print("The length must be greater than 60") - return - converter = format_converter.Converter(feeds) + return None + converter = format_converter.Converter([feed]) if to_json: logger.info("Data is converted to json format and printing is started") print(converter.to_json_format(str_len=len_each_line)) From 8c6e11745147ded371ecb7c8c2aaedb016fd0ca5 Mon Sep 17 00:00:00 2001 From: Zviger Date: Fri, 15 Nov 2019 00:25:42 +0300 Subject: [PATCH 11/21] Fixed typehinting and etc. --- app/core.py | 2 +- app/support_files/app_logger.py | 3 ++- app/support_files/args_parser.py | 4 +++- app/support_files/dtos.py | 6 ++++++ app/support_files/format_converter.py | 3 ++- app/support_files/rss_parser.py | 7 +++++-- app/support_files/rss_reader.py | 2 +- requirements.txt | 4 ++-- 8 files changed, 22 insertions(+), 9 deletions(-) diff --git a/app/core.py b/app/core.py index 2bb95bf..a7713b3 100644 --- a/app/core.py +++ b/app/core.py @@ -1,7 +1,7 @@ from app.support_files.rss_reader import Reader -def main(): +def main() -> None: Reader.exec_console_args() diff --git a/app/support_files/app_logger.py b/app/support_files/app_logger.py index f48dd3f..8ee247b 100644 --- a/app/support_files/app_logger.py +++ b/app/support_files/app_logger.py @@ -2,10 +2,11 @@ This module provides functions to work with logging. """ import logging +from logging import Logger import sys -def init_logger(name: str) -> logging.Logger: +def init_logger(name: str) -> Logger: """ Initialize and return logger object. :param name: Name of the logger object. diff --git a/app/support_files/args_parser.py b/app/support_files/args_parser.py index 77ac039..433ed68 100644 --- a/app/support_files/args_parser.py +++ b/app/support_files/args_parser.py @@ -2,10 +2,12 @@ This module is a parser of console arguments for this project. """ import argparse +from argparse import Namespace + import app -def get_args(): +def get_args() -> Namespace: """ Function, that parse console args. :return: An object that provides the values ​​of parsed arguments. diff --git a/app/support_files/dtos.py b/app/support_files/dtos.py index ebc4e79..70c2fe2 100644 --- a/app/support_files/dtos.py +++ b/app/support_files/dtos.py @@ -7,6 +7,9 @@ @dataclass class Item: + """ + This class represents each item in feed. + """ title: str = "no title" link: str = "no link" author: str = "no author" @@ -17,6 +20,9 @@ class Item: @dataclass class Feed: + """ + This class represents feed. + """ title: str = "no title" link: str = "no link" items: List[Item] = field(default_factory=list) diff --git a/app/support_files/format_converter.py b/app/support_files/format_converter.py index cfcfff4..69eec3b 100644 --- a/app/support_files/format_converter.py +++ b/app/support_files/format_converter.py @@ -6,6 +6,7 @@ import json import dataclasses from typing import List + from app.support_files.dtos import Feed @@ -14,7 +15,7 @@ class Converter: This class represents format converter for parsed data from RSS. """ - def __init__(self, feeds: List[Feed]): + def __init__(self, feeds: List[Feed]) -> None: """ :param feeds: Parsed data from RSS. """ diff --git a/app/support_files/rss_parser.py b/app/support_files/rss_parser.py index e8f08bc..0509c68 100644 --- a/app/support_files/rss_parser.py +++ b/app/support_files/rss_parser.py @@ -1,8 +1,11 @@ """ This module contains class for parsing RSS. """ -import feedparser +from typing import Dict, Any + from bs4 import BeautifulSoup +import feedparser + from app.support_files.dtos import Item, Feed FEED_FIELD_MAPPING = {"title": "title", @@ -15,7 +18,7 @@ "published": "published"} -def apply_field_mapping(field_mapping: dict, source: dict) -> dict: +def apply_field_mapping(field_mapping: Dict[str, str], source: Dict[str, str]) -> Dict[str, Any]: return {v: source.get(k) for k, v in field_mapping.items() if source.get(k)} diff --git a/app/support_files/rss_reader.py b/app/support_files/rss_reader.py index 7c5347b..7d7737c 100644 --- a/app/support_files/rss_reader.py +++ b/app/support_files/rss_reader.py @@ -10,7 +10,7 @@ class Reader: Class for fork with RSS. """ @staticmethod - def exec_console_args(): + def exec_console_args() -> None: """ Execute console commands. """ diff --git a/requirements.txt b/requirements.txt index 03d85c6..f2e0f0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -beautifulsoup4 -feedparser \ No newline at end of file +feedparser==5.2.1 +beautifulsoup4==4.8.1 From 1582369aaf61a641e727db6ee87941b057ffeb28 Mon Sep 17 00:00:00 2001 From: Zviger Date: Fri, 15 Nov 2019 01:36:57 +0300 Subject: [PATCH 12/21] Working with time is reworked --- README.md | 18 ++++++++++-------- app/support_files/dtos.py | 3 ++- app/support_files/format_converter.py | 3 ++- app/support_files/rss_parser.py | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 3796f6f..32acff3 100644 --- a/README.md +++ b/README.md @@ -23,20 +23,22 @@ optional arguments: ### Json structure ```json [ - {"title": "Yahoo News - Latest News & Headlines", + { + "title": "Yahoo News - Latest News & Headlines", "link": "https://www.yahoo.com/news", "items": [ - {"title": "CouldPresident Trump be impeached and removed from office \u2014 but still reelected?", - "link": "https://news.yahoo.com/could-president-trump-be-impeached-and-removed-from-office-but-still-reelected-184643831.html", + { + "title": "Sorry, Hillary: Democrats don't need a savior", + "link": "https://news.yahoo.com/sorry-hillary-democrats-dont-need-a-savior-194253123.html", "author": "no author", - "published": "Tue, 12 Nov 2019 13:46:43 -0500", - "description": "What happens when a presidentialimpeachment inquiry runs into a presidential election year? The United States in uncharted territory.", - "img_links": + "published_parsed": [2019, 11, 13, 19, 42, 53, 2, 317, 0], + "description": "With the Iowa caucuses fast approaching, Hillary Clinton is just the latest in the colorful cast of characters who seem to have surveyed the sprawling Democratic field, sensed something lacking and decided that \u201csomething\u201d might be them.", + "img_links": [ - "http://l2.yimg.com/uu/api/res/1.2/7LKu1VqFsBWR.ZGGf.U.zQ--/YXBwaWQ9eXRhY2h5b247aD04Njt3PTEzMDs-/https://s.yimg.com/os/creatr-images/2019-11/fae1bf00-0581-11ea-93ff-f43ed8c16284" + "http://l.yimg.com/uu/api/res/1.2/xq3Ser6KXPfV6aeoxbq9Uw--/YXBwaWQ9eXRhY2h5b247aD04Njt3PTEzMDs-/https://media-mbst-pub-ue1.s3.amazonaws.com/creatr-uploaded-images/2019-11/14586fd0-064d-11ea-b7df-7288f8d8c1a7" ] - } + } ] } ] diff --git a/app/support_files/dtos.py b/app/support_files/dtos.py index 70c2fe2..bbd7882 100644 --- a/app/support_files/dtos.py +++ b/app/support_files/dtos.py @@ -2,6 +2,7 @@ This module contains data classes to work with feeds. """ from dataclasses import dataclass, field +from time import struct_time, localtime, time from typing import List @@ -13,7 +14,7 @@ class Item: title: str = "no title" link: str = "no link" author: str = "no author" - published: str = "no date" + published_parsed: struct_time = localtime(time()) description: str = "description" img_links: List[str] = field(default_factory=list) diff --git a/app/support_files/format_converter.py b/app/support_files/format_converter.py index 69eec3b..bbcd2c1 100644 --- a/app/support_files/format_converter.py +++ b/app/support_files/format_converter.py @@ -6,6 +6,7 @@ import json import dataclasses from typing import List +from time import strftime, altzone from app.support_files.dtos import Feed @@ -35,7 +36,7 @@ def to_console_format(self, str_len: int = 80) -> str: for item in feed.items: strings.append(separator) strings.append(f"Author: {item.author}") - strings.append(f"Published: {item.published}") + strings.append(f"Published: {strftime('%a, %d %b %Y %X', item.published_parsed)} {altzone / 3600}") strings.append("\n") strings.append(f"Title: {item.title}") strings.append(f"Description: {item.description}") diff --git a/app/support_files/rss_parser.py b/app/support_files/rss_parser.py index 0509c68..a7cbba8 100644 --- a/app/support_files/rss_parser.py +++ b/app/support_files/rss_parser.py @@ -15,7 +15,7 @@ "link": "link", "author": "author", "description": "description", - "published": "published"} + "published_parsed": "published_parsed"} def apply_field_mapping(field_mapping: Dict[str, str], source: Dict[str, str]) -> Dict[str, Any]: From 1c0b75a9466dd52d390135e36f97e68f7adf7060 Mon Sep 17 00:00:00 2001 From: Zviger Date: Thu, 21 Nov 2019 16:33:14 +0300 Subject: [PATCH 13/21] Added caching and bug fixes --- README.md | 6 ++ app/__init__.py | 2 +- app/support_files/args_parser.py | 1 + app/support_files/database.py | 95 +++++++++++++++++++++++++++ app/support_files/dtos.py | 4 ++ app/support_files/exeptions.py | 17 +++++ app/support_files/format_converter.py | 17 ++--- app/support_files/rss_parser.py | 9 ++- app/support_files/rss_reader.py | 43 +++++++++--- requirements.txt | 1 + 10 files changed, 173 insertions(+), 22 deletions(-) create mode 100644 app/support_files/database.py create mode 100644 app/support_files/exeptions.py diff --git a/README.md b/README.md index 32acff3..eb66469 100644 --- a/README.md +++ b/README.md @@ -43,4 +43,10 @@ optional arguments: } ] ``` +### Cashing +The news is saved to the database when news output commands are executed. MongoDB is used as a database management system. +When the --date parameter is used, news is downloaded from the database by the entered date and the entered RSS link. +Features: +* The --limit parameter affects the amount of data loaded into the database. +* Date must be written in the yearmonthday (example - 19991113) format. diff --git a/app/__init__.py b/app/__init__.py index f2dc0e4..fd24f38 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1 +1 @@ -__version__ = "2.0" +__version__ = "3.0" diff --git a/app/support_files/args_parser.py b/app/support_files/args_parser.py index 433ed68..d2209ad 100644 --- a/app/support_files/args_parser.py +++ b/app/support_files/args_parser.py @@ -19,6 +19,7 @@ def get_args() -> Namespace: parser.add_argument("--verbose", action="store_true", help="Print result as JSON in stdout", default=False) parser.add_argument("--json", action="store_true", help="Outputs verbose status messages", default=False) parser.add_argument("--length", type=int, help="Sets the length of each line of news output", default=120) + parser.add_argument("--date", type=str, help="Search past news by date in format %Y%m%d (19991311)", default=None) parser.parse_args() args = parser.parse_args() return args diff --git a/app/support_files/database.py b/app/support_files/database.py new file mode 100644 index 0000000..3e5beb6 --- /dev/null +++ b/app/support_files/database.py @@ -0,0 +1,95 @@ +""" +This module contains class to work with database. +""" +from dataclasses import asdict +from typing import Optional +from time import strptime, mktime, altzone, localtime, struct_time + +from pymongo import MongoClient + +from app.support_files.dtos import Feed, Item +from app.support_files.exeptions import FindFeedError, DateError + + +class DB: + """ + Class to work with database. + """ + + def __init__(self) -> None: + client = MongoClient() + self._db = client["feed_db"] + self._collection = self._db["feed_collection"] + + def insert_feed(self, feed: Feed) -> None: + """ + Insert feed in database. + If this feed exists in the database, then news is added that was not there. + :param feed: Feed, which should be inserted. + """ + cashed_feed = self.find_feed_by_link(feed.rss_link) + + if cashed_feed is not None: + items = set(feed.items) + cashed_items = set(cashed_feed.items) + result_items = list(set(items).union(set(cashed_items))) + result_items = list(map(asdict, result_items)) + self._collection.update_one({"rss_link": feed.rss_link}, {"$set": {"items": result_items}}) + else: + self._collection.insert_one(asdict(feed)) + + def find_feed_by_link(self, link: str) -> Optional[Feed]: + """ + Looks for feed in the database by rss link and returns it. + :param link: Rss link. + :return: Feed, if it exist, otherwise None. + """ + dict_feed = self._collection.find_one({"rss_link": link}) + if dict_feed is None: + return None + del dict_feed["_id"] + feed = Feed(**dict_feed) + feed.items = [Item(**item) for item in dict_feed["items"]] + return feed + + def find_feed_by_link_and_date(self, link: str, date: str, limit: int = -1) -> Feed: + """ + Looks for feed in the database by rss link and date and returns it. + Raise DateError, in it not exist. + :param link: Rss link. + :param date: Need date. + :param limit: Limit count of returned items. + :return: Feed, if it exist. + """ + try: + date = strptime(date, "%Y%m%d") + except ValueError as err: + raise DateError(err.__str__()) + feed = self.find_feed_by_link(link) + if feed is None: + raise FindFeedError("This feed is not cashed") + result_items = [] + count = limit + for item in feed.items: + i_date = struct_time(item.published_parsed) + l_i_date = localtime(mktime(tuple(i_date)) - altzone) + if (l_i_date.tm_year, l_i_date.tm_mon, l_i_date.tm_mday) == (date.tm_year, date.tm_mon, date.tm_mday): + result_items.append(item) + count -= 1 + if count == 0: + break + feed.items = result_items + return feed + + def truncate_collection(self) -> None: + """ + Truncate database. + """ + self._collection.delete_many({}) + + +if __name__ == "__main__": + db = DB() + db.find_feed_by_link_and_date("", "201") + print([len(feed["items"]) for feed in db._collection.find({})]) + print([feed["rss_link"] for feed in db._collection.find({})]) diff --git a/app/support_files/dtos.py b/app/support_files/dtos.py index bbd7882..58fd6bb 100644 --- a/app/support_files/dtos.py +++ b/app/support_files/dtos.py @@ -18,12 +18,16 @@ class Item: description: str = "description" img_links: List[str] = field(default_factory=list) + def __hash__(self) -> int: + return hash(str(self.__dict__)) + @dataclass class Feed: """ This class represents feed. """ + rss_link: str title: str = "no title" link: str = "no link" items: List[Item] = field(default_factory=list) diff --git a/app/support_files/exeptions.py b/app/support_files/exeptions.py new file mode 100644 index 0000000..a078b4b --- /dev/null +++ b/app/support_files/exeptions.py @@ -0,0 +1,17 @@ +""" +This module provides exception classes. +""" + + +class FindFeedError(Exception): + """ + This class should be raised, if received some problems with getting feed. + """ + pass + + +class DateError(ValueError): + """ + This class should be raised, if received some problems with converting date. + """ + pass diff --git a/app/support_files/format_converter.py b/app/support_files/format_converter.py index bbcd2c1..4ba0538 100644 --- a/app/support_files/format_converter.py +++ b/app/support_files/format_converter.py @@ -6,7 +6,7 @@ import json import dataclasses from typing import List -from time import strftime, altzone +from time import strftime, altzone, mktime, localtime from app.support_files.dtos import Feed @@ -29,14 +29,16 @@ def to_console_format(self, str_len: int = 80) -> str: :return: Converted data. """ strings = [] - separator = "-" * str_len + out_separator = "*" * str_len + in_separator = "-" * str_len for feed in self.__feeds: - strings.append(separator) + strings.append(out_separator) strings.append(f"Feed: {feed.title}") for item in feed.items: - strings.append(separator) + strings.append(in_separator) strings.append(f"Author: {item.author}") - strings.append(f"Published: {strftime('%a, %d %b %Y %X', item.published_parsed)} {altzone / 3600}") + published = localtime(mktime(tuple(item.published_parsed)) - altzone) + strings.append(f"Published: {strftime('%a, %d %b %Y %X', published)} {-altzone / 3600}") strings.append("\n") strings.append(f"Title: {item.title}") strings.append(f"Description: {item.description}") @@ -45,9 +47,8 @@ def to_console_format(self, str_len: int = 80) -> str: strings.append("Image links:") for img_link in item.img_links: strings.append(f"{img_link}") - strings.append(separator) - strings.append("\n") - strings.pop() + strings.append(in_separator) + strings.append(out_separator) strings = map(lambda s: textwrap.fill(s, width=str_len) + "\n", strings) diff --git a/app/support_files/rss_parser.py b/app/support_files/rss_parser.py index a7cbba8..5f57843 100644 --- a/app/support_files/rss_parser.py +++ b/app/support_files/rss_parser.py @@ -36,14 +36,16 @@ def __init__(self, url: str): def parse_feed(self, items_limit: int = -1) -> Feed: """ Parse the RSS file. - :param items_limit: Limit count of returned items + :param items_limit: Limit count of returned items. """ data = feedparser.parse(self.url) - if data.bozo != 0 or data.status != 200: + if data.bozo != 0: + raise ConnectionError("Some problems with connection") + if data.status != 200: raise ConnectionError("Invalid url") feed = data.get("feed", {}) feed_data = apply_field_mapping(FEED_FIELD_MAPPING, feed) - feed = Feed(**feed_data) + feed_data["rss_link"] = self.url items_data = [apply_field_mapping(ITEM_FIELD_MAPPING, item) for item in data.get("entries", [])[:items_limit]] for item_data in items_data: @@ -51,6 +53,7 @@ def parse_feed(self, items_limit: int = -1) -> Feed: item_data["img_links"] = [link.get("src") for link in soup.find_all("img") if link.get("src")] item_data["description"] = soup.text + feed = Feed(**feed_data) feed.items = [Item(**item_data) for item_data in items_data] return feed diff --git a/app/support_files/rss_reader.py b/app/support_files/rss_reader.py index 7d7737c..0282d39 100644 --- a/app/support_files/rss_reader.py +++ b/app/support_files/rss_reader.py @@ -1,7 +1,13 @@ """ This module contains class for fork with RSS. """ -from app.support_files import rss_parser, args_parser, format_converter, app_logger +from app.support_files import ( + rss_parser, + args_parser, + format_converter, + app_logger, + database, + exeptions) class Reader: @@ -18,8 +24,9 @@ def exec_console_args() -> None: _args = args_parser.get_args() logger.disabled = not _args.verbose logger.info("Program started") - logger.info(f"Parsing {_args.source} started") - parser = rss_parser.Parser(_args.source) + source = _args.source.rstrip("/") + logger.info(f"Parsing {source} started") + parser = rss_parser.Parser(source) limit = _args.limit to_json = _args.json @@ -28,12 +35,28 @@ def exec_console_args() -> None: print("The limit must be -1 or greater than 0") return None - try: - feed = parser.parse_feed(limit) - except ConnectionError as err: - print(err) - return None - logger.info(f"Parsing {_args.source} finished") + logger.info("Connecting with database") + db = database.DB() + if _args.date is None: + try: + feed = parser.parse_feed(limit) + except ConnectionError as err: + print(err) + return None + logger.info(f"Parsing {source} finished") + + logger.info("Loading parsed data to database") + db.insert_feed(feed) + else: + logger.info("Load data from database") + try: + feed = db.find_feed_by_link_and_date(source, _args.date, limit) + except exeptions.FindFeedError as err: + print(err) + return None + except exeptions.DateError as err: + print(err) + return None len_each_line = _args.length if len_each_line < 60: @@ -46,7 +69,7 @@ def exec_console_args() -> None: else: logger.info("Data is converted to console format and printing is started") print(converter.to_console_format(str_len=len_each_line)) - logger.info("Printing is stoped") + logger.info("Printing is stopped") if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index f2e0f0a..fe7ee2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ feedparser==5.2.1 beautifulsoup4==4.8.1 +pymongo==3.9.0 From e85d9431ae3eb66e475bb7e93cd8589a3d5bae9d Mon Sep 17 00:00:00 2001 From: Zviger Date: Sun, 24 Nov 2019 20:42:17 +0300 Subject: [PATCH 14/21] Working with docker is added --- Dockerfile | 14 ++++++++++++++ README.md | 13 ++++++++++++- app/support_files/database.py | 2 +- docker-compose.yml | 12 ++++++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c2e9719 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.8 + +RUN mkdir /code + +WORKDIR /code + +ADD app/support_files code/app/support_files +ADD requirements.txt code/requirements.txt +ADD app/__init__.py code/app/__init__.py +ADD app/core.py code/app/core.py +ADD README.md code/README.md +ADD setup.py code/setup.py +RUN cd code; python3.8 setup.py install + diff --git a/README.md b/README.md index eb66469..7c6a2a9 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,19 @@ ## It is a one-shot command-line RSS reader by Zviger. ### Installation -```text Clone this repository and run setup.py file with parameters "install --user" +or +Download docker [https://docs.docker.com/] and docker-compose [https://docs.docker.com/compose/install/] +after this run command: +```text +docker-compose up -d ``` +and +```text +docker exec -it rss_reader bash +``` +Fine! + +Now you can write in the docker console "rss_reader" with some parameters ### User interface ```text usage: frbz.py [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] source diff --git a/app/support_files/database.py b/app/support_files/database.py index 3e5beb6..5e2cbcc 100644 --- a/app/support_files/database.py +++ b/app/support_files/database.py @@ -17,7 +17,7 @@ class DB: """ def __init__(self) -> None: - client = MongoClient() + client = MongoClient("mongodb://mongo:27017/") self._db = client["feed_db"] self._collection = self._db["feed_collection"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4340653 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3.0' +services: + app: + build: + . + container_name: rss_reader + stdin_open: true + tty: true + mongo: + image: mongo + ports: + - "27017:27017" \ No newline at end of file From 657dde8c5821d7e3183a0ba4698ada0ddf812895 Mon Sep 17 00:00:00 2001 From: Zviger Date: Sun, 24 Nov 2019 21:41:45 +0300 Subject: [PATCH 15/21] Verion, --help and readme are fixed --- README.md | 6 +++++- app/__init__.py | 2 +- app/support_files/args_parser.py | 2 +- app/support_files/rss_reader.py | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7c6a2a9..62247be 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,9 @@ Fine! Now you can write in the docker console "rss_reader" with some parameters ### User interface ```text -usage: frbz.py [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] source +usage: rss_reader [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] [--date DATE] source + +It is a python command-line rss reader positional arguments: source RSS URL @@ -29,6 +31,8 @@ optional arguments: --verbose Print result as JSON in stdout --json Outputs verbose status messages --length LENGTH Sets the length of each line of news output + --date DATE Search past news by date in format yeardaymonth (19991311) + ``` ### Json structure diff --git a/app/__init__.py b/app/__init__.py index fd24f38..ec2d1d5 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1 +1 @@ -__version__ = "3.0" +__version__ = "3.1" diff --git a/app/support_files/args_parser.py b/app/support_files/args_parser.py index d2209ad..dc4f1ba 100644 --- a/app/support_files/args_parser.py +++ b/app/support_files/args_parser.py @@ -19,7 +19,7 @@ def get_args() -> Namespace: parser.add_argument("--verbose", action="store_true", help="Print result as JSON in stdout", default=False) parser.add_argument("--json", action="store_true", help="Outputs verbose status messages", default=False) parser.add_argument("--length", type=int, help="Sets the length of each line of news output", default=120) - parser.add_argument("--date", type=str, help="Search past news by date in format %Y%m%d (19991311)", default=None) + parser.add_argument("--date", help="Search past news by date in format yeardaymonth (19991311)") parser.parse_args() args = parser.parse_args() return args diff --git a/app/support_files/rss_reader.py b/app/support_files/rss_reader.py index 0282d39..f38fe2a 100644 --- a/app/support_files/rss_reader.py +++ b/app/support_files/rss_reader.py @@ -37,7 +37,7 @@ def exec_console_args() -> None: logger.info("Connecting with database") db = database.DB() - if _args.date is None: + if _args.date == "": try: feed = parser.parse_feed(limit) except ConnectionError as err: From 737b21f3bc6c53845f081342bcb6adcf5be39a0e Mon Sep 17 00:00:00 2001 From: Zviger Date: Tue, 26 Nov 2019 15:56:36 +0300 Subject: [PATCH 16/21] Docker files are saved --- Dockerfile | 2 +- README.md | 2 +- app/saved_files/sad | 0 app/support_files/adas.mobi | 20 +++++++++++++ app/support_files/args_parser.py | 3 +- app/support_files/database.py | 4 ++- app/support_files/format_converter.py | 43 +++++++++++++++++++++++++-- app/support_files/rss_reader.py | 2 +- app/support_files/templates/html/feed | 6 ++++ app/support_files/templates/html/item | 9 ++++++ app/support_files/templates/html/main | 14 +++++++++ docker-compose.yml | 11 +++++-- 12 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 app/saved_files/sad create mode 100644 app/support_files/adas.mobi create mode 100644 app/support_files/templates/html/feed create mode 100644 app/support_files/templates/html/item create mode 100644 app/support_files/templates/html/main diff --git a/Dockerfile b/Dockerfile index c2e9719..4384804 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ RUN mkdir /code WORKDIR /code -ADD app/support_files code/app/support_files +ADD app/saved_files code/app/saved_files ADD requirements.txt code/requirements.txt ADD app/__init__.py code/app/__init__.py ADD app/core.py code/app/core.py diff --git a/README.md b/README.md index 62247be..ffcabd8 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ docker-compose up -d ``` and ```text -docker exec -it rss_reader bash +docker exec -it rss_reader bash (this command u will run every time, when u need to use reader) ``` Fine! diff --git a/app/saved_files/sad b/app/saved_files/sad new file mode 100644 index 0000000..e69de29 diff --git a/app/support_files/adas.mobi b/app/support_files/adas.mobi new file mode 100644 index 0000000..2eb1770 --- /dev/null +++ b/app/support_files/adas.mobi @@ -0,0 +1,20 @@ + + + + + Title + + + +
+ +
+ + \ No newline at end of file diff --git a/app/support_files/args_parser.py b/app/support_files/args_parser.py index dc4f1ba..7648192 100644 --- a/app/support_files/args_parser.py +++ b/app/support_files/args_parser.py @@ -19,7 +19,8 @@ def get_args() -> Namespace: parser.add_argument("--verbose", action="store_true", help="Print result as JSON in stdout", default=False) parser.add_argument("--json", action="store_true", help="Outputs verbose status messages", default=False) parser.add_argument("--length", type=int, help="Sets the length of each line of news output", default=120) - parser.add_argument("--date", help="Search past news by date in format yeardaymonth (19991311)") + parser.add_argument("--date", type=str, help="Search past news by date in format yeardaymonth (19991311)", + default=None) parser.parse_args() args = parser.parse_args() return args diff --git a/app/support_files/database.py b/app/support_files/database.py index 5e2cbcc..274cd9e 100644 --- a/app/support_files/database.py +++ b/app/support_files/database.py @@ -1,6 +1,7 @@ """ This module contains class to work with database. """ +import os from dataclasses import asdict from typing import Optional from time import strptime, mktime, altzone, localtime, struct_time @@ -17,7 +18,8 @@ class DB: """ def __init__(self) -> None: - client = MongoClient("mongodb://mongo:27017/") + mongo_host = os.getenv('MONGO_HOST', '127.0.0.1') + client = MongoClient(f"mongodb://{mongo_host}:27017/") self._db = client["feed_db"] self._collection = self._db["feed_collection"] diff --git a/app/support_files/format_converter.py b/app/support_files/format_converter.py index 4ba0538..688abdc 100644 --- a/app/support_files/format_converter.py +++ b/app/support_files/format_converter.py @@ -9,6 +9,12 @@ from time import strftime, altzone, mktime, localtime from app.support_files.dtos import Feed +from app.support_files.rss_parser import Parser + + +def convert_date(date): + published = localtime(mktime(tuple(date)) - altzone) + return " ".join([strftime('%a, %d %b %Y %X', published), str(-altzone / 3600)]) class Converter: @@ -37,8 +43,7 @@ def to_console_format(self, str_len: int = 80) -> str: for item in feed.items: strings.append(in_separator) strings.append(f"Author: {item.author}") - published = localtime(mktime(tuple(item.published_parsed)) - altzone) - strings.append(f"Published: {strftime('%a, %d %b %Y %X', published)} {-altzone / 3600}") + strings.append(f"Published: {convert_date(item.published_parsed)}") strings.append("\n") strings.append(f"Title: {item.title}") strings.append(f"Description: {item.description}") @@ -64,3 +69,37 @@ def to_json_format(self, str_len: int = 80) -> str: """ dicts_of_feeds = list(map(dataclasses.asdict, self.__feeds)) return textwrap.fill(json.dumps(dicts_of_feeds), width=str_len) + + def to_html_format(self) -> str: + with open("templates/html/main", "r") as main_file: + main_template = main_file.read() + with open("templates/html/feed", "r") as feed_file: + feed_template = feed_file.read() + with open("templates/html/item", "r") as item_file: + item_template = item_file.read() + feed_str_s = [] + for feed in self.__feeds: + item_str_s = [] + for item in feed.items: + item_img = "http://view.dreamstalk.ca/breeze5/images/no-photo.png" + try: + kek = item.img_links[0] + except IndexError: + pass + item_str_s.append(item_template.format(item_title=item.title, + item_link=item.link, + item_author=item.author, + item_published=convert_date(item.published_parsed), + item_description=item.description, + item_img=item_img)) + feed_str_s.append(feed_template.format(feed_title=feed.title, + feed_link=feed.link, + items="\n".join(item_str_s))) + result_str = main_template.format(feeds="\n".join(feed_str_s)) + return result_str + + +if __name__ == "__main__": + parser = Parser("https://news.yahoo.com/rss/") + converter = Converter([parser.parse_feed(items_limit=3)]) + print(converter.to_html_format()) diff --git a/app/support_files/rss_reader.py b/app/support_files/rss_reader.py index f38fe2a..0282d39 100644 --- a/app/support_files/rss_reader.py +++ b/app/support_files/rss_reader.py @@ -37,7 +37,7 @@ def exec_console_args() -> None: logger.info("Connecting with database") db = database.DB() - if _args.date == "": + if _args.date is None: try: feed = parser.parse_feed(limit) except ConnectionError as err: diff --git a/app/support_files/templates/html/feed b/app/support_files/templates/html/feed new file mode 100644 index 0000000..0d9ea5a --- /dev/null +++ b/app/support_files/templates/html/feed @@ -0,0 +1,6 @@ +
+{feed_title} +
+{items} +
+
\ No newline at end of file diff --git a/app/support_files/templates/html/item b/app/support_files/templates/html/item new file mode 100644 index 0000000..8e1d764 --- /dev/null +++ b/app/support_files/templates/html/item @@ -0,0 +1,9 @@ +
+{item_title} +
+
Author: {item_author}
+
Published: {item_published}
+

{item_description}

+Responsive image +
+
\ No newline at end of file diff --git a/app/support_files/templates/html/main b/app/support_files/templates/html/main new file mode 100644 index 0000000..6b95c19 --- /dev/null +++ b/app/support_files/templates/html/main @@ -0,0 +1,14 @@ + + + + +Title + + + +
+{feeds} +
+ + \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4340653..8810696 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.0' +version: "3.6" services: app: build: @@ -6,7 +6,14 @@ services: container_name: rss_reader stdin_open: true tty: true + environment: + MONGO_HOST: mongo + volumes: + - type: bind + source: ./app/saved_files + target: /code/app/saved_files + consistency: delegated mongo: image: mongo ports: - - "27017:27017" \ No newline at end of file + - "27017:27017" From 3dee3789c7e40f8280afeeb40f362def61d683df Mon Sep 17 00:00:00 2001 From: Zviger Date: Thu, 28 Nov 2019 17:51:52 +0300 Subject: [PATCH 17/21] Saving data in html and fb2 are added Added more logging. Saving images url is reworked. --- Dockerfile | 14 --- README.md | 31 +++--- app/__init__.py | 2 +- app/saved_files/sad | 0 app/support_files/adas.mobi | 20 ---- app/support_files/app_logger.py | 8 ++ app/support_files/args_parser.py | 2 + app/support_files/config.py | 5 + .../{database.py => db_manager.py} | 13 ++- app/support_files/exeptions.py | 14 +++ app/support_files/file_manager.py | 37 +++++++ app/support_files/format_converter.py | 104 +++++++++++++++--- app/support_files/rss_parser.py | 16 ++- app/support_files/rss_reader.py | 53 +++++++-- app/support_files/templates/fb2/binary | 3 + app/support_files/templates/fb2/feed | 11 ++ app/support_files/templates/fb2/image | 1 + app/support_files/templates/fb2/item | 20 ++++ app/support_files/templates/fb2/main | 23 ++++ app/support_files/templates/html/image | 1 + app/support_files/templates/html/item | 6 +- app/support_files/templates/html/main | 2 +- docker-compose.yml | 19 ---- requirements.txt | 1 + setup.py | 9 +- 25 files changed, 300 insertions(+), 115 deletions(-) delete mode 100644 Dockerfile delete mode 100644 app/saved_files/sad delete mode 100644 app/support_files/adas.mobi create mode 100644 app/support_files/config.py rename app/support_files/{database.py => db_manager.py} (87%) create mode 100644 app/support_files/file_manager.py create mode 100644 app/support_files/templates/fb2/binary create mode 100644 app/support_files/templates/fb2/feed create mode 100644 app/support_files/templates/fb2/image create mode 100644 app/support_files/templates/fb2/item create mode 100644 app/support_files/templates/fb2/main create mode 100644 app/support_files/templates/html/image delete mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 4384804..0000000 --- a/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.8 - -RUN mkdir /code - -WORKDIR /code - -ADD app/saved_files code/app/saved_files -ADD requirements.txt code/requirements.txt -ADD app/__init__.py code/app/__init__.py -ADD app/core.py code/app/core.py -ADD README.md code/README.md -ADD setup.py code/setup.py -RUN cd code; python3.8 setup.py install - diff --git a/README.md b/README.md index ffcabd8..c6bf93c 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,12 @@ ## It is a one-shot command-line RSS reader by Zviger. ### Installation -Clone this repository and run setup.py file with parameters "install --user" -or -Download docker [https://docs.docker.com/] and docker-compose [https://docs.docker.com/compose/install/] -after this run command: +Clone this repository and run ```text -docker-compose up -d +pip install . --user ``` -and -```text -docker exec -it rss_reader bash (this command u will run every time, when u need to use reader) -``` -Fine! - -Now you can write in the docker console "rss_reader" with some parameters ### User interface ```text -usage: rss_reader [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] [--date DATE] source - -It is a python command-line rss reader +usage: rss-reader [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] [--date DATE] [--to_html PATH] [--to_fb2 PATH] source positional arguments: source RSS URL @@ -32,7 +20,8 @@ optional arguments: --json Outputs verbose status messages --length LENGTH Sets the length of each line of news output --date DATE Search past news by date in format yeardaymonth (19991311) - + --to_html PATH Save news by path in html format + --to_fb2 PATH Save news by path in fb2 format ``` ### Json structure @@ -64,4 +53,12 @@ When the --date parameter is used, news is downloaded from the database by the e Features: * The --limit parameter affects the amount of data loaded into the database. -* Date must be written in the yearmonthday (example - 19991113) format. +* Date must be written in the yearmonthday (example - 19991113) format. + +### Saving in files +Using the "--to_html" and "--to_fb2" parameters, you can save files at a given path. +The path should be written in the style of UNIX systems (example: ./some/folder). +File names are formed using the "feed[index].[format]" template (example: feed13.html). +File indices go sequentially and a new file fills this sequence or is set to the end. +What does this mean: if, for example, there are files "feed1.html" and "feed3.html", +a new file will be created with the name "feed2.html". \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py index ec2d1d5..b777be9 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1 +1 @@ -__version__ = "3.1" +__version__ = "4.0" diff --git a/app/saved_files/sad b/app/saved_files/sad deleted file mode 100644 index e69de29..0000000 diff --git a/app/support_files/adas.mobi b/app/support_files/adas.mobi deleted file mode 100644 index 2eb1770..0000000 --- a/app/support_files/adas.mobi +++ /dev/null @@ -1,20 +0,0 @@ - - - - - Title - - - -
- -
- - \ No newline at end of file diff --git a/app/support_files/app_logger.py b/app/support_files/app_logger.py index 8ee247b..ad6e31e 100644 --- a/app/support_files/app_logger.py +++ b/app/support_files/app_logger.py @@ -20,3 +20,11 @@ def init_logger(name: str) -> Logger: # add handler to logger object logger.addHandler(stream_handler) return logger + + +def get_logger(name: str) -> Logger: + """ + Return logger object. + :param name: Name of the logger object. + """ + return logging.getLogger(name) diff --git a/app/support_files/args_parser.py b/app/support_files/args_parser.py index 7648192..2feb468 100644 --- a/app/support_files/args_parser.py +++ b/app/support_files/args_parser.py @@ -21,6 +21,8 @@ def get_args() -> Namespace: parser.add_argument("--length", type=int, help="Sets the length of each line of news output", default=120) parser.add_argument("--date", type=str, help="Search past news by date in format yeardaymonth (19991311)", default=None) + parser.add_argument("--to_html", metavar="PATH", type=str, help="Save news by path in html format", default=None) + parser.add_argument("--to_fb2", metavar="PATH", type=str, help="Save news by path in fb2 format", default=None) parser.parse_args() args = parser.parse_args() return args diff --git a/app/support_files/config.py b/app/support_files/config.py new file mode 100644 index 0000000..926092d --- /dev/null +++ b/app/support_files/config.py @@ -0,0 +1,5 @@ +""" +This module provides configuration strings +""" + +APP_NAME = "rss-reader" diff --git a/app/support_files/database.py b/app/support_files/db_manager.py similarity index 87% rename from app/support_files/database.py rename to app/support_files/db_manager.py index 274cd9e..ee037f9 100644 --- a/app/support_files/database.py +++ b/app/support_files/db_manager.py @@ -10,9 +10,11 @@ from app.support_files.dtos import Feed, Item from app.support_files.exeptions import FindFeedError, DateError +from app.support_files.config import APP_NAME +from app.support_files.app_logger import get_logger -class DB: +class DBManager: """ Class to work with database. """ @@ -22,6 +24,7 @@ def __init__(self) -> None: client = MongoClient(f"mongodb://{mongo_host}:27017/") self._db = client["feed_db"] self._collection = self._db["feed_collection"] + self._logger = get_logger(APP_NAME) def insert_feed(self, feed: Feed) -> None: """ @@ -29,7 +32,9 @@ def insert_feed(self, feed: Feed) -> None: If this feed exists in the database, then news is added that was not there. :param feed: Feed, which should be inserted. """ + self._logger.info("Loading data from database to join with inserted data is started") cashed_feed = self.find_feed_by_link(feed.rss_link) + self._logger.info("Loading data from database to join with inserted data is finished") if cashed_feed is not None: items = set(feed.items) @@ -39,6 +44,7 @@ def insert_feed(self, feed: Feed) -> None: self._collection.update_one({"rss_link": feed.rss_link}, {"$set": {"items": result_items}}) else: self._collection.insert_one(asdict(feed)) + self._logger.info("New and old data are joined") def find_feed_by_link(self, link: str) -> Optional[Feed]: """ @@ -91,7 +97,4 @@ def truncate_collection(self) -> None: if __name__ == "__main__": - db = DB() - db.find_feed_by_link_and_date("", "201") - print([len(feed["items"]) for feed in db._collection.find({})]) - print([feed["rss_link"] for feed in db._collection.find({})]) + DBManager().truncate_collection() diff --git a/app/support_files/exeptions.py b/app/support_files/exeptions.py index a078b4b..c7b5a74 100644 --- a/app/support_files/exeptions.py +++ b/app/support_files/exeptions.py @@ -15,3 +15,17 @@ class DateError(ValueError): This class should be raised, if received some problems with converting date. """ pass + + +class DirError(Exception): + """ + This class should be raised, if received path is not a directory. + """ + pass + + +class DirExistsError(Exception): + """ + This class should be raised, if directory which was received by bath not exists. + """ + pass diff --git a/app/support_files/file_manager.py b/app/support_files/file_manager.py new file mode 100644 index 0000000..5ac4efe --- /dev/null +++ b/app/support_files/file_manager.py @@ -0,0 +1,37 @@ +""" +This module contains functions to work with files. +""" +from re import findall +from pathlib import Path + +from app.support_files.exeptions import DirError, DirExistsError + + +def store_str_to_file(data: str, path: str, file_format: str, file_name: str = "feed") -> None: + """ + Saves data to a folder at a given path to a file with a given name, to which the index is added, and the format. + The file name is based on files with a specific file name format that are already in the folder. + File indices go sequentially and a new file fills this sequence or is set to the end. + """ + true_path = Path(path) + if not true_path.exists(): + raise DirExistsError(f"This directory not exists: {true_path}") + if not true_path.is_dir(): + raise DirError(f"Is not a directory: {true_path}") + file_indexes = [] + for _dir in true_path.iterdir(): + if findall(fr"{file_name}\d+.{file_format}", _dir.name): + file_indexes.append(int(findall(r"\d+", _dir.name)[0])) + file_indexes = sorted(file_indexes) + current_index = 1 + for index in file_indexes: + if index - current_index > 1: + break + else: + current_index += 1 + with open(true_path.joinpath("".join([file_name, str(current_index), ".", file_format])), "w") as file: + file.write(data) + + +if __name__ == "__main__": + print(Path(".").name) diff --git a/app/support_files/format_converter.py b/app/support_files/format_converter.py index 688abdc..a15461a 100644 --- a/app/support_files/format_converter.py +++ b/app/support_files/format_converter.py @@ -1,22 +1,50 @@ """ This module contains class for converting parsed data from RSS. """ +import base64 import textwrap import functools import json import dataclasses from typing import List -from time import strftime, altzone, mktime, localtime +from time import strftime, altzone, mktime, localtime, ctime, time, struct_time + +import urllib3 from app.support_files.dtos import Feed from app.support_files.rss_parser import Parser +from app.support_files.config import APP_NAME +from app.support_files.app_logger import get_logger -def convert_date(date): +def convert_date(date: struct_time) -> str: + """ + Converts date too human readable format. + """ published = localtime(mktime(tuple(date)) - altzone) return " ".join([strftime('%a, %d %b %Y %X', published), str(-altzone / 3600)]) +def get_templates(template_type: str, template_names: List[str]) -> List[str]: + """ + Reads templates from files. + """ + templates = [] + for template_name in template_names: + with open(f"app/support_files/templates/{template_type}/{template_name}", "r") as main_file: + templates.append(main_file.read()) + return templates + + +def get_img_by_url(url: str) -> str: + """ + Gets img in base64 format by url. + """ + http = urllib3.PoolManager() + response = http.request("GET", url) + return str(base64.b64encode(response.data), "utf-8") + + class Converter: """ This class represents format converter for parsed data from RSS. @@ -27,6 +55,7 @@ def __init__(self, feeds: List[Feed]) -> None: :param feeds: Parsed data from RSS. """ self.__feeds = feeds + self._logger = get_logger(APP_NAME) def to_console_format(self, str_len: int = 80) -> str: """ @@ -71,35 +100,74 @@ def to_json_format(self, str_len: int = 80) -> str: return textwrap.fill(json.dumps(dicts_of_feeds), width=str_len) def to_html_format(self) -> str: - with open("templates/html/main", "r") as main_file: - main_template = main_file.read() - with open("templates/html/feed", "r") as feed_file: - feed_template = feed_file.read() - with open("templates/html/item", "r") as item_file: - item_template = item_file.read() + """ + Convert data to html format. + :return: Converted data. + """ + template_names = ["main", "feed", "item", "image"] + main_template, feed_template, item_template, image_template = get_templates("html", template_names) + feed_str_s = [] + for feed in self.__feeds: + item_str_s = [] + for item in feed.items: + item_img_links = ["http://view.dreamstalk.ca/breeze5/images/no-photo.png"] + if item.img_links: + item_img_links = item.img_links + img_str_s = [] + for item_img_link in item_img_links: + img_str_s.append(image_template.format(item_img_link=item_img_link)) + item_str_s.append(item_template.format(item_title=item.title, + item_link=item.link, + item_author=item.author, + item_published=convert_date(item.published_parsed), + item_description=item.description, + item_images="\n".join(img_str_s))) + feed_str_s.append(feed_template.format(feed_title=feed.title, + feed_link=feed.link, + items="\n".join(item_str_s))) + result_str = main_template.format(feeds="\n".join(feed_str_s), + title="Feeds") + return result_str + + def to_fb2_format(self) -> str: + """ + Convert data to html format. + :return: Converted data. + """ + template_names = ["main", "feed", "item", "image", "binary"] + main_template, feed_template, item_template, image_template, binary_template =\ + get_templates("fb2", template_names) feed_str_s = [] + img_content_str_s = [] + img_index = 0 for feed in self.__feeds: item_str_s = [] for item in feed.items: - item_img = "http://view.dreamstalk.ca/breeze5/images/no-photo.png" - try: - kek = item.img_links[0] - except IndexError: - pass + item_img_links = ["http://view.dreamstalk.ca/breeze5/images/no-photo.png"] + if item.img_links: + item_img_links = item.img_links + img_str_s = [] + for item_img_link in item_img_links[:1]: + img_str_s.append(image_template.format(img_index=img_index)) + self._logger.info(f"Downloading and converting image from {item_img_link} to binary are started") + img_content_str_s.append(binary_template.format(img_index=img_index, + img_content=get_img_by_url(item_img_link))) + self._logger.info(f"Downloading and converting image from {item_img_link} to binary are finished") + img_index += 1 item_str_s.append(item_template.format(item_title=item.title, item_link=item.link, item_author=item.author, item_published=convert_date(item.published_parsed), item_description=item.description, - item_img=item_img)) + item_images="\n".join(img_str_s))) feed_str_s.append(feed_template.format(feed_title=feed.title, feed_link=feed.link, items="\n".join(item_str_s))) - result_str = main_template.format(feeds="\n".join(feed_str_s)) + result_str = main_template.format(date=ctime(time()), + feeds="\n".join(feed_str_s), + img_contents="\n".join(img_content_str_s)) return result_str if __name__ == "__main__": - parser = Parser("https://news.yahoo.com/rss/") - converter = Converter([parser.parse_feed(items_limit=3)]) - print(converter.to_html_format()) + print(Converter([Parser("https://news.yahoo.com/rss/").parse_feed(items_limit=3)]).to_html_format()) diff --git a/app/support_files/rss_parser.py b/app/support_files/rss_parser.py index 5f57843..4cf5032 100644 --- a/app/support_files/rss_parser.py +++ b/app/support_files/rss_parser.py @@ -7,6 +7,8 @@ import feedparser from app.support_files.dtos import Item, Feed +from app.support_files.config import APP_NAME +from app.support_files.app_logger import get_logger FEED_FIELD_MAPPING = {"title": "title", "link": "link"} @@ -15,7 +17,8 @@ "link": "link", "author": "author", "description": "description", - "published_parsed": "published_parsed"} + "published_parsed": "published_parsed", + "media_content": "img_links"} def apply_field_mapping(field_mapping: Dict[str, str], source: Dict[str, str]) -> Dict[str, Any]: @@ -32,32 +35,37 @@ def __init__(self, url: str): :param url: Url of RSS. """ self.url = url + self._logger = get_logger(APP_NAME) def parse_feed(self, items_limit: int = -1) -> Feed: """ Parse the RSS file. :param items_limit: Limit count of returned items. """ + self._logger.info(f"Reading {self.url} is started") data = feedparser.parse(self.url) if data.bozo != 0: raise ConnectionError("Some problems with connection") if data.status != 200: raise ConnectionError("Invalid url") + self._logger.info(f"Reading {self.url} is finished") + self._logger.info("Converting read data to standard form is started") feed = data.get("feed", {}) feed_data = apply_field_mapping(FEED_FIELD_MAPPING, feed) feed_data["rss_link"] = self.url items_data = [apply_field_mapping(ITEM_FIELD_MAPPING, item) for item in data.get("entries", [])[:items_limit]] for item_data in items_data: - soup = BeautifulSoup(item_data["description"], 'html.parser') - item_data["img_links"] = [link.get("src") for link in soup.find_all("img") if link.get("src")] + soup = BeautifulSoup(item_data.get("description", ""), 'html.parser') item_data["description"] = soup.text + item_data["img_links"] = [item["url"] for item in item_data.get("img_links", [])] feed = Feed(**feed_data) feed.items = [Item(**item_data) for item_data in items_data] + self._logger.info("Converting read data to standard form is finished") return feed if __name__ == "__main__": - parser = Parser("https://news.tut.by/rss/economics.rss") + parser = Parser("http://www.bbc.co.uk/music/genres/classical/reviews.rss") print(parser.parse_feed(2)) diff --git a/app/support_files/rss_reader.py b/app/support_files/rss_reader.py index 0282d39..2332c5f 100644 --- a/app/support_files/rss_reader.py +++ b/app/support_files/rss_reader.py @@ -6,8 +6,10 @@ args_parser, format_converter, app_logger, - database, + db_manager, exeptions) +from app.support_files.file_manager import store_str_to_file +from app.support_files.config import APP_NAME class Reader: @@ -20,7 +22,7 @@ def exec_console_args() -> None: """ Execute console commands. """ - logger = app_logger.init_logger("console_app") + logger = app_logger.init_logger(APP_NAME) _args = args_parser.get_args() logger.disabled = not _args.verbose logger.info("Program started") @@ -30,13 +32,15 @@ def exec_console_args() -> None: limit = _args.limit to_json = _args.json + to_html_path = _args.to_html + to_fb2_path = _args.to_fb2 if limit < 1 and limit != -1: print("The limit must be -1 or greater than 0") return None logger.info("Connecting with database") - db = database.DB() + db = db_manager.DBManager() if _args.date is None: try: feed = parser.parse_feed(limit) @@ -45,10 +49,11 @@ def exec_console_args() -> None: return None logger.info(f"Parsing {source} finished") - logger.info("Loading parsed data to database") + logger.info("Loading parsed data to database is started") db.insert_feed(feed) + logger.info("Loading parsed data to database is finished") else: - logger.info("Load data from database") + logger.info("Load data from database is started") try: feed = db.find_feed_by_link_and_date(source, _args.date, limit) except exeptions.FindFeedError as err: @@ -57,19 +62,45 @@ def exec_console_args() -> None: except exeptions.DateError as err: print(err) return None + logger.info("Load data from database is finished") len_each_line = _args.length if len_each_line < 60: print("The length must be greater than 60") return None converter = format_converter.Converter([feed]) - if to_json: - logger.info("Data is converted to json format and printing is started") - print(converter.to_json_format(str_len=len_each_line)) + + if to_html_path: + logger.info("Saving data in html format in file is started") + try: + store_str_to_file(converter.to_html_format(), to_html_path, "html") + except exeptions.DirExistsError as err: + print(err) + return None + except exeptions.DirError as err: + print(err) + return None + logger.info("Saving data in html format in file is finished") + elif to_fb2_path: + logger.info("Saving data in fb2 format in file is started") + try: + store_str_to_file(converter.to_fb2_format(), to_fb2_path, "fb2") + except exeptions.DirExistsError as err: + print(err) + return None + except exeptions.DirError as err: + print(err) + return None + logger.info("Saving data in fb2 format in file is finished") else: - logger.info("Data is converted to console format and printing is started") - print(converter.to_console_format(str_len=len_each_line)) - logger.info("Printing is stopped") + if to_json: + logger.info("Data is converted to json format and printing is started") + print(converter.to_json_format(str_len=len_each_line)) + else: + logger.info("Data is converted to console format and printing is started") + print(converter.to_console_format(str_len=len_each_line)) + logger.info("Printing is finished") + return None if __name__ == "__main__": diff --git a/app/support_files/templates/fb2/binary b/app/support_files/templates/fb2/binary new file mode 100644 index 0000000..6620e68 --- /dev/null +++ b/app/support_files/templates/fb2/binary @@ -0,0 +1,3 @@ + +{img_content} + \ No newline at end of file diff --git a/app/support_files/templates/fb2/feed b/app/support_files/templates/fb2/feed new file mode 100644 index 0000000..6475567 --- /dev/null +++ b/app/support_files/templates/fb2/feed @@ -0,0 +1,11 @@ + +<p> +{feed_title} +</p> + +

+Link: {feed_link} +

+
+{items} +
\ No newline at end of file diff --git a/app/support_files/templates/fb2/image b/app/support_files/templates/fb2/image new file mode 100644 index 0000000..9c6cdd9 --- /dev/null +++ b/app/support_files/templates/fb2/image @@ -0,0 +1 @@ +

\ No newline at end of file diff --git a/app/support_files/templates/fb2/item b/app/support_files/templates/fb2/item new file mode 100644 index 0000000..50391c3 --- /dev/null +++ b/app/support_files/templates/fb2/item @@ -0,0 +1,20 @@ + +

+{item_title} +

+ +

+Link: {item_link} +

+

+Author: {item_author} +

+

+Published: {item_published} +

+ +

+{item_description} +

+ +{item_images} \ No newline at end of file diff --git a/app/support_files/templates/fb2/main b/app/support_files/templates/fb2/main new file mode 100644 index 0000000..2fc2fdf --- /dev/null +++ b/app/support_files/templates/fb2/main @@ -0,0 +1,23 @@ + + + + + feeds + ZvigerRogert + Feeds + + + ZvigerRogert + calibre 4.4.0 + {date} + + + + + +
+ {feeds} +
+ +{img_contents} + \ No newline at end of file diff --git a/app/support_files/templates/html/image b/app/support_files/templates/html/image new file mode 100644 index 0000000..b6ce13d --- /dev/null +++ b/app/support_files/templates/html/image @@ -0,0 +1 @@ +Responsive image \ No newline at end of file diff --git a/app/support_files/templates/html/item b/app/support_files/templates/html/item index 8e1d764..58eeb24 100644 --- a/app/support_files/templates/html/item +++ b/app/support_files/templates/html/item @@ -1,9 +1,11 @@
{item_title} +
Author: {item_author}
Published: {item_published}
-

{item_description}

-Responsive image +
+

{item_description}

+{item_images}
\ No newline at end of file diff --git a/app/support_files/templates/html/main b/app/support_files/templates/html/main index 6b95c19..ee4fcb8 100644 --- a/app/support_files/templates/html/main +++ b/app/support_files/templates/html/main @@ -2,7 +2,7 @@ -Title +{title} diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 8810696..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: "3.6" -services: - app: - build: - . - container_name: rss_reader - stdin_open: true - tty: true - environment: - MONGO_HOST: mongo - volumes: - - type: bind - source: ./app/saved_files - target: /code/app/saved_files - consistency: delegated - mongo: - image: mongo - ports: - - "27017:27017" diff --git a/requirements.txt b/requirements.txt index fe7ee2b..5f50945 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +urllib3==1.25.7 feedparser==5.2.1 beautifulsoup4==4.8.1 pymongo==3.9.0 diff --git a/setup.py b/setup.py index 4e18362..966519d 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,15 @@ -import setuptools import os + +import setuptools + import app +from app.support_files.config import APP_NAME with open("requirements.txt") as fp: install_requires = fp.read() setuptools.setup( - name="rss_reader", + name=APP_NAME, version=app.__version__, author="Budzich Maxim", author_email="131119999@gmail.com", @@ -16,6 +19,6 @@ python_requires=">=3.8", install_requires=install_requires, entry_points={ - "console_scripts": ["rss_reader=app.core:main"], + "console_scripts": [f"{APP_NAME}=app.core:main"], } ) From 3ca72af0ef0f3f376e604abac3bcac6d88571377 Mon Sep 17 00:00:00 2001 From: Zviger Date: Sat, 30 Nov 2019 03:26:03 +0300 Subject: [PATCH 18/21] Added color text --- app/__init__.py | 2 +- app/support_files/args_parser.py | 1 + app/support_files/format_converter.py | 58 +++++++++++++++++++++------ app/support_files/rss_reader.py | 3 +- requirements.txt | 1 + 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index b777be9..a7fd423 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1 +1 @@ -__version__ = "4.0" +__version__ = "5.0" diff --git a/app/support_files/args_parser.py b/app/support_files/args_parser.py index 2feb468..bf1d46f 100644 --- a/app/support_files/args_parser.py +++ b/app/support_files/args_parser.py @@ -23,6 +23,7 @@ def get_args() -> Namespace: default=None) parser.add_argument("--to_html", metavar="PATH", type=str, help="Save news by path in html format", default=None) parser.add_argument("--to_fb2", metavar="PATH", type=str, help="Save news by path in fb2 format", default=None) + parser.add_argument("--colorize", action="store_true", help="Make console text display colorful", default=False) parser.parse_args() args = parser.parse_args() return args diff --git a/app/support_files/format_converter.py b/app/support_files/format_converter.py index a15461a..8d33e33 100644 --- a/app/support_files/format_converter.py +++ b/app/support_files/format_converter.py @@ -10,6 +10,7 @@ from time import strftime, altzone, mktime, localtime, ctime, time, struct_time import urllib3 +from colored import fg, bg, attr from app.support_files.dtos import Feed from app.support_files.rss_parser import Parser @@ -36,6 +37,19 @@ def get_templates(template_type: str, template_names: List[str]) -> List[str]: return templates +def set_length(str_len: str): + """ + Makes text the same length wide, + """ + def decorator(func): + def wrapper(*args, **kwargs): + text = textwrap.fill(args[0], width=str_len) + result = func(text, *args[1:], **kwargs) + return result + return wrapper + return decorator + + def get_img_by_url(url: str) -> str: """ Gets img in base64 format by url. @@ -57,34 +71,51 @@ def __init__(self, feeds: List[Feed]) -> None: self.__feeds = feeds self._logger = get_logger(APP_NAME) - def to_console_format(self, str_len: int = 80) -> str: + def to_console_format(self, str_len: int = 80, col_en: bool = False) -> str: """ Convert data to console format. + :param col_en: Enable colorizing, if true. :param str_len: Length of output strings. :return: Converted data. """ + @set_length(str_len) + def set_color(text: str, text_color: str, enabled: bool) -> str: + """ + Changes the text_color of the text if enabled is true. + :return: Colorized text. + """ + color_str = "" + reset = "" + if enabled: + color_str = f"{fg(text_color)}" + reset = f" {attr('reset')}" + return color_str + text + reset strings = [] - out_separator = "*" * str_len - in_separator = "-" * str_len + out_separator = set_color(f"{'*' * str_len}", "green_4", col_en) + in_separator = set_color(f"{'-' * str_len}", "chartreuse_3b", col_en) for feed in self.__feeds: strings.append(out_separator) - strings.append(f"Feed: {feed.title}") + strings.append(set_color(f"Feed: {feed.title}", "gold_1", col_en)) for item in feed.items: strings.append(in_separator) - strings.append(f"Author: {item.author}") - strings.append(f"Published: {convert_date(item.published_parsed)}") + strings.append(set_color(f"Author: {item.author}", "light_green_3", col_en)) + strings.append(set_color(f"Published: {convert_date(item.published_parsed)}", "light_cyan_1", col_en)) + strings.append("\n") + strings.append(set_color("Title:", "yellow_3a", col_en)) + strings.append(set_color(f"\t{item.title}", "gold_1", col_en)) strings.append("\n") - strings.append(f"Title: {item.title}") - strings.append(f"Description: {item.description}") + strings.append(set_color("Description:", "yellow_3a", col_en)) + strings.append(set_color(f"\t{item.description}", "light_green_3", col_en)) strings.append("\n") - strings.append(f"Link: {item.link}") - strings.append("Image links:") + strings.append(set_color("Link:", "yellow_3a", col_en)) + strings.append(set_color(f"\t{item.link}", "wheat_1", col_en)) + strings.append(set_color("Image links:", "yellow_3a", col_en)) for img_link in item.img_links: - strings.append(f"{img_link}") + strings.append(set_color(f"\t{img_link}", "light_cyan_1", col_en)) strings.append(in_separator) strings.append(out_separator) - strings = map(lambda s: textwrap.fill(s, width=str_len) + "\n", strings) + strings = map(lambda s: s + "\n", strings) result_string = functools.reduce(lambda a, b: a + b, strings) @@ -170,4 +201,5 @@ def to_fb2_format(self) -> str: if __name__ == "__main__": - print(Converter([Parser("https://news.yahoo.com/rss/").parse_feed(items_limit=3)]).to_html_format()) + print(Converter([Parser("https://news.yahoo.com/rss/").parse_feed(items_limit=1)]). + to_console_format(col_en=True, str_len=120)) diff --git a/app/support_files/rss_reader.py b/app/support_files/rss_reader.py index 2332c5f..3e58066 100644 --- a/app/support_files/rss_reader.py +++ b/app/support_files/rss_reader.py @@ -34,6 +34,7 @@ def exec_console_args() -> None: to_json = _args.json to_html_path = _args.to_html to_fb2_path = _args.to_fb2 + colorize = _args.colorize if limit < 1 and limit != -1: print("The limit must be -1 or greater than 0") @@ -98,7 +99,7 @@ def exec_console_args() -> None: print(converter.to_json_format(str_len=len_each_line)) else: logger.info("Data is converted to console format and printing is started") - print(converter.to_console_format(str_len=len_each_line)) + print(converter.to_console_format(str_len=len_each_line, col_en=colorize)) logger.info("Printing is finished") return None diff --git a/requirements.txt b/requirements.txt index 5f50945..cb9a5fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +colored==1.4.0 urllib3==1.25.7 feedparser==5.2.1 beautifulsoup4==4.8.1 From 4d0f72c04f154edd4cfb7c34839dd492de75d31d Mon Sep 17 00:00:00 2001 From: Zviger Date: Sun, 1 Dec 2019 12:02:58 +0300 Subject: [PATCH 19/21] Tests are added --- README.md | 9 +++- app/support_files/app_logger.py | 2 +- app/support_files/args_parser.py | 2 +- app/support_files/db_manager.py | 20 ++++---- app/support_files/exeptions.py | 7 +++ app/support_files/file_manager.py | 2 +- app/support_files/format_converter.py | 14 +++--- app/support_files/rss_parser.py | 7 ++- app/support_files/rss_reader.py | 8 +++- requirements.txt | 4 +- tests/__init__.py | 0 tests/test_args_parser.py | 69 +++++++++++++++++++++++++++ 12 files changed, 119 insertions(+), 25 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test_args_parser.py diff --git a/README.md b/README.md index c6bf93c..810c853 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,15 @@ Clone this repository and run ```text pip install . --user ``` +Also, you must have installed and running MongoDB. +Run +```text +service mongod status +``` +to make sure that Mongodb is running. ### User interface ```text -usage: rss-reader [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] [--date DATE] [--to_html PATH] [--to_fb2 PATH] source +usage: rss-reader [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] [--date DATE] [--to_html PATH] [--to_fb2 PATH] [--colorize] source positional arguments: source RSS URL @@ -22,6 +28,7 @@ optional arguments: --date DATE Search past news by date in format yeardaymonth (19991311) --to_html PATH Save news by path in html format --to_fb2 PATH Save news by path in fb2 format + --colorize Make console text display colorful ``` ### Json structure diff --git a/app/support_files/app_logger.py b/app/support_files/app_logger.py index ad6e31e..5a5e6c8 100644 --- a/app/support_files/app_logger.py +++ b/app/support_files/app_logger.py @@ -2,8 +2,8 @@ This module provides functions to work with logging. """ import logging -from logging import Logger import sys +from logging import Logger def init_logger(name: str) -> Logger: diff --git a/app/support_files/args_parser.py b/app/support_files/args_parser.py index bf1d46f..6740969 100644 --- a/app/support_files/args_parser.py +++ b/app/support_files/args_parser.py @@ -15,7 +15,7 @@ def get_args() -> Namespace: parser = argparse.ArgumentParser(description="It is a python command-line rss reader") parser.add_argument("source", help="RSS URL") parser.add_argument("--version", action="version", version=f"%(prog)s {app.__version__}", help="Print version info") - parser.add_argument("-l", "--limit", type=int, help="Limit news topics if this parameter provided", default=-1) + parser.add_argument("-l", "--limit", type=int, help="Limit news topics if this parameter provided", default=None) parser.add_argument("--verbose", action="store_true", help="Print result as JSON in stdout", default=False) parser.add_argument("--json", action="store_true", help="Outputs verbose status messages", default=False) parser.add_argument("--length", type=int, help="Sets the length of each line of news output", default=120) diff --git a/app/support_files/db_manager.py b/app/support_files/db_manager.py index ee037f9..b733d24 100644 --- a/app/support_files/db_manager.py +++ b/app/support_files/db_manager.py @@ -3,15 +3,15 @@ """ import os from dataclasses import asdict -from typing import Optional from time import strptime, mktime, altzone, localtime, struct_time +from typing import Optional -from pymongo import MongoClient +from pymongo import MongoClient, errors -from app.support_files.dtos import Feed, Item -from app.support_files.exeptions import FindFeedError, DateError -from app.support_files.config import APP_NAME from app.support_files.app_logger import get_logger +from app.support_files.config import APP_NAME +from app.support_files.dtos import Feed, Item +from app.support_files.exeptions import FindFeedError, DateError, DBConnectError class DBManager: @@ -22,6 +22,10 @@ class DBManager: def __init__(self) -> None: mongo_host = os.getenv('MONGO_HOST', '127.0.0.1') client = MongoClient(f"mongodb://{mongo_host}:27017/") + try: + client.server_info() + except errors.ServerSelectionTimeoutError: + raise DBConnectError(f"Can't connect to database with host - {mongo_host} and port - 27017") self._db = client["feed_db"] self._collection = self._db["feed_collection"] self._logger = get_logger(APP_NAME) @@ -79,8 +83,8 @@ def find_feed_by_link_and_date(self, link: str, date: str, limit: int = -1) -> F result_items = [] count = limit for item in feed.items: - i_date = struct_time(item.published_parsed) - l_i_date = localtime(mktime(tuple(i_date)) - altzone) + item_date = struct_time(item.published_parsed) + l_i_date = localtime(mktime(tuple(item_date)) - altzone) if (l_i_date.tm_year, l_i_date.tm_mon, l_i_date.tm_mday) == (date.tm_year, date.tm_mon, date.tm_mday): result_items.append(item) count -= 1 @@ -97,4 +101,4 @@ def truncate_collection(self) -> None: if __name__ == "__main__": - DBManager().truncate_collection() + DBManager() diff --git a/app/support_files/exeptions.py b/app/support_files/exeptions.py index c7b5a74..76bf698 100644 --- a/app/support_files/exeptions.py +++ b/app/support_files/exeptions.py @@ -29,3 +29,10 @@ class DirExistsError(Exception): This class should be raised, if directory which was received by bath not exists. """ pass + + +class DBConnectError(Exception): + """ + This class should be raised, if received some problems with connection with database. + """ + pass diff --git a/app/support_files/file_manager.py b/app/support_files/file_manager.py index 5ac4efe..1d6fcfc 100644 --- a/app/support_files/file_manager.py +++ b/app/support_files/file_manager.py @@ -1,8 +1,8 @@ """ This module contains functions to work with files. """ -from re import findall from pathlib import Path +from re import findall from app.support_files.exeptions import DirError, DirExistsError diff --git a/app/support_files/format_converter.py b/app/support_files/format_converter.py index 8d33e33..4be6600 100644 --- a/app/support_files/format_converter.py +++ b/app/support_files/format_converter.py @@ -2,20 +2,20 @@ This module contains class for converting parsed data from RSS. """ import base64 -import textwrap +import dataclasses import functools import json -import dataclasses +import textwrap from typing import List from time import strftime, altzone, mktime, localtime, ctime, time, struct_time import urllib3 -from colored import fg, bg, attr +from colored import fg, attr +from app.support_files.app_logger import get_logger +from app.support_files.config import APP_NAME from app.support_files.dtos import Feed from app.support_files.rss_parser import Parser -from app.support_files.config import APP_NAME -from app.support_files.app_logger import get_logger def convert_date(date: struct_time) -> str: @@ -43,7 +43,7 @@ def set_length(str_len: str): """ def decorator(func): def wrapper(*args, **kwargs): - text = textwrap.fill(args[0], width=str_len) + text = textwrap.fill(args[0], width=str_len) # first argument mast be str result = func(text, *args[1:], **kwargs) return result return wrapper @@ -115,7 +115,7 @@ def set_color(text: str, text_color: str, enabled: bool) -> str: strings.append(in_separator) strings.append(out_separator) - strings = map(lambda s: s + "\n", strings) + strings = "\n".join(strings) result_string = functools.reduce(lambda a, b: a + b, strings) diff --git a/app/support_files/rss_parser.py b/app/support_files/rss_parser.py index 4cf5032..c7ae265 100644 --- a/app/support_files/rss_parser.py +++ b/app/support_files/rss_parser.py @@ -53,8 +53,11 @@ def parse_feed(self, items_limit: int = -1) -> Feed: feed = data.get("feed", {}) feed_data = apply_field_mapping(FEED_FIELD_MAPPING, feed) feed_data["rss_link"] = self.url + entries = data.get("entries", []) + if items_limit > 0: + entries = entries[:items_limit] items_data = [apply_field_mapping(ITEM_FIELD_MAPPING, item) - for item in data.get("entries", [])[:items_limit]] + for item in entries] for item_data in items_data: soup = BeautifulSoup(item_data.get("description", ""), 'html.parser') item_data["description"] = soup.text @@ -68,4 +71,4 @@ def parse_feed(self, items_limit: int = -1) -> Feed: if __name__ == "__main__": parser = Parser("http://www.bbc.co.uk/music/genres/classical/reviews.rss") - print(parser.parse_feed(2)) + print(parser.parse_feed(1)) diff --git a/app/support_files/rss_reader.py b/app/support_files/rss_reader.py index 3e58066..0a21dd3 100644 --- a/app/support_files/rss_reader.py +++ b/app/support_files/rss_reader.py @@ -8,8 +8,8 @@ app_logger, db_manager, exeptions) -from app.support_files.file_manager import store_str_to_file from app.support_files.config import APP_NAME +from app.support_files.file_manager import store_str_to_file class Reader: @@ -41,7 +41,11 @@ def exec_console_args() -> None: return None logger.info("Connecting with database") - db = db_manager.DBManager() + try: + db = db_manager.DBManager() + except exeptions.DBConnectError as err: + print(err) + return None if _args.date is None: try: feed = parser.parse_feed(limit) diff --git a/requirements.txt b/requirements.txt index cb9a5fa..68ccbd6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -colored==1.4.0 +colored==1.4.1 urllib3==1.25.7 feedparser==5.2.1 beautifulsoup4==4.8.1 -pymongo==3.9.0 +pymongo==3.9.0 \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_args_parser.py b/tests/test_args_parser.py new file mode 100644 index 0000000..045140d --- /dev/null +++ b/tests/test_args_parser.py @@ -0,0 +1,69 @@ +import unittest +from unittest import TestCase +from unittest.mock import patch, Mock +import time + +from app.support_files.rss_parser import Parser +from app.support_files.dtos import Item, Feed + + +class TestArgsParser(TestCase): + + def setUp(self): + self.parser = Parser("rss_link") + + def test_parsing_method(self): + with patch("feedparser.parse") as parse_mock: + def mock_get(key, default=None): + mock_dict = {"feed": {"title": "feed_title", + "link": "feed_link"}, + "entries": + [{"title": "item_title", + "link": "item_link", + "author": "item_author", + "published_parsed": time.struct_time((2019, 11, 30, 13, 45, 0, 5, 334, 0)), + "description": "item_description", + "media_content": []}]} + + return mock_dict.get(key, default) + + attrs = {"bozo": 0, + "status": 200, + "get": mock_get} + parse_object_mock = Mock() + parse_object_mock.configure_mock(**attrs) + parse_mock.return_value = parse_object_mock + test_feed = Feed(rss_link="rss_link", + title="feed_title", + link="feed_link", + items=[Item(title="item_title", + link="item_link", + author="item_author", + published_parsed=time.struct_time((2019, 11, 30, 13, 45, 0, 5, 334, 0)), + description="item_description", + img_links=[])]) + self.assertEqual(test_feed, self.parser.parse_feed()) + + def test_parsing_method_exception_1(self): + with patch("feedparser.parse") as parse_mock: + attrs = {"bozo": 1, + "status": 200} + parse_object_mock = Mock() + parse_object_mock.configure_mock(**attrs) + parse_mock.return_value = parse_object_mock + with self.assertRaises(ConnectionError): + self.parser.parse_feed() + + def test_parsing_method_exception_2(self): + with patch("feedparser.parse") as parse_mock: + attrs = {"bozo": 0, + "status": 404} + parse_object_mock = Mock() + parse_object_mock.configure_mock(**attrs) + parse_mock.return_value = parse_object_mock + with self.assertRaises(ConnectionError): + self.parser.parse_feed() + + +if __name__ == "__main__": + unittest.main() From 68174e9d2608d3d47f4968d8307100dfce1e3e9c Mon Sep 17 00:00:00 2001 From: Zviger Date: Sun, 1 Dec 2019 12:34:16 +0300 Subject: [PATCH 20/21] Fixed bag with '--limit' argument --- app/support_files/args_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/support_files/args_parser.py b/app/support_files/args_parser.py index 6740969..bf1d46f 100644 --- a/app/support_files/args_parser.py +++ b/app/support_files/args_parser.py @@ -15,7 +15,7 @@ def get_args() -> Namespace: parser = argparse.ArgumentParser(description="It is a python command-line rss reader") parser.add_argument("source", help="RSS URL") parser.add_argument("--version", action="version", version=f"%(prog)s {app.__version__}", help="Print version info") - parser.add_argument("-l", "--limit", type=int, help="Limit news topics if this parameter provided", default=None) + parser.add_argument("-l", "--limit", type=int, help="Limit news topics if this parameter provided", default=-1) parser.add_argument("--verbose", action="store_true", help="Print result as JSON in stdout", default=False) parser.add_argument("--json", action="store_true", help="Outputs verbose status messages", default=False) parser.add_argument("--length", type=int, help="Sets the length of each line of news output", default=120) From 13dc68694c2e56927fa56ca3c7cc6593614d623d Mon Sep 17 00:00:00 2001 From: Zviger Date: Fri, 6 Dec 2019 01:17:42 +0300 Subject: [PATCH 21/21] MongoDB is dockerized MongoDB now work in Docker container. Normal working with static files is added --- MANIFEST.in | 2 ++ README.md | 51 ++++++++++++++++++++++++--- app/__init__.py | 2 +- app/support_files/format_converter.py | 7 ++-- docker-compose.yml | 6 ++++ setup.py | 1 + 6 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 MANIFEST.in create mode 100644 docker-compose.yml diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..fe036a6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include app/support_files/templates/fb2/* +include app/support_files/templates/html/* \ No newline at end of file diff --git a/README.md b/README.md index 810c853..0f7143a 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,58 @@ ## It is a one-shot command-line RSS reader by Zviger. ### Installation -Clone this repository and run +Install [Python3.8](https://www.python.org/downloads/) + +Install [pip](https://pip.pypa.io/en/stable/installing/) + +Install GIT. +This [link](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) +may be useful in this matter. +Clone this repository in the folder you need using this command +```text +git clone https://github.com/Zviger/PythonHomework +``` +after change the branch to the project branch and run +```text +git checkout final_proj +``` +Now you can install the application itself using this command from the application folder ```text pip install . --user ``` -Also, you must have installed and running MongoDB. +You will also need MongoDB. You can install and run +MongoDB on your system using this [link](https://docs.mongodb.com/manual/installation/). +But you can install [Docker](https://docs.docker.com/install/) and +[Docker-Compose](https://docs.docker.com/compose/install/). + + +To start the container with MongoDB, run the following command in the application folder +```text +docker-compose up +``` +* The application and database are connected through port 27017. + +You can stop container with MongoDB by command +```text +docker-compose stop +``` +and run again +```text +docker-compose start +``` +You can execute the command +```text +docker-compose down +``` +but you will lose all saved data from the database. + + +Congratulations! + Run ```text -service mongod status +rss-reader --help ``` -to make sure that Mongodb is running. +to learn about the features of the application and start using it. ### User interface ```text usage: rss-reader [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] [--date DATE] [--to_html PATH] [--to_fb2 PATH] [--colorize] source diff --git a/app/__init__.py b/app/__init__.py index a7fd423..07f3d47 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1 +1 @@ -__version__ = "5.0" +__version__ = "5.1" diff --git a/app/support_files/format_converter.py b/app/support_files/format_converter.py index 4be6600..19c9b58 100644 --- a/app/support_files/format_converter.py +++ b/app/support_files/format_converter.py @@ -7,11 +7,13 @@ import json import textwrap from typing import List +from pathlib import Path from time import strftime, altzone, mktime, localtime, ctime, time, struct_time import urllib3 from colored import fg, attr +import app from app.support_files.app_logger import get_logger from app.support_files.config import APP_NAME from app.support_files.dtos import Feed @@ -31,8 +33,9 @@ def get_templates(template_type: str, template_names: List[str]) -> List[str]: Reads templates from files. """ templates = [] + app_path = Path(app.__path__[0]).joinpath() for template_name in template_names: - with open(f"app/support_files/templates/{template_type}/{template_name}", "r") as main_file: + with open(app_path.joinpath(f"support_files/templates/{template_type}/{template_name}"), "r") as main_file: templates.append(main_file.read()) return templates @@ -128,7 +131,7 @@ def to_json_format(self, str_len: int = 80) -> str: :return: Converted data. """ dicts_of_feeds = list(map(dataclasses.asdict, self.__feeds)) - return textwrap.fill(json.dumps(dicts_of_feeds), width=str_len) + return textwrap.fill(json.dumps(dicts_of_feeds, ensure_ascii=False), width=str_len) def to_html_format(self) -> str: """ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..61356b9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,6 @@ +version: '3.6' +services: + mongo: + image: mongo + ports: + - "27017:27017" \ No newline at end of file diff --git a/setup.py b/setup.py index 966519d..c26f628 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ packages=setuptools.find_packages(), python_requires=">=3.8", install_requires=install_requires, + include_package_data=True, entry_points={ "console_scripts": [f"{APP_NAME}=app.core:main"], }