Remove image downloading

The special casing of Wordpress and image downloading was not reliable
for me so I have removed it, and tried to simplify the code in the
process. If you still need this functionality you will want to pin v0.2.1.

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Ed Summers
+Copyright (c) Ed Summers
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,6 +1,6 @@
 *feediverse* will read RSS/Atom feeds and send the messages as Mastodon posts.
-Please use responsibly! *feediverse* is kind of the same thing as [feed2toot]
-but it's just one module that works with Python 3 ... and I was bored.
+It's meant to add a little bit of spice to your timeline from other places.
+Please use it responsibly.
 
 ## Install
 
@@ -18,6 +18,8 @@ Once *feediverse* is configured you can add it to your crontab:
 
     */15 * * * * /usr/local/bin/feediverse    
 
+Run `feediverse --help` to show the command line options.
+
 ## Post Format
 
 You can customize the post format by opening the configuration file (default is
@@ -32,7 +34,11 @@ like so:
     Bookmark: {title} {url} {summary}
 
 `{hashtags}` will look for tags in the feed entry and turn them into a space
-separated list of hashtags.
+separated list of hashtags. For some feeds (e.g. youtube-rss) you should use `{link}` instead of `{url}`.
+
+`{content}` is the whole content of the feed entry (with html-tags
+stripped). Please be aware that this might easily exceed Mastodon's
+limit of 512 characters.
 
 ## Multiple Feeds
 
@@ -45,20 +51,3 @@ Since *feeds* is a list you can add additional feeds to watch if you want.
       - url: https://example.org/feed/
         template: "dot org: {title} {url}"
 
-## Why?
-
-I created *feediverse* because I wanted to send my Pinboard bookmarks to
-Mastodon.  I've got an IFTTT recipe that does this for Twitter, but IFTTT
-doesn't appear to work with Mastodon yet. That being said *feediverse* should
-work with any RSS or Atom feed (thanks to [feedparser]).
-
-## Warning!
-
-Please be responsible. Don't fill up Mastodon with tons of junk just because you
-can. That kind of toxic behavior is why a lot of people are trying to establish
-other forms of social media like Mastodon.
-
-[feed2toot]: https://gitlab.com/chaica/feed2toot/
-[feedparser]: http://feedparser.org/
-
-

feediverse.py

@@ -1,16 +1,36 @@
 #!/usr/bin/env python3
 
 import os
+import re
 import sys
 import yaml
+import argparse
 import dateutil
 import feedparser
 
+from bs4 import BeautifulSoup
 from mastodon import Mastodon
-from datetime import datetime, timezone
+from datetime import datetime, timezone, MINYEAR
+
+DEFAULT_CONFIG_FILE = os.path.join("~", ".feediverse")
 
 def main():
-    config_file = get_config_file()
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-n", "--dry-run", action="store_true",
+                        help=("perform a trial run with no changes made: "
+                              "don't toot, don't save config"))
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="be verbose")
+    parser.add_argument("-c", "--config",
+                        help="config file to use",
+                        default=os.path.expanduser(DEFAULT_CONFIG_FILE))
+
+    args = parser.parse_args()
+    config_file = args.config
+
+    if args.verbose:
+        print("using config file", config_file)
+
     if not os.path.isfile(config_file):
         setup(config_file)        
 
@@ -23,62 +43,101 @@ def main():
         access_token=config['access_token']
     )
 
+    newest_post = config['updated']
     for feed in config['feeds']:
+        if args.verbose:
+            print(f"fetching {feed['url']} entries since {config['updated']}")
         for entry in get_feed(feed['url'], config['updated']):
-            masto.status_post(feed['template'].format(**entry))
+            newest_post = max(newest_post, entry['updated'])
+            if args.verbose:
+                print(entry)
+            if args.dry_run:
+                print("trial run, not tooting ", entry["title"][:50])
+                continue
+            masto.status_post(feed['template'].format(**entry)[:499])
 
-    save_config(config, config_file)
-
-def get_config_file():
-    if __name__ == "__main__" and len(sys.argv) > 1:
-        config_file = sys.argv[1]
-    else:
-        config_file = os.path.join(os.path.expanduser("~"), ".feediverse")
-    return config_file
-
-def save_config(config, config_file):
-    copy = dict(config)
-    copy['updated'] = datetime.now(tz=timezone.utc).isoformat()
-    with open(config_file, 'w') as fh:
-        fh.write(yaml.dump(copy, default_flow_style=False))
-
-def read_config(config_file):
-    config = {}
-    with open(config_file) as fh:
-        config = yaml.load(fh)
-        if 'updated' in config:
-            config['updated'] = dateutil.parser.parse(config['updated'])
-        else:
-            config['updated'] = datetime.now(tz=timezone.utc)
-    return config
+    if not args.dry_run:
+        config['updated'] = newest_post.isoformat()
+        save_config(config, config_file)
 
 def get_feed(feed_url, last_update):
-    new_entries = 0
     feed = feedparser.parse(feed_url)
-    for entry in feed.entries:
-        e = get_entry(entry)
-        if last_update is None or e['updated'] > last_update:
-            new_entries += 1
-            yield e
-    return new_entries
+    if last_update:
+        entries = [e for e in feed.entries
+                   if dateutil.parser.parse(e['updated']) > last_update]
+    else:
+        entries = feed.entries
+    entries.sort(key=lambda e: e.updated_parsed)
+    for entry in entries:
+        yield get_entry(entry)
 
 def get_entry(entry):
     hashtags = []
     for tag in entry.get('tags', []):
-        for t in tag['term'].split(' '):
-            hashtags.append('#{}'.format(t))
+        t = tag['term'].replace(' ', '_').replace('.', '').replace('-', '')
+        hashtags.append('#{}'.format(t))
+    summary = entry.get('summary', '')
+    content = entry.get('content', '') or ''
+    if content:
+        content = cleanup(content[0].get('value', ''))
+    url = entry.id
     return {
-        'url': entry.id,
-        'title': entry.title,
-        'summary': entry.get('summary', ''),
+        'url': url,
+        'link': entry.link,
+        'title': cleanup(entry.title),
+        'summary': cleanup(summary),
+        'content': content,
         'hashtags': ' '.join(hashtags),
-        'updated': dateutil.parser.parse(entry['updated']),
+        'updated': dateutil.parser.parse(entry['updated'])
     }
 
+def cleanup(text):
+    html = BeautifulSoup(text, 'html.parser')
+    text = html.get_text()
+    text = re.sub('\xa0+', ' ', text)
+    text = re.sub('  +', ' ', text)
+    text = re.sub(' +\n', '\n', text)
+    text = re.sub('\n\n\n+', '\n\n', text, flags=re.M)
+    return text.strip()
+
+def find_urls(html):
+    if not html:
+        return
+    urls = []
+    soup = BeautifulSoup(html, 'html.parser')
+    for tag in soup.find_all(["a", "img"]):
+        if tag.name == "a":
+            url = tag.get("href")
+        elif tag.name == "img":
+            url = tag.get("src")
+        if url and url not in urls:
+            urls.append(url)
+    return urls
+
+def yes_no(question):
+    res = input(question + ' [y/n] ')
+    return res.lower() in "y1"
+
+def save_config(config, config_file):
+    copy = dict(config)
+    with open(config_file, 'w') as fh:
+        fh.write(yaml.dump(copy, default_flow_style=False))
+
+def read_config(config_file):
+    config = {
+        'updated': datetime(MINYEAR, 1, 1, 0, 0, 0, 0, timezone.utc)
+    }
+    with open(config_file) as fh:
+        cfg = yaml.load(fh, yaml.SafeLoader)
+        if 'updated' in cfg:
+            cfg['updated'] = dateutil.parser.parse(cfg['updated'])
+    config.update(cfg)
+    return config
+
 def setup(config_file):
     url = input('What is your Mastodon Instance URL? ')
-    have_app = input('Do you have your app credentials already? [y/n] ')
-    if have_app.lower() == 'y':
+    have_app = yes_no('Do you have your app credentials already?')
+    if have_app:
         name = 'feediverse'
         client_id = input('What is your app\'s client id: ')
         client_secret = input('What is your client secret: ')
@@ -98,6 +157,7 @@ def setup(config_file):
         access_token = m.log_in(username, password)
 
     feed_url = input('RSS/Atom feed URL to watch: ')
+    old_posts = yes_no('Shall already existing entries be tooted, too?')
     config = {
         'name': name,
         'url': url,
@@ -108,6 +168,8 @@ def setup(config_file):
             {'url': feed_url, 'template': '{title} {url}'}
         ]
     }
+    if not old_posts:
+        config['updated'] = datetime.now(tz=timezone.utc).isoformat()
     save_config(config, config_file)
     print("")
     print("Your feediverse configuration has been saved to {}".format(config_file))

setup.py

@@ -5,8 +5,8 @@ with open("README.md") as f:
 
 setup(
     name='feediverse',
-    version='0.0.9',
-    python_requires='>=2.7',
+    version='0.3.0',
+    python_requires='>=3.3',
     url='https://github.com/edsu/feediverse',
     author='Ed Summers',
     author_email='ehs@pobox.com',
@@ -14,6 +14,10 @@ setup(
     description='Connect an RSS Feed to Mastodon',
     long_description=long_description,
     long_description_content_type="text/markdown",
-    install_requires=['feedparser', 'mastodon.py', 'python-dateutil', 'pyyaml'],
+    install_requires=['beautifulsoup4',
+                      'feedparser',
+                      'mastodon.py',
+                      'python-dateutil',
+                      'pyyaml'],
     entry_points={'console_scripts': ['feediverse = feediverse:main']}
 )