al_monitor.recipe

#!/usr/bin/env  python2
# -*- coding: utf-8 -*-
__license__ = 'GPL v3'
__copyright__ = '2014, spswerling'
'''
http://www.al-monitor.com/
'''
import string
import inspect
import datetime
import re
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup


class AlMonitor(BasicNewsRecipe):
    title = u'Al Monitor'
    __author__ = u'spswerling'
    description = 'a The Pulse of the Middle East'
    no_stylesheets = True
    encoding = 'utf-8'
    category = 'news'
    language = 'en'
    publication_type = 'newspaper'
    cover_img_url = 'http://www.al-monitor.com/modules/almcontent/a-img/elements/logo.png'
    masthead_url = cover_img_url
    remove_empty_feeds = True

    # on kindle, images can make things kind of fat. Slim them down.
    recursions = 0
    compress_news_images = True
    compress_news_images_max_size = 7
    scale_news_images = (150, 200)  # (kindle touch: 600x800)
    useHighResImages = False
    oldest_article = 1.5
    max_articles_per_section = 15

    sections = [
        (u'egypt', u'http://www.al-monitor.com/pulse/egypt-pulse'),
        (u'gulf', u'http://www.al-monitor.com/pulse/gulf-pulse'),
        (u'iran', u'http://www.al-monitor.com/pulse/iran-pulse'),
        (u'iraq', u'http://www.al-monitor.com/pulse/iraq-pulse'),
        (u'israel', u'http://www.al-monitor.com/pulse/israel-pulse'),
        (u'lebanon', u'http://www.al-monitor.com/pulse/lebanon-pulse'),
        (u'palistine', u'http://www.al-monitor.com/pulse/palistine-pulse'),
        (u'syria', u'http://www.al-monitor.com/pulse/syria-pulse'),
        (u'turkey', u'http://www.al-monitor.com/pulse/turkey-pulse'),
    ]

    # util for creating remove_tags and keep_tags style regex matchers
    def tag_matcher(elt, attr, rgx_str):
        return dict(name=elt, attrs={attr: re.compile(rgx_str, re.IGNORECASE)})

    remove_tags = [
        dict(attrs={'id': [
            'header',
            'pulsebanner',
            'relatedarticles',
            'sidecolumn',
            'disqus',
            'footer',
            'footer2',
            'footer3',
            'mobile-extras',
        ]}),
        tag_matcher('hr', 'id', 'spacer'),
        tag_matcher('a', 'title', 'print this article'),
        tag_matcher('div', 'class', 'extras'),
        tag_matcher('div', 'class', '^clear$'),
        tag_matcher('div', 'class', '^overlay$'),
        tag_matcher('div', 'class', 'shareTag'),
    ]

    articles = {}
    urls_done = []

    def parse_index(self):
        for section in self.sections:
            self.parse_section(section[0], section[1])
        ans = []
        for k in self.articles:
            ans.append((string.capwords(k), self.articles[k]))
        return ans

    def parse_section(self, section, url):

        self.articles[section] = []

        try:
            self._p('process section  ' + section + ', url: ' + url)
            soup = self.index_to_soup(url)
        except:
            self._p('Unable to spider section')
            return []

        self._p('Got section. Processing links.')

        for link in soup.findAll('a'):
            href = link.get('href')
            text = self.text(link)
            if text and ('pulse/originals' in href):
                self.process_link(section, link)

    def process_link(self, section, link):
        title = self.text(link)
        if len(title) > 120:
            title = title[0:120] + '...'
        href = link.get('href')
        if not href:
            self._p("BAD HREF: " + str(link))
            return
        self.queue_article_link(section, href, title)

    def queue_article_link(self, section, url, title):
        full_url = self.abs_url(url)
        if full_url in self.urls_done:
            self._p('Skip (already Qd): ' + ' - '.join([section, title, url]))
            return

        self._p('Q: ' + ' - '.join([section, title, url]))
        self.urls_done.append(full_url)
        if len(self.articles[section]) >= self.max_articles_per_section:
            return
        self.articles[section].append(
            dict(title=title,
                 url=full_url,
                 date='',
                 description='',
                 author='',
                 content=''))

    def preprocess_raw_html(self, raw_html, url):
        reason_to_skip = self.should_skip_article(BeautifulSoup(raw_html))
        if reason_to_skip:
            self._p('Skipping article: ' + reason_to_skip + ', ' + url)
            # Next line will show up as an error in the logs, but ignore, see
            #   http://www.mobileread.com/forums/showthread.php?p=2931136
            return None
        else:
            return super(self.__class__, self).preprocess_raw_html(raw_html, url)

    def populate_article_metadata(self, article, soup, first):
        summary_node = soup.find('div', {'id': 'summary'})
        if summary_node:
            summary = self.text(summary_node)
            self._p('Summary: ' + summary)
            article.text_summary = summary
        else:
            self._p('No summary')

    def should_skip_article(self, soup):
        date = self.scrape_article_date(soup)
        if not date:
            return False

        age = (datetime.datetime.now() - date).days
        if (age > self.oldest_article):
            return "too old"
        return False

    def scrape_article_date(self, soup):
        for span in soup.findAll('span'):
            txt = self.text(span)
            rgx = re.compile('Posted ([a-zA-Z]+ \d\d?, \d\d\d\d).*')
            hit = rgx.match(txt)
            if hit:
                return self.date_from_string(txt)

        return None

    def date_from_string(self, datestring):
        try:
            # eg: Posted September 17, 2014
            dt = datetime.datetime.strptime(datestring, "Posted %B %d, %Y")
        except:
            dt = None

        if dt:
            self._p('From string "' + datestring + '", datetime: ' + str(dt))
        else:
            self._p('Could not get datetime from ' + datestring)

        return dt

    def abs_url(self, url):
        if 'www.al-monitor.com' in url:
            abs_url = url
        elif url[0] == '/':
            abs_url = 'http://www.al-monitor.com' + url
        else:
            self._p('Not sure how to make abs_url: ' + url)
            raise

        if '#' in abs_url:
            abs_url = ''.join(abs_url.split('#')[0:-1])

        return abs_url

    def text(self, n):
        return self.tag_to_string(n).strip()

    def _dbg_soup_node(self, node):
        s = '   cls: ' + str(node.get('class')).strip() + \
            '  id: ' + str(node.get('id')).strip() + \
            '  role: ' + str(node.get('role')).strip() + \
            ' txt: ' + self.text(node)
        return s

    def _p(self, msg):
        curframe = inspect.currentframe()
        calframe = inspect.getouterframes(curframe, 2)
        calname = calframe[1][3].upper()
        print('[' + calname + '] ' + msg[0:100])