pax_global_header 0000666 0000000 0000000 00000000064 13514067645 0014525 g ustar 00root root 0000000 0000000 52 comment=6d52c0c92f2617476fe0b131c1eebba68676b2fb
woob-6d52c0c92f2617476fe0b131c1eebba68676b2fb-modules-tumblr/ 0000775 0000000 0000000 00000000000 13514067645 0022520 5 ustar 00root root 0000000 0000000 woob-6d52c0c92f2617476fe0b131c1eebba68676b2fb-modules-tumblr/modules/ 0000775 0000000 0000000 00000000000 13514067645 0024170 5 ustar 00root root 0000000 0000000 woob-6d52c0c92f2617476fe0b131c1eebba68676b2fb-modules-tumblr/modules/tumblr/ 0000775 0000000 0000000 00000000000 13514067645 0025475 5 ustar 00root root 0000000 0000000 woob-6d52c0c92f2617476fe0b131c1eebba68676b2fb-modules-tumblr/modules/tumblr/__init__.py 0000664 0000000 0000000 00000001560 13514067645 0027610 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2017 Vincent A
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this weboob module. If not, see .
from __future__ import unicode_literals
from .module import TumblrModule
__all__ = ['TumblrModule']
woob-6d52c0c92f2617476fe0b131c1eebba68676b2fb-modules-tumblr/modules/tumblr/browser.py 0000664 0000000 0000000 00000010510 13514067645 0027527 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2017 Vincent A
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this weboob module. If not, see .
from __future__ import unicode_literals
from datetime import datetime
import re
from weboob.tools.json import json
from weboob.browser.browsers import APIBrowser
from weboob.browser.filters.standard import CleanText
from weboob.capabilities.gallery import BaseImage
from weboob.capabilities.image import Thumbnail
class TumblrBrowser(APIBrowser):
def __init__(self, baseurl, *args, **kwargs):
super(TumblrBrowser, self).__init__(*args, **kwargs)
self.BASEURL = baseurl
def consent(self):
response = self.open(self.BASEURL)
html = response.text
token = re.search(r'name="tumblr-form-key".*?content="([^"]*)"', html).group(1)
data = {
"eu_resident": False, # i don't want to live on this planet anymore
"gdpr_is_acceptable_age": True,
"gdpr_consent_core": True,
"gdpr_consent_first_party_ads": True,
"gdpr_consent_third_party_ads": True,
"gdpr_consent_search_history": True,
"redirect_to": self.BASEURL,
}
headers = {
'X-tumblr-form-key': token,
'Referer': response.url,
}
super(TumblrBrowser, self).request('https://www.tumblr.com/svc/privacy/consent', data=data, headers=headers)
def request(self, *args, **kwargs):
def perform():
# JSONP
r = super(TumblrBrowser, self).open(*args, **kwargs).text
r = re.sub(r'^var tumblr_api_read = (.*);$', r'\1', r)
return json.loads(r)
try:
return perform()
except ValueError:
self.consent()
return perform()
def get_title_icon(self):
r = self.request('/api/read/json?type=photo&num=1&start=0&filter=text')
icon = None
if r['posts']:
icon = r['posts'][0]['tumblelog']['avatar_url_512']
return (r['tumblelog']['title'], icon)
def iter_images(self, gallery):
index = 0
offset = 0
step = 50
while True:
r = self.request('/api/read/json?type=photo&filter=text', params={'start': offset, 'num': step})
for post in r['posts']:
# main photo only if single
if not post['photos']:
img = BaseImage(
index=index,
gallery=gallery,
url=post['photo-url-1280'],
thumbnail=Thumbnail(post['photo-url-250']),
)
img.id = post['id']
index += 1
img.title = CleanText().filter(post['photo-caption'])
img.date = datetime.strptime(post['date-gmt'], '%Y-%m-%d %H:%M:%S %Z')
img._page_url = post["url"]
yield img
# if multiple
for photo in post['photos']:
img = BaseImage(
index=index,
gallery=gallery,
url=photo['photo-url-1280'],
thumbnail=Thumbnail(photo['photo-url-250']),
)
img.id = '%s.%s' % (post['id'], photo['offset'])
index += 1
img.title = CleanText().filter(photo['caption'] or post['photo-caption'])
img.date = datetime.strptime(post['date-gmt'], '%Y-%m-%d %H:%M:%S %Z')
img._page_url = post["url"]
yield img
offset += step
if not r['posts'] or offset >= r['posts-total']:
break
woob-6d52c0c92f2617476fe0b131c1eebba68676b2fb-modules-tumblr/modules/tumblr/favicon.png 0000664 0000000 0000000 00000001270 13514067645 0027630 0 ustar 00root root 0000000 0000000 PNG
IHDR >a bKGD V ul pHYs tIME
+9/MS iTXtComment Created with GIMPd.e IDATxAV@EaJ"IV 7q"WU:\_@ @ @ @ @ %|#cppBVW f h PO= PT ? $A 61yT[Jj+xmўYaLUYNJ)@`.1:Ziݣe [Ӌgz?J]?n'AA/@ uAh)쫯!F-@ . Fz ȝ$
m m )@ uPhH Z B
lY+ Z B Av`k&
Өօ==$=
Sr*p-QU
- |s @{F Px#NT i@46(hΚ @ @ @ @ PoǛE IENDB` woob-6d52c0c92f2617476fe0b131c1eebba68676b2fb-modules-tumblr/modules/tumblr/module.py 0000664 0000000 0000000 00000005144 13514067645 0027340 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2017 Vincent A
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this weboob module. If not, see .
from __future__ import unicode_literals
from weboob.browser.exceptions import ClientError, HTTPNotFound
from weboob.capabilities.gallery import CapGallery, BaseGallery, BaseImage, Thumbnail
from weboob.tools.backend import Module, BackendConfig
from weboob.tools.compat import urlparse
from weboob.tools.value import Value
from .browser import TumblrBrowser
__all__ = ['TumblrModule']
class TumblrModule(Module, CapGallery):
NAME = 'tumblr'
DESCRIPTION = 'images in tumblr blogs'
MAINTAINER = 'Vincent A'
EMAIL = 'dev@indigo.re'
LICENSE = 'AGPLv3+'
VERSION = '1.6'
CONFIG = BackendConfig(Value('url', label='URL of the tumblr', regexp='https?://.+'))
BROWSER = TumblrBrowser
def create_default_browser(self):
return self.create_browser(self.url())
def url(self):
return self.config['url'].get()
def get_gallery(self, _id):
title, icon = self.browser.get_title_icon()
if icon:
icon = Thumbnail(icon)
return BaseGallery(_id, title=title, url=self.url(), thumbnail=icon)
def search_galleries(self, pattern, sortby=CapGallery.SEARCH_RELEVANCE):
pattern = pattern.lower()
url = self.url()
if pattern in url or pattern in self.browser.get_title_icon()[0].lower():
yield self.get_gallery(urlparse(url).netloc)
def iter_gallery_images(self, gallery):
for img in self.browser.iter_images(gallery):
yield img
def fill_img(self, img, fields):
if 'data' in fields:
try:
img.data = self.browser.open(img.url).content
except (ClientError, HTTPNotFound):
img.data = b''
if 'thumbnail' in fields and img.thumbnail:
self.fill_img(img.thumbnail, ('data',))
OBJECTS = {
BaseImage: fill_img,
BaseGallery: fill_img,
}
woob-6d52c0c92f2617476fe0b131c1eebba68676b2fb-modules-tumblr/modules/tumblr/test.py 0000664 0000000 0000000 00000002355 13514067645 0027033 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2017 Vincent A
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this weboob module. If not, see .
from __future__ import unicode_literals
from weboob.tools.test import BackendTest, skip_without_config
class TumblrTest(BackendTest):
MODULE = 'tumblr'
@skip_without_config('url')
def test_tumblr(self):
gall = self.backend.get_gallery('noname')
assert gall
assert gall.url
for img, _ in zip(self.backend.iter_gallery_images(gall), range(10)):
assert img.url
self.backend.fillobj(img)
assert img.data