nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix factorization of comic readers

Browse source 
there is still a race condition during the selection of the comic reader
class; sometimes, the URL matches for mangatoshokan but for some reason
mangafox is choosen instead
This commit is contained in:
Noé Rubinstein 2011-08-30 10:18:26 +02:00 • committed by Romain Bignon
commit 076a12f4e5
7 changed files with 48 additions and 177 deletions
Download patch file Download diff file Expand all files Collapse all files

17
weboob/backends/genericcomicreader/backend.py
View file

@ -29,19 +29,19 @@ __all__ = ['GenericComicReaderBackend']
class DisplayPage(BasePage):
def get_page(self, gallery):
src = self.document.xpath(self.backend.IMG_SRC_XPATH)[0]
src = self.document.xpath(self.browser.params['img_src_xpath'])[0]
return BaseImage(src,
gallery=gallery,
url=src)
def page_list(self):
return self.document.xpath(self.backend.PAGE_LIST_XPATH)
return self.document.xpath(self.browser.params['page_list_xpath'])
class GenericComicReaderBrowser(BaseBrowser):
def __init__(self, *args, **kwargs):
self.PAGES = self.backend.PAGES
def __init__(self, browser_params, *args, **kwargs):
self.params = browser_params
BaseBrowser.__init__(self, *args, **kwargs)
def iter_gallery_images(self, gallery):
@ -49,7 +49,7 @@ class GenericComicReaderBrowser(BaseBrowser):
assert self.is_on_page(DisplayPage)
for p in self.page.page_list():
self.location(self.backend.PAGE_LOCATION % p)
self.location(self.params['page_to_location'] % p)
assert self.is_on_page(DisplayPage)
yield self.page.get_page(gallery)
@ -67,6 +67,11 @@ class GenericComicReaderBackend(BaseBackend, ICapGallery):
LICENSE = 'AGPLv3+'
BROWSER = GenericComicReaderBrowser
def create_default_browser(self):
b = self.create_browser(self.BROWSER_PARAMS)
b.PAGES = self.PAGES
return b
def iter_gallery_images(self, gallery):
with self.browser:
return self.browser.iter_gallery_images(gallery)
@ -82,7 +87,7 @@ class GenericComicReaderBackend(BaseBackend, ICapGallery):
else:
return None
gallery = BaseGallery(_id, url=(self.ID_TO_URL % _id))
with self.browser:
return gallery

6
weboob/backends/genericcomicreader/__init__.py → weboob/backends/mangafox/__init__.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
@ -17,6 +17,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .mangafox import MangafoxBackend
from .backend import MangafoxBackend
__all__ = ['MangafoxBackend', 'MangatoshokanBackend', 'SimplyreaditBackend']
__all__ = ['MangafoxBackend']

13
weboob/backends/genericcomicreader/mangafox.py → weboob/backends/mangafox/backend.py
View file

@ -17,17 +17,18 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import GenericComicReaderBackend, DisplayPage
from ..genericcomicreader.backend import GenericComicReaderBackend, DisplayPage
__all__ = ['MangafoxBackend']
class MangafoxBackend(GenericComicReaderBackend):
NAME = 'mangafox'
DESCRIPTION = 'Mangafox manga reading site'
IMG_SRC_XPATH = "//img[@id='image']/attribute::src"
PAGE_LIST_XPATH = "(//select[@onchange='change_page(this)'])[1]/option/@value"
PAGE_TO_LOCATION = "%s.html"
ID_TO_URL = 'http://www.mangafox.com/manga/%s'
ID_REGEXP = r'/?[^/]+/[^/]+(?:/[^/]+)?/?'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@id='image']/attribute::src",
page_list_xpath="(//select[@onchange='change_page(this)'])[1]/option/@value",
page_to_location="%s.html")
ID_REGEXP = r'[^/]+/[^/]+(?:/[^/]+)?'
URL_REGEXP = r'.+mangafox.com/manga/(%s).+' % ID_REGEXP
ID_TO_URL = 'http://www.mangafox.com/manga/%s'
PAGES = { r'http://.+\.mangafox.\w+/manga/[^/]+/[^/]+/([^/]+/)?.+\.html': DisplayPage }

16
weboob/backends/genericcomicreader/mangatoshokan.py → weboob/backends/mangatoshokan/__init__.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
@ -17,18 +17,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import GenericComicReaderBackend, DisplayPage
from .backend import MangatoshokanBackend
__all__ = ['MangatoshokanBackend']
class MangatoshokanBackend(GenericComicReaderBackend):
NAME = 'mangatoshokan'
DESCRIPTION = 'Mangatoshokan manga reading site'
DOMAIN = "www.mangatoshokan.com"
IMG_SRC_XPATH = "//img[@id='readerPage']/@src"
PAGE_LIST_XPATH = "(//select[@class='headerSelect'])[1]/option/@value"
PAGE_TO_LOCATION = 'http://%s%%s' % DOMAIN
ID_TO_URL = 'http://www.mangatoshokan.com/read/%s'
ID_REGEXP = r'[^/]+(?:/[^/]+)*'
URL_REGEXP = r'.+mangatoshokan.com/read/(%s).+' % ID_REGEXP
PAGES = { r'http://.+\.mangatoshokan.com/read/.+': DisplayPage }

80
weboob/backends/mangatoshokan/backend.py
View file

@ -17,75 +17,19 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
import re
from weboob.capabilities.gallery import ICapGallery, BaseGallery, BaseImage
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BaseBrowser, BasePage
from ..genericcomicreader.backend import GenericComicReaderBackend, DisplayPage
__all__ = ['MangatoshokanBackend']
class DisplayPage(BasePage):
def get_page(self, gallery):
src = self.document.xpath("//img[@id='readerPage']/@src")[0]
return BaseImage(src,
gallery=gallery,
url=src)
def page_list(self):
return self.document.xpath("(//select[@class='headerSelect'])[1]/option/@value")
class MangatoshokanBrowser(BaseBrowser):
DOMAIN = "www.mangatoshokan.com"
PAGES = { r'http://.+\.mangatoshokan.com/read/.+': DisplayPage }
def iter_gallery_images(self, gallery):
self.location(gallery.url)
assert self.is_on_page(DisplayPage)
for p in self.page.page_list():
self.location('http://%s%s' % (self.DOMAIN, p))
assert self.is_on_page(DisplayPage)
yield self.page.get_page(gallery)
def fill_image(self, image, fields):
if 'data' in fields:
image.data = self.readurl(image.url)
class MangatoshokanBackend(BaseBackend, ICapGallery):
class MangatoshokanBackend(GenericComicReaderBackend):
NAME = 'mangatoshokan'
MAINTAINER = 'Noé Rubinstein'
EMAIL = 'noe.rubinstein@gmail.com'
VERSION = '0.9'
DESCRIPTION = 'mangatoshokan.com'
LICENSE = 'AGPLv3+'
BROWSER = MangatoshokanBrowser
def iter_gallery_images(self, gallery):
with self.browser:
return self.browser.iter_gallery_images(gallery)
def get_gallery(self, _id):
match = re.match(r'(?:(?:.+mangatoshokan.com/read)?/)?([^/]+(?:/[^/]+)*)', _id)
if match is None:
return None
_id = match.group(1)
gallery = BaseGallery(_id, url=('http://www.mangatoshokan.com/read/%s' % _id))
with self.browser:
return gallery
def fill_gallery(self, gallery, fields):
gallery.title = gallery.id
def fill_image(self, image, fields):
with self.browser:
self.browser.fill_image(image, fields)
OBJECTS = {
BaseGallery: fill_gallery,
BaseImage: fill_image }
DESCRIPTION = 'Mangatoshokan manga reading site'
DOMAIN = "www.mangatoshokan.com"
BROWSER_PARAMS = dict(
img_src_xpath="//img[@id='readerPage']/@src",
page_list_xpath="(//select[@class='headerSelect'])[1]/option/@value",
page_to_location='http://%s%%s' % DOMAIN)
ID_TO_URL = 'http://www.mangatoshokan.com/read/%s'
ID_REGEXP = r'[^/]+(?:/[^/]+)*'
URL_REGEXP = r'.+mangatoshokan.com/read/(%s)' % ID_REGEXP
PAGES = { r'http://.+mangatoshokan.com/read/.+': DisplayPage }

15
weboob/backends/genericcomicreader/simplyreadit.py → weboob/backends/simplyreadit/__init__.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
@ -17,17 +17,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import GenericComicReaderBackend, DisplayPage
from .backend import SimplyreaditBackend
__all__ = ['SimplyreaditBackend']
class SimplyreaditBackend(GenericComicReaderBackend):
NAME = 'simplyreadit'
DESCRIPTION = 'Simplyreadit manga reading site'
IMG_SRC_XPATH = "//img[@class='open']/@src"
PAGE_LIST_XPATH = "(//div[contains(@class,'dropdown_right')]/ul[@class='dropdown'])[1]/li/a/@href"
PAGE_TO_LOCATION = "%s"
ID_TO_URL = 'http://www.simplyread.it/reader/read/%s'
ID_REGEXP = r'[^/]+(?:/[^/]+)*'
URL_REGEXP = r'.+symplyread.it/reader/read/(%s)' % ID_REGEXP
PAGES = { r'http://.+\.simplyread.it/reader/read/.+': DisplayPage }

78
weboob/backends/simplyreadit/backend.py
View file

@ -17,74 +17,18 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
import re
from weboob.capabilities.gallery import ICapGallery, BaseGallery, BaseImage
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BaseBrowser, BasePage
from ..genericcomicreader.backend import GenericComicReaderBackend, DisplayPage
__all__ = ['SimplyreaditBackend']
class DisplayPage(BasePage):
def get_page(self, gallery):
src = self.document.xpath("//img[@class='open']/@src")[0]
return BaseImage(src,
gallery=gallery,
url=src)
def page_list(self):
return self.document.xpath("(//div[contains(@class,'dropdown_right')]/ul[@class='dropdown'])[1]/li/a/@href")
class SimplyreaditBrowser(BaseBrowser):
PAGES = { r'http://.+\.simplyread.it/reader/read/.+': DisplayPage }
def iter_gallery_images(self, gallery):
self.location(gallery.url)
assert self.is_on_page(DisplayPage)
for p in self.page.page_list():
self.location(p)
assert self.is_on_page(DisplayPage)
yield self.page.get_page(gallery)
def fill_image(self, image, fields):
if 'data' in fields:
image.data = self.readurl(image.url)
class SimplyreaditBackend(BaseBackend, ICapGallery):
class SimplyreaditBackend(GenericComicReaderBackend):
NAME = 'simplyreadit'
MAINTAINER = 'Noé Rubinstein'
EMAIL = 'noe.rubinstein@gmail.com'
VERSION = '0.9'
DESCRIPTION = 'Simplyread.it'
LICENSE = 'AGPLv3+'
BROWSER = SimplyreaditBrowser
def iter_gallery_images(self, gallery):
with self.browser:
return self.browser.iter_gallery_images(gallery)
def get_gallery(self, _id):
match = re.match(r'(?:(?:.+symplyread.it/reader/read/)?/)?([^/]+(?:/[^/]+)*)', _id)
if match is None:
return None
_id = match.group(1)
gallery = BaseGallery(_id, url=('http://www.simplyread.it/reader/read/%s' % _id))
with self.browser:
return gallery
def fill_gallery(self, gallery, fields):
gallery.title = gallery.id
def fill_image(self, image, fields):
with self.browser:
self.browser.fill_image(image, fields)
OBJECTS = {
BaseGallery: fill_gallery,
BaseImage: fill_image }
DESCRIPTION = 'Simplyreadit manga reading site'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@class='open']/@src",
page_list_xpath="(//div[contains(@class,'dropdown_right')]/ul[@class='dropdown'])[1]/li/a/@href",
page_to_location="%s")
ID_TO_URL = 'http://www.simplyread.it/reader/read/%s'
ID_REGEXP = r'[^/]+(?:/[^/]+)*'
URL_REGEXP = r'.+simplyread.it/reader/read/(%s)/page/.+' % ID_REGEXP
PAGES = { r'http://.+\.simplyread.it/reader/read/.+': DisplayPage }