mirror of
https://github.com/moparisthebest/wallabag
synced 2024-11-27 11:22:17 -05:00
Fix #47 - update Readability class (thx to http://code.fivefilters.org/php-readability/)
This commit is contained in:
nicosomb
parent
c1c9f252f7
commit
0ace6cab0b
@ -2,6 +2,8 @@
|
||||
/**
|
||||
* Arc90's Readability ported to PHP for FiveFilters.org
|
||||
* Based on readability.js version 1.7.1 (without multi-page support)
|
||||
* Updated to allow HTML5 parsing with html5lib
|
||||
* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds
|
||||
* ------------------------------------------------------
|
||||
* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
|
||||
* Arc90's project URL: http://lab.arc90.com/experiments/readability/
|
||||
@ -10,7 +12,7 @@
|
||||
* More information: http://fivefilters.org/content-only/
|
||||
* License: Apache License, Version 2.0
|
||||
* Requires: PHP5
|
||||
* Date: 2011-07-22
|
||||
* Date: 2012-09-19
|
||||
*
|
||||
* Differences between the PHP port and the original
|
||||
* ------------------------------------------------------
|
||||
@ -62,14 +64,8 @@ $r->init();
|
||||
echo $r->articleContent->innerHTML;
|
||||
*/
|
||||
|
||||
|
||||
class Readability
|
||||
{
|
||||
/* constants */
|
||||
const FLAG_STRIP_UNLIKELYS = 1;
|
||||
const FLAG_WEIGHT_CLASSES = 2;
|
||||
const FLAG_CLEAN_CONDITIONALLY = 4;
|
||||
|
||||
public $version = '1.7.1-without-multi-page';
|
||||
public $convertLinksToFootnotes = false;
|
||||
public $revertForcedParagraphElements = true;
|
||||
@ -78,9 +74,10 @@ class Readability
|
||||
public $dom;
|
||||
public $url = null; // optional - URL where HTML was retrieved
|
||||
public $debug = false;
|
||||
public $lightClean = true; // preserves more content (experimental) added 2012-09-19
|
||||
protected $body = null; //
|
||||
protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
|
||||
protected $flags = self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4; // Start with all flags set.
|
||||
protected $flags = 7; // 1 | 2 | 4; // Start with all flags set.
|
||||
protected $success = false; // indicates whether we were able to extract or not
|
||||
|
||||
/**
|
||||
@ -88,37 +85,47 @@ class Readability
|
||||
* Defined up here so we don't instantiate them repeatedly in loops.
|
||||
**/
|
||||
public $regexps = array(
|
||||
'unlikelyCandidates' => '/combx|comment|comments|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i',
|
||||
'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',
|
||||
'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
|
||||
'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|attachment/i',
|
||||
'negative' => '/combx|comment|comments|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
|
||||
'divToPElements' => '/<(a|blockquote|dl|div|ol|p|pre|table|ul)/i',
|
||||
'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',
|
||||
'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
|
||||
'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
|
||||
'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
|
||||
'replaceFonts' => '/<(\/?)font[^>]*>/i',
|
||||
// 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
|
||||
'normalize' => '/\s{2,}/',
|
||||
'killBreaks' => '/(<br\s*\/?>(\s| ?)*){1,}/',
|
||||
'video' => '/http:\/\/(www\.)?(youtube|vimeo|dailymotion)\.com/i',
|
||||
'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',
|
||||
'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
|
||||
);
|
||||
|
||||
/* constants */
|
||||
const FLAG_STRIP_UNLIKELYS = 1;
|
||||
const FLAG_WEIGHT_CLASSES = 2;
|
||||
const FLAG_CLEAN_CONDITIONALLY = 4;
|
||||
|
||||
/**
|
||||
* Create instance of Readability
|
||||
* @param string UTF-8 encoded string
|
||||
* @param string (optional) URL associated with HTML (used for footnotes)
|
||||
* @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')
|
||||
*/
|
||||
function __construct($html, $url=null)
|
||||
function __construct($html, $url=null, $parser='libxml')
|
||||
{
|
||||
$this->url = $url;
|
||||
/* Turn all double br's into p's */
|
||||
$html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);
|
||||
$html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);
|
||||
$html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
|
||||
if (trim($html) == '') $html = '<html></html>';
|
||||
if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {
|
||||
// all good
|
||||
} else {
|
||||
$this->dom = new DOMDocument();
|
||||
$this->dom->preserveWhiteSpace = false;
|
||||
$this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
|
||||
if (trim($html) == '') $html = '<html></html>';
|
||||
@$this->dom->loadHTML($html);
|
||||
$this->url = $url;
|
||||
}
|
||||
$this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
|
||||
}
|
||||
|
||||
/**
|
||||
@ -185,7 +192,6 @@ class Readability
|
||||
$articleContent = $this->dom->createElement('div');
|
||||
$articleContent->setAttribute('id', 'readability-content');
|
||||
$articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';
|
||||
return $this->success;
|
||||
}
|
||||
|
||||
$overlay->setAttribute('id', 'readOverlay');
|
||||
@ -215,7 +221,7 @@ class Readability
|
||||
* Debug
|
||||
*/
|
||||
protected function dbg($msg) {
|
||||
if ($this->debug) echo '* ',$msg, '<br />', "\n";
|
||||
if ($this->debug) echo '* ',$msg, "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
@ -422,7 +428,7 @@ class Readability
|
||||
* If there is only one h2, they are probably using it
|
||||
* as a header and not a subheader, so remove it since we already have a header.
|
||||
***/
|
||||
if ($articleContent->getElementsByTagName('h2')->length == 1) {
|
||||
if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {
|
||||
$this->clean($articleContent, 'h2');
|
||||
}
|
||||
$this->clean($articleContent, 'iframe');
|
||||
@ -441,8 +447,9 @@ class Readability
|
||||
$imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
|
||||
$embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
|
||||
$objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
|
||||
$iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;
|
||||
|
||||
if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
|
||||
if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
|
||||
{
|
||||
$articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
|
||||
}
|
||||
@ -566,7 +573,7 @@ class Readability
|
||||
}
|
||||
else
|
||||
{
|
||||
// EXPERIMENTAL
|
||||
/* EXPERIMENTAL */
|
||||
// TODO: change these p elements back to text nodes after processing
|
||||
for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
|
||||
$childNode = $node->childNodes->item($i);
|
||||
@ -736,23 +743,6 @@ class Readability
|
||||
}
|
||||
}
|
||||
|
||||
/* Look for a special classname */
|
||||
if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('class') && $siblingNode->getAttribute('class') != '')
|
||||
{
|
||||
if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('class'))) {
|
||||
$append = true;
|
||||
}
|
||||
}
|
||||
|
||||
/* Look for a special classname */
|
||||
if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('id') && $siblingNode->getAttribute('id') != '')
|
||||
{
|
||||
if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('id'))) {
|
||||
$append = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if ($append)
|
||||
{
|
||||
$this->dbg('Appending node: ' . $siblingNode->nodeName);
|
||||
@ -965,13 +955,15 @@ class Readability
|
||||
* Clean a node of all elements of type "tag".
|
||||
* (Unless it's a youtube/vimeo video. People love movies.)
|
||||
*
|
||||
* Updated 2012-09-18 to preserve youtube/vimeo iframes
|
||||
*
|
||||
* @param DOMElement $e
|
||||
* @param string $tag
|
||||
* @return void
|
||||
*/
|
||||
public function clean($e, $tag) {
|
||||
$targetList = $e->getElementsByTagName($tag);
|
||||
$isEmbed = ($tag == 'object' || $tag == 'embed');
|
||||
$isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');
|
||||
|
||||
for ($y=$targetList->length-1; $y >= 0; $y--) {
|
||||
/* Allow youtube and vimeo videos through as people usually want to see those. */
|
||||
@ -1036,6 +1028,7 @@ class Readability
|
||||
$img = $tagsList->item($i)->getElementsByTagName('img')->length;
|
||||
$li = $tagsList->item($i)->getElementsByTagName('li')->length-100;
|
||||
$input = $tagsList->item($i)->getElementsByTagName('input')->length;
|
||||
$a = $tagsList->item($i)->getElementsByTagName('a')->length;
|
||||
|
||||
$embedCount = 0;
|
||||
$embeds = $tagsList->item($i)->getElementsByTagName('embed');
|
||||
@ -1044,28 +1037,69 @@ class Readability
|
||||
$embedCount++;
|
||||
}
|
||||
}
|
||||
$embeds = $tagsList->item($i)->getElementsByTagName('iframe');
|
||||
for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
|
||||
if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
|
||||
$embedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
$linkDensity = $this->getLinkDensity($tagsList->item($i));
|
||||
$contentLength = strlen($this->getInnerText($tagsList->item($i)));
|
||||
$toRemove = false;
|
||||
|
||||
if ( $img > $p ) {
|
||||
if ($this->lightClean) {
|
||||
$this->dbg('Light clean...');
|
||||
if ( ($img > $p) && ($img > 4) ) {
|
||||
$this->dbg(' more than 4 images and more image elements than paragraph elements');
|
||||
$toRemove = true;
|
||||
} else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
|
||||
$this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
|
||||
$toRemove = true;
|
||||
} else if ( $input > floor($p/3) ) {
|
||||
$this->dbg(' too many <input> elements');
|
||||
$toRemove = true;
|
||||
} else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
|
||||
} else if ($contentLength < 25 && ($embedCount === 0 && ($img === 0 || $img > 2))) {
|
||||
$this->dbg(' content length less than 25 chars, 0 embeds and either 0 images or more than 2 images');
|
||||
$toRemove = true;
|
||||
} else if($weight < 25 && $linkDensity > 0.2) {
|
||||
$this->dbg(' weight smaller than 25 and link density above 0.2');
|
||||
$toRemove = true;
|
||||
} else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
|
||||
$this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');
|
||||
$toRemove = true;
|
||||
} else if($embedCount > 3) {
|
||||
$this->dbg(' more than 3 embeds');
|
||||
$toRemove = true;
|
||||
}
|
||||
} else {
|
||||
$this->dbg('Standard clean...');
|
||||
if ( $img > $p ) {
|
||||
$this->dbg(' more image elements than paragraph elements');
|
||||
$toRemove = true;
|
||||
} else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
|
||||
$this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
|
||||
$toRemove = true;
|
||||
} else if ( $input > floor($p/3) ) {
|
||||
$this->dbg(' too many <input> elements');
|
||||
$toRemove = true;
|
||||
} else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
|
||||
$this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
|
||||
$toRemove = true;
|
||||
} else if($weight < 25 && $linkDensity > 0.2) {
|
||||
$this->dbg(' weight smaller than 25 and link density above 0.2');
|
||||
$toRemove = true;
|
||||
} else if($weight >= 25 && $linkDensity > 0.5) {
|
||||
$this->dbg(' weight above 25 but link density greater than 0.5');
|
||||
$toRemove = true;
|
||||
} else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
|
||||
$this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');
|
||||
$toRemove = true;
|
||||
}
|
||||
}
|
||||
|
||||
if ($toRemove) {
|
||||
//$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);
|
||||
$tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
|
||||
}
|
||||
}
|
||||
@ -1101,4 +1135,3 @@ class Readability
|
||||
$this->flags = $this->flags & ~$flag;
|
||||
}
|
||||
}
|
||||
?>
|
||||
Loading…
Reference in New Issue
Block a user