From 29c6fd460700e67fa538af337dcfce2787f2cc4e Mon Sep 17 00:00:00 2001 From: nicosomb Date: Tue, 23 Apr 2013 09:38:57 +0200 Subject: [PATCH 01/15] messages d'erreur si pas possible d'ajouter ou de supprimer un lien --- inc/functions.php | 35 +++++++++++++++++++++++------------ inc/store/sqlite.class.php | 2 ++ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/inc/functions.php b/inc/functions.php index 205f396..750d430 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -125,8 +125,6 @@ function prepare_url($url) } } - $msg->add('e', 'error during url preparation'); - logm('error during url preparation'); return FALSE; } @@ -311,26 +309,39 @@ function action_to_do($action, $url, $id = 0) if (MyTool::isUrl($url)) { if($parametres_url = prepare_url($url)) { - $store->add($url, $parametres_url['title'], $parametres_url['content']); - $last_id = $store->getLastId(); - if (DOWNLOAD_PICTURES) { - $content = filtre_picture($parametres_url['content'], $url, $last_id); + if ($store->add($url, $parametres_url['title'], $parametres_url['content'])) { + $last_id = $store->getLastId(); + if (DOWNLOAD_PICTURES) { + $content = filtre_picture($parametres_url['content'], $url, $last_id); + } + $msg->add('s', 'the link has been added successfully'); } - $msg->add('s', 'the link has been added successfully'); + else { + $msg->add('e', 'error during insertion : the link wasn\'t added'); + } + } + else { + $msg->add('e', 'error during url preparation : the link wasn\'t added'); + logm('error during url preparation'); } } else { - $msg->add('e', 'the link has been added successfully'); + $msg->add('e', 'error during url preparation : the link is not valid'); logm($url . ' is not a valid url'); } logm('add link ' . $url); break; case 'delete': - remove_directory(ABS_PATH . $id); - $store->deleteById($id); - $msg->add('s', 'the link has been deleted successfully'); - logm('delete link #' . $id); + if ($store->deleteById($id)) { + remove_directory(ABS_PATH . $id); + $msg->add('s', 'the link has been deleted successfully'); + logm('delete link #' . $id); + } + else { + $msg->add('e', 'the link wasn\'t deleted'); + logm('error : can\'t delete link #' . $id); + } break; case 'toggle_fav' : $store->favoriteById($id); diff --git a/inc/store/sqlite.class.php b/inc/store/sqlite.class.php index d5208a2..cda412e 100644 --- a/inc/store/sqlite.class.php +++ b/inc/store/sqlite.class.php @@ -107,6 +107,7 @@ class Sqlite extends Store { $sql_action = 'INSERT INTO entries ( url, title, content ) VALUES (?, ?, ?)'; $params_action = array($url, $title, $content); $query = $this->executeQuery($sql_action, $params_action); + return $query; } public function deleteById($id) { @@ -114,6 +115,7 @@ class Sqlite extends Store { $sql_action = "DELETE FROM entries WHERE id=?"; $params_action = array($id); $query = $this->executeQuery($sql_action, $params_action); + return $query; } public function favoriteById($id) { From 4ddbd26787ebbf4c5e01061ac9dbee4f8c21421b Mon Sep 17 00:00:00 2001 From: memiks Date: Tue, 23 Apr 2013 07:22:19 -0500 Subject: [PATCH 02/15] Ajout du global msg et gestion du retour dans Readability --- inc/Readability.php | 7 ++++--- inc/functions.php | 29 +++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/inc/Readability.php b/inc/Readability.php index c50bf2e..19298c1 100644 --- a/inc/Readability.php +++ b/inc/Readability.php @@ -80,7 +80,7 @@ class Readability public $debug = false; protected $body = null; // protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later - protected $flags = self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4; // Start with all flags set. + protected $flags = Self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4; // Start with all flags set. protected $success = false; // indicates whether we were able to extract or not /** @@ -90,7 +90,7 @@ class Readability public $regexps = array( 'unlikelyCandidates' => '/combx|comment|comments|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i', 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i', - 'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i', + 'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|attachment/i', 'negative' => '/combx|comment|comments|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i', 'divToPElements' => '/<(a|blockquote|dl|div|ol|p|pre|table|ul)/i', 'replaceBrs' => '/(]*>[ \n\r\t]*){2,}/i', @@ -106,7 +106,7 @@ class Readability * Create instance of Readability * @param string UTF-8 encoded string * @param string (optional) URL associated with HTML (used for footnotes) - */ + */ function __construct($html, $url=null) { /* Turn all double br's into p's */ @@ -185,6 +185,7 @@ class Readability $articleContent = $this->dom->createElement('div'); $articleContent->setAttribute('id', 'readability-content'); $articleContent->innerHTML = '

Sorry, Readability was unable to parse this page for content.

'; + return $this->success; } $overlay->setAttribute('id', 'readOverlay'); diff --git a/inc/functions.php b/inc/functions.php index 205f396..b27120c 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -39,6 +39,10 @@ function get_external_file($url) curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); curl_setopt($curl, CURLOPT_HEADER, false); + // FOR SSL do not verified certificate + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE); + curl_setopt($curl, CURLOPT_AUTOREFERER, TRUE ); + // FeedBurner requires a proper USER-AGENT... curl_setopt($curl, CURL_HTTP_VERSION_1_1, true); curl_setopt($curl, CURLOPT_ENCODING, "gzip, deflate"); @@ -54,7 +58,15 @@ function get_external_file($url) } else { // create http context and add timeout and user-agent - $context = stream_context_create(array('http'=>array('timeout' => $timeout,'header'=> "User-Agent: ".$useragent,/*spoot Mozilla Firefox*/'follow_location' => true))); + $context = stream_context_create(array( + 'http'=>array('timeout' => $timeout, + 'header'=> "User-Agent: ".$useragent, /*spoot Mozilla Firefox*/ + 'follow_location' => true), + // FOR SSL do not verified certificate + 'ssl' => array('verify_peer' => false, + 'allow_self_signed' => true) + ) + ); // only download page lesser than 4MB $data = @file_get_contents($url, false, $context, -1, 4000000); // We download at most 4 MB from source. @@ -98,6 +110,8 @@ function get_external_file($url) */ function prepare_url($url) { + global $msg; + $parametres = array(); $url = html_entity_decode(trim($url)); @@ -108,14 +122,21 @@ function prepare_url($url) $i=strpos($url,'#xtor=RSS-'); if ($i!==false) $url=substr($url,0,$i); $title = $url; - if (!preg_match('!^https?://!i', $url)) - $url = 'http://' . $url; + $html = Encoding::toUTF8(get_external_file($url,15)); + // If get_external_file if not able to retrieve HTTPS content try the same URL with HTTP protocol + if (!preg_match('!^https?://!i', $url) && (!isset($html) || strlen($html) <= 0)) { + $url = 'http://' . $url; + $html = Encoding::toUTF8(get_external_file($url,15)); + } - $html = Encoding::toUTF8(get_external_file($url,15)); if (isset($html) and strlen($html) > 0) { $r = new Readability($html, $url); + $r->convertLinksToFootnotes = CONVERT_LINKS_FOOTNOTES; + $r->debug=true; + $r->revertForcedParagraphElements = REVERT_FORCED_PARAGRAPH_ELEMENTS; + if($r->init()) { $content = $r->articleContent->innerHTML; From ae9571694f94ef363dba8564fef6b43ae833b508 Mon Sep 17 00:00:00 2001 From: memiks Date: Tue, 23 Apr 2013 07:29:16 -0500 Subject: [PATCH 03/15] Correction erreur sur le Self --- inc/Readability.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/Readability.php b/inc/Readability.php index 19298c1..2ce90f6 100644 --- a/inc/Readability.php +++ b/inc/Readability.php @@ -80,7 +80,7 @@ class Readability public $debug = false; protected $body = null; // protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later - protected $flags = Self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4; // Start with all flags set. + protected $flags = self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4; // Start with all flags set. protected $success = false; // indicates whether we were able to extract or not /** From cdcc8d2533d2ed65ac6a89c9a6d0041de7361ce1 Mon Sep 17 00:00:00 2001 From: memiks Date: Tue, 23 Apr 2013 08:09:54 -0500 Subject: [PATCH 04/15] Remove debug on Readability output --- inc/functions.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/inc/functions.php b/inc/functions.php index b27120c..0303aab 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -134,8 +134,7 @@ function prepare_url($url) $r = new Readability($html, $url); $r->convertLinksToFootnotes = CONVERT_LINKS_FOOTNOTES; - $r->debug=true; - $r->revertForcedParagraphElements = REVERT_FORCED_PARAGRAPH_ELEMENTS; + $r->revertForcedParagraphElements = REVERT_FORCED_PARAGRAPH_ELEMENTS; if($r->init()) { @@ -372,4 +371,4 @@ function logm($message) { $t = strval(date('Y/m/d_H:i:s')).' - '.$_SERVER["REMOTE_ADDR"].' - '.strval($message)."\n"; file_put_contents('./log.txt',$t,FILE_APPEND); -} \ No newline at end of file +} From 8d5aab49c185313338245f9c3a878628e16d7c85 Mon Sep 17 00:00:00 2001 From: nicosomb Date: Tue, 23 Apr 2013 15:18:22 +0200 Subject: [PATCH 05/15] =?UTF-8?q?lien=20corrig=C3=A9=20dans=20le=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6700415..70a0764 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ You can : ## Requirements & installation You have to install [sqlite for php](http://www.php.net/manual/en/book.sqlite.php) on your server. -Get the [latest version](https://github.com/nicosomb/poche) of poche on github. Unzip it and upload it on your server. poche must have write access on assets, cache and db directories. +Get the [latest version](https://github.com/inthepoche/poche) of poche on github. Unzip it and upload it on your server. poche must have write access on assets, cache and db directories. That's all, **poche works** ! From 2987031bc3d30437e7b34c3fe912a0ccbcc87f10 Mon Sep 17 00:00:00 2001 From: nicosomb Date: Tue, 23 Apr 2013 15:25:12 +0200 Subject: [PATCH 06/15] Fixed #73 - Can't Poch url with special caracter --- inc/config.php | 1 + inc/functions.php | 36 ++++++++++++++++++------------------ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/inc/config.php b/inc/config.php index 8bafd59..737f521 100644 --- a/inc/config.php +++ b/inc/config.php @@ -16,6 +16,7 @@ if (!is_dir('db/')) { define ('ABS_PATH', 'assets/'); define ('CONVERT_LINKS_FOOTNOTES', TRUE); +define ('REVERT_FORCED_PARAGRAPH_ELEMENTS',FALSE); define ('DOWNLOAD_PICTURES', TRUE); $storage_type = 'sqlite'; # sqlite or file diff --git a/inc/functions.php b/inc/functions.php index b830b61..c6cde82 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -39,9 +39,9 @@ function get_external_file($url) curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); curl_setopt($curl, CURLOPT_HEADER, false); - // FOR SSL do not verified certificate + // FOR SSL do not verified certificate curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE); - curl_setopt($curl, CURLOPT_AUTOREFERER, TRUE ); + curl_setopt($curl, CURLOPT_AUTOREFERER, TRUE ); // FeedBurner requires a proper USER-AGENT... curl_setopt($curl, CURL_HTTP_VERSION_1_1, true); @@ -59,14 +59,14 @@ function get_external_file($url) // create http context and add timeout and user-agent $context = stream_context_create(array( - 'http'=>array('timeout' => $timeout, - 'header'=> "User-Agent: ".$useragent, /*spoot Mozilla Firefox*/ - 'follow_location' => true), - // FOR SSL do not verified certificate - 'ssl' => array('verify_peer' => false, - 'allow_self_signed' => true) - ) - ); + 'http'=>array('timeout' => $timeout, + 'header'=> "User-Agent: ".$useragent, /*spoot Mozilla Firefox*/ + 'follow_location' => true), + // FOR SSL do not verified certificate + 'ssl' => array('verify_peer' => false, + 'allow_self_signed' => true) + ) + ); // only download page lesser than 4MB $data = @file_get_contents($url, false, $context, -1, 4000000); // We download at most 4 MB from source. @@ -122,19 +122,19 @@ function prepare_url($url) $i=strpos($url,'#xtor=RSS-'); if ($i!==false) $url=substr($url,0,$i); $title = $url; - $html = Encoding::toUTF8(get_external_file($url,15)); - // If get_external_file if not able to retrieve HTTPS content try the same URL with HTTP protocol - if (!preg_match('!^https?://!i', $url) && (!isset($html) || strlen($html) <= 0)) { - $url = 'http://' . $url; - $html = Encoding::toUTF8(get_external_file($url,15)); - } + $html = Encoding::toUTF8(get_external_file($url,15)); + // If get_external_file if not able to retrieve HTTPS content try the same URL with HTTP protocol + if (!preg_match('!^https?://!i', $url) && (!isset($html) || strlen($html) <= 0)) { + $url = 'http://' . $url; + $html = Encoding::toUTF8(get_external_file($url,15)); + } if (isset($html) and strlen($html) > 0) { $r = new Readability($html, $url); - + $r->convertLinksToFootnotes = CONVERT_LINKS_FOOTNOTES; - $r->revertForcedParagraphElements = REVERT_FORCED_PARAGRAPH_ELEMENTS; + $r->revertForcedParagraphElements = REVERT_FORCED_PARAGRAPH_ELEMENTS; if($r->init()) { From c1c9f252f7f0133903dc4143a0db8ed8104f8733 Mon Sep 17 00:00:00 2001 From: nicosomb Date: Tue, 23 Apr 2013 15:37:03 +0200 Subject: [PATCH 07/15] display flash message in article view --- inc/functions.php | 2 -- tpl/view.html | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/inc/functions.php b/inc/functions.php index c6cde82..7db9e90 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -110,8 +110,6 @@ function get_external_file($url) */ function prepare_url($url) { - global $msg; - $parametres = array(); $url = html_entity_decode(trim($url)); diff --git a/tpl/view.html b/tpl/view.html index 4384631..af94df2 100644 --- a/tpl/view.html +++ b/tpl/view.html @@ -38,6 +38,7 @@

{$title}

view original
+ {include="messages"}
{$content} From 0ace6cab0b934b7b8e4d4de4acca16294b7b12df Mon Sep 17 00:00:00 2001 From: nicosomb Date: Tue, 23 Apr 2013 15:57:23 +0200 Subject: [PATCH 08/15] Fix #47 - update Readability class (thx to http://code.fivefilters.org/php-readability/) --- inc/Readability.php | 2165 ++++++++++++++++++++++--------------------- 1 file changed, 1099 insertions(+), 1066 deletions(-) diff --git a/inc/Readability.php b/inc/Readability.php index 2ce90f6..d28d28f 100644 --- a/inc/Readability.php +++ b/inc/Readability.php @@ -1,7 +1,9 @@ init(); echo $r->articleContent->innerHTML; */ - class Readability { - /* constants */ - const FLAG_STRIP_UNLIKELYS = 1; - const FLAG_WEIGHT_CLASSES = 2; - const FLAG_CLEAN_CONDITIONALLY = 4; - - public $version = '1.7.1-without-multi-page'; - public $convertLinksToFootnotes = false; - public $revertForcedParagraphElements = true; - public $articleTitle; - public $articleContent; - public $dom; - public $url = null; // optional - URL where HTML was retrieved - public $debug = false; - protected $body = null; // - protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later - protected $flags = self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4; // Start with all flags set. - protected $success = false; // indicates whether we were able to extract or not - - /** - * All of the regular expressions in use within readability. - * Defined up here so we don't instantiate them repeatedly in loops. - **/ - public $regexps = array( - 'unlikelyCandidates' => '/combx|comment|comments|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i', - 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i', - 'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|attachment/i', - 'negative' => '/combx|comment|comments|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i', - 'divToPElements' => '/<(a|blockquote|dl|div|ol|p|pre|table|ul)/i', - 'replaceBrs' => '/(]*>[ \n\r\t]*){2,}/i', - 'replaceFonts' => '/<(\/?)font[^>]*>/i', - // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim() - 'normalize' => '/\s{2,}/', - 'killBreaks' => '/((\s| ?)*){1,}/', - 'video' => '/http:\/\/(www\.)?(youtube|vimeo|dailymotion)\.com/i', - 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i' - ); - - /** - * Create instance of Readability - * @param string UTF-8 encoded string - * @param string (optional) URL associated with HTML (used for footnotes) - */ - function __construct($html, $url=null) - { - /* Turn all double br's into p's */ - $html = preg_replace($this->regexps['replaceBrs'], '

', $html); - $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html); - $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8"); - $this->dom = new DOMDocument(); - $this->dom->preserveWhiteSpace = false; - $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement'); - if (trim($html) == '') $html = ''; - @$this->dom->loadHTML($html); - $this->url = $url; - } - - /** - * Get article title element - * @return DOMElement - */ - public function getTitle() { - return $this->articleTitle; - } - - /** - * Get article content element - * @return DOMElement - */ - public function getContent() { - return $this->articleContent; - } - - /** - * Runs readability. - * - * Workflow: - * 1. Prep the document by removing script tags, css, etc. - * 2. Build readability's DOM tree. - * 3. Grab the article content from the current dom tree. - * 4. Replace the current DOM tree with the new one. - * 5. Read peacefully. - * - * @return boolean true if we found content, false otherwise - **/ - public function init() - { - if (!isset($this->dom->documentElement)) return false; - $this->removeScripts($this->dom); - //die($this->getInnerHTML($this->dom->documentElement)); - - // Assume successful outcome - $this->success = true; - - $bodyElems = $this->dom->getElementsByTagName('body'); - if ($bodyElems->length > 0) { - if ($this->bodyCache == null) { - $this->bodyCache = $bodyElems->item(0)->innerHTML; - } - if ($this->body == null) { - $this->body = $bodyElems->item(0); - } - } - - $this->prepDocument(); - - //die($this->dom->documentElement->parentNode->nodeType); - //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement)); - //die($this->getInnerHTML($this->dom->documentElement)); - - /* Build readability's DOM tree */ - $overlay = $this->dom->createElement('div'); - $innerDiv = $this->dom->createElement('div'); - $articleTitle = $this->getArticleTitle(); - $articleContent = $this->grabArticle(); - - if (!$articleContent) { - $this->success = false; - $articleContent = $this->dom->createElement('div'); - $articleContent->setAttribute('id', 'readability-content'); - $articleContent->innerHTML = '

Sorry, Readability was unable to parse this page for content.

'; - return $this->success; - } - - $overlay->setAttribute('id', 'readOverlay'); - $innerDiv->setAttribute('id', 'readInner'); - - /* Glue the structure of our document together. */ - $innerDiv->appendChild($articleTitle); - $innerDiv->appendChild($articleContent); - $overlay->appendChild($innerDiv); - - /* Clear the old HTML, insert the new content. */ - $this->body->innerHTML = ''; - $this->body->appendChild($overlay); - //document.body.insertBefore(overlay, document.body.firstChild); - $this->body->removeAttribute('style'); - - $this->postProcessContent($articleContent); - - // Set title and content instance variables - $this->articleTitle = $articleTitle; - $this->articleContent = $articleContent; - - return $this->success; - } - - /** - * Debug - */ - protected function dbg($msg) { - if ($this->debug) echo '* ',$msg, '
', "\n"; - } - - /** - * Run any post-process modifications to article content as necessary. - * - * @param DOMElement - * @return void - */ - public function postProcessContent($articleContent) { - if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) { - $this->addFootnotes($articleContent); - } - } - - /** - * Get the article title as an H1. - * - * @return DOMElement - */ - protected function getArticleTitle() { - $curTitle = ''; - $origTitle = ''; - - try { - $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0)); - } catch(Exception $e) {} - - if (preg_match('/ [\|\-] /', $curTitle)) - { - $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle); - - if (count(explode(' ', $curTitle)) < 3) { - $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle); - } - } - else if (strpos($curTitle, ': ') !== false) - { - $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle); - - if (count(explode(' ', $curTitle)) < 3) { - $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle); - } - } - else if(strlen($curTitle) > 150 || strlen($curTitle) < 15) - { - $hOnes = $this->dom->getElementsByTagName('h1'); - if($hOnes->length == 1) - { - $curTitle = $this->getInnerText($hOnes->item(0)); - } - } - - $curTitle = trim($curTitle); - - if (count(explode(' ', $curTitle)) <= 4) { - $curTitle = $origTitle; - } - - $articleTitle = $this->dom->createElement('h1'); - $articleTitle->innerHTML = $curTitle; - - return $articleTitle; - } - - /** - * Prepare the HTML document for readability to scrape it. - * This includes things like stripping javascript, CSS, and handling terrible markup. - * - * @return void - **/ - protected function prepDocument() { - /** - * In some cases a body element can't be found (if the HTML is totally hosed for example) - * so we create a new body node and append it to the document. - */ - if ($this->body == null) - { - $this->body = $this->dom->createElement('body'); - $this->dom->documentElement->appendChild($this->body); - } - $this->body->setAttribute('id', 'readabilityBody'); - - /* Remove all style tags in head */ - $styleTags = $this->dom->getElementsByTagName('style'); - for ($i = $styleTags->length-1; $i >= 0; $i--) - { - $styleTags->item($i)->parentNode->removeChild($styleTags->item($i)); - } - - /* Turn all double br's into p's */ - /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */ - //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '

').replace(readability.regexps.replaceFonts, '<$1span>'); - // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree. - // Manipulating innerHTML as it's done in JS is not possible in PHP. - } - - /** - * For easier reading, convert this document to have footnotes at the bottom rather than inline links. - * @see http://www.roughtype.com/archives/2010/05/experiments_in.php - * - * @return void - **/ - public function addFootnotes($articleContent) { - $footnotesWrapper = $this->dom->createElement('div'); - $footnotesWrapper->setAttribute('id', 'readability-footnotes'); - $footnotesWrapper->innerHTML = '

References

'; - - $articleFootnotes = $this->dom->createElement('ol'); - $articleFootnotes->setAttribute('id', 'readability-footnotes-list'); - $footnotesWrapper->appendChild($articleFootnotes); - - $articleLinks = $articleContent->getElementsByTagName('a'); - - $linkCount = 0; - for ($i = 0; $i < $articleLinks->length; $i++) - { - $articleLink = $articleLinks->item($i); - $footnoteLink = $articleLink->cloneNode(true); - $refLink = $this->dom->createElement('a'); - $footnote = $this->dom->createElement('li'); - $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST); - if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST); - //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host, - $linkText = $this->getInnerText($articleLink); - - if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) { - continue; - } - - $linkCount++; - - /** Add a superscript reference after the article link */ - $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount); - $refLink->innerHTML = '[' . $linkCount . ']'; - $refLink->setAttribute('class', 'readability-DoNotFootnote'); - $refLink->setAttribute('style', 'color: inherit;'); - - //TODO: does this work or should we use DOMNode.isSameNode()? - if ($articleLink->parentNode->lastChild == $articleLink) { - $articleLink->parentNode->appendChild($refLink); - } else { - $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling); - } - - $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;'); - $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount); - - $footnote->innerHTML = '^ '; - - $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText); - $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount); - - $footnote->appendChild($footnoteLink); - if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . ' (' . $linkDomain . ')'; - - $articleFootnotes->appendChild($footnote); - } - - if ($linkCount > 0) { - $articleContent->appendChild($footnotesWrapper); - } - } - - /** - * Reverts P elements with class 'readability-styled' - * to text nodes - which is what they were before. - * - * @param DOMElement - * @return void - */ - function revertReadabilityStyledElements($articleContent) { - $xpath = new DOMXPath($articleContent->ownerDocument); - $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent); - //$elems = $articleContent->getElementsByTagName('p'); - for ($i = $elems->length-1; $i >= 0; $i--) { - $e = $elems->item($i); - $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e); - //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') { - // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e); - //} - } - } - - /** - * Prepare the article node for display. Clean out any inline styles, - * iframes, forms, strip extraneous

tags, etc. - * - * @param DOMElement - * @return void - */ - function prepArticle($articleContent) { - $this->cleanStyles($articleContent); - $this->killBreaks($articleContent); - if ($this->revertForcedParagraphElements) { - $this->revertReadabilityStyledElements($articleContent); - } - - /* Clean out junk from the article content */ - $this->cleanConditionally($articleContent, 'form'); - $this->clean($articleContent, 'object'); - $this->clean($articleContent, 'h1'); - - /** - * If there is only one h2, they are probably using it - * as a header and not a subheader, so remove it since we already have a header. - ***/ - if ($articleContent->getElementsByTagName('h2')->length == 1) { - $this->clean($articleContent, 'h2'); - } - $this->clean($articleContent, 'iframe'); - - $this->cleanHeaders($articleContent); - - /* Do these last as the previous stuff may have removed junk that will affect these */ - $this->cleanConditionally($articleContent, 'table'); - $this->cleanConditionally($articleContent, 'ul'); - $this->cleanConditionally($articleContent, 'div'); - - /* Remove extra paragraphs */ - $articleParagraphs = $articleContent->getElementsByTagName('p'); - for ($i = $articleParagraphs->length-1; $i >= 0; $i--) - { - $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length; - $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length; - $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length; - - if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '') - { - $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i)); - } - } - - try { - $articleContent->innerHTML = preg_replace('/]*>\s*

innerHTML); - //articleContent.innerHTML = articleContent.innerHTML.replace(/]*>\s*

dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e); - } - } - - /** - * Initialize a node with the readability object. Also checks the - * className/id for special names to add to its score. - * - * @param Element - * @return void - **/ - protected function initializeNode($node) { - $readability = $this->dom->createAttribute('readability'); - $readability->value = 0; // this is our contentScore - $node->setAttributeNode($readability); - - switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case - case 'DIV': - $readability->value += 5; - break; - - case 'PRE': - case 'TD': - case 'BLOCKQUOTE': - $readability->value += 3; - break; - - case 'ADDRESS': - case 'OL': - case 'UL': - case 'DL': - case 'DD': - case 'DT': - case 'LI': - case 'FORM': - $readability->value -= 3; - break; - - case 'H1': - case 'H2': - case 'H3': - case 'H4': - case 'H5': - case 'H6': - case 'TH': - $readability->value -= 5; - break; - } - $readability->value += $this->getClassWeight($node); - } - - /*** - * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is - * most likely to be the stuff a user wants to read. Then return it wrapped up in a div. - * - * @return DOMElement - **/ - protected function grabArticle($page=null) { - $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS); - if (!$page) $page = $this->dom; - $allElements = $page->getElementsByTagName('*'); - /** - * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs - * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.) - * - * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5 - * TODO: Shouldn't this be a reverse traversal? - **/ - $node = null; - $nodesToScore = array(); - for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) { - //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) { - //$node = $targetList->item($nodeIndex); - $tagName = strtoupper($node->tagName); - /* Remove unlikely candidates */ - if ($stripUnlikelyCandidates) { - $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id'); - if ( - preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) && - !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) && - $tagName != 'BODY' - ) - { - $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString); - //$nodesToRemove[] = $node; - $node->parentNode->removeChild($node); - $nodeIndex--; - continue; - } - } - - if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') { - $nodesToScore[] = $node; - } - - /* Turn all divs that don't have children block level elements into p's */ - if ($tagName == 'DIV') { - if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) { - //$this->dbg('Altering div to p'); - $newNode = $this->dom->createElement('p'); - try { - $newNode->innerHTML = $node->innerHTML; - //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node); - $node->parentNode->replaceChild($newNode, $node); - $nodeIndex--; - $nodesToScore[] = $node; // or $newNode? - } - catch(Exception $e) { - $this->dbg('Could not alter div to p, reverting back to div.: ' . $e); - } - } - else - { - // EXPERIMENTAL - // TODO: change these p elements back to text nodes after processing - for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) { - $childNode = $node->childNodes->item($i); - if ($childNode->nodeType == 3) { // XML_TEXT_NODE - //$this->dbg('replacing text node with a p tag with the same content.'); - $p = $this->dom->createElement('p'); - $p->innerHTML = $childNode->nodeValue; - $p->setAttribute('style', 'display: inline;'); - $p->setAttribute('class', 'readability-styled'); - $childNode->parentNode->replaceChild($p, $childNode); - } - } - } - } - } - - /** - * Loop through all paragraphs, and assign a score to them based on how content-y they look. - * Then add their score to their parent node. - * - * A score is determined by things like number of commas, class names, etc. Maybe eventually link density. - **/ - $candidates = array(); - for ($pt=0; $pt < count($nodesToScore); $pt++) { - $parentNode = $nodesToScore[$pt]->parentNode; - // $grandParentNode = $parentNode ? $parentNode->parentNode : null; - $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null); - $innerText = $this->getInnerText($nodesToScore[$pt]); - - if (!$parentNode || !isset($parentNode->tagName)) { - continue; - } - - /* If this paragraph is less than 25 characters, don't even count it. */ - if(strlen($innerText) < 25) { - continue; - } - - /* Initialize readability data for the parent. */ - if (!$parentNode->hasAttribute('readability')) - { - $this->initializeNode($parentNode); - $candidates[] = $parentNode; - } - - /* Initialize readability data for the grandparent. */ - if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName)) - { - $this->initializeNode($grandParentNode); - $candidates[] = $grandParentNode; - } - - $contentScore = 0; - - /* Add a point for the paragraph itself as a base. */ - $contentScore++; - - /* Add points for any commas within this paragraph */ - $contentScore += count(explode(',', $innerText)); - - /* For every 100 characters in this paragraph, add another point. Up to 3 points. */ - $contentScore += min(floor(strlen($innerText) / 100), 3); - - /* Add the score to the parent. The grandparent gets half. */ - $parentNode->getAttributeNode('readability')->value += $contentScore; - - if ($grandParentNode) { - $grandParentNode->getAttributeNode('readability')->value += $contentScore/2; - } - } - - /** - * After we've calculated scores, loop through all of the possible candidate nodes we found - * and find the one with the highest score. - **/ - $topCandidate = null; - for ($c=0, $cl=count($candidates); $c < $cl; $c++) - { - /** - * Scale the final candidates score based on link density. Good content should have a - * relatively small link density (5% or less) and be mostly unaffected by this operation. - **/ - $readability = $candidates[$c]->getAttributeNode('readability'); - $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c])); - - $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value); - - if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) { - $topCandidate = $candidates[$c]; - } - } - - /** - * If we still have no top candidate, just use the body as a last resort. - * We also have to copy the body node so it is something we can modify. - **/ - if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY') - { - $topCandidate = $this->dom->createElement('div'); - if ($page instanceof DOMDocument) { - if (!isset($page->documentElement)) { - // we don't have a body either? what a mess! :) - } else { - $topCandidate->innerHTML = $page->documentElement->innerHTML; - $page->documentElement->innerHTML = ''; - $page->documentElement->appendChild($topCandidate); - } - } else { - $topCandidate->innerHTML = $page->innerHTML; - $page->innerHTML = ''; - $page->appendChild($topCandidate); - } - $this->initializeNode($topCandidate); - } - - /** - * Now that we have the top candidate, look through its siblings for content that might also be related. - * Things like preambles, content split by ads that we removed, etc. - **/ - $articleContent = $this->dom->createElement('div'); - $articleContent->setAttribute('id', 'readability-content'); - $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2); - $siblingNodes = $topCandidate->parentNode->childNodes; - if (!isset($siblingNodes)) { - $siblingNodes = new stdClass; - $siblingNodes->length = 0; - } - - for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++) - { - $siblingNode = $siblingNodes->item($s); - $append = false; - - $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : '')); - - //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown')); - - if ($siblingNode === $topCandidate) - // or if ($siblingNode->isSameNode($topCandidate)) - { - $append = true; - } - - $contentBonus = 0; - /* Give a bonus if sibling nodes and top candidates have the example same classname */ - if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') { - $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2; - } - - if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) - { - $append = true; - } - - if (strtoupper($siblingNode->nodeName) == 'P') { - $linkDensity = $this->getLinkDensity($siblingNode); - $nodeContent = $this->getInnerText($siblingNode); - $nodeLength = strlen($nodeContent); - - if ($nodeLength > 80 && $linkDensity < 0.25) - { - $append = true; - } - else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent)) - { - $append = true; - } - } - - /* Look for a special classname */ - if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('class') && $siblingNode->getAttribute('class') != '') - { - if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('class'))) { - $append = true; - } - } - - /* Look for a special classname */ - if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('id') && $siblingNode->getAttribute('id') != '') - { - if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('id'))) { - $append = true; - } - } - - - if ($append) - { - $this->dbg('Appending node: ' . $siblingNode->nodeName); - - $nodeToAppend = null; - $sibNodeName = strtoupper($siblingNode->nodeName); - if ($sibNodeName != 'DIV' && $sibNodeName != 'P') { - /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */ - - $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.'); - $nodeToAppend = $this->dom->createElement('div'); - try { - $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id')); - $nodeToAppend->innerHTML = $siblingNode->innerHTML; - } - catch(Exception $e) - { - $this->dbg('Could not alter siblingNode to div, reverting back to original.'); - $nodeToAppend = $siblingNode; - $s--; - $sl--; - } - } else { - $nodeToAppend = $siblingNode; - $s--; - $sl--; - } - - /* To ensure a node does not interfere with readability styles, remove its classnames */ - $nodeToAppend->removeAttribute('class'); - - /* Append sibling and subtract from our list because it removes the node when you append to another node */ - $articleContent->appendChild($nodeToAppend); - } - } - - /** - * So we have all of the content that we need. Now we clean it up for presentation. - **/ - $this->prepArticle($articleContent); - - /** - * Now that we've gone through the full algorithm, check to see if we got any meaningful content. - * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher - * likelihood of finding the content, and the sieve approach gives us a higher likelihood of - * finding the -right- content. - **/ - if (strlen($this->getInnerText($articleContent, false)) < 250) - { - // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7 - // in the meantime, we check and create an empty element if it's not there. - if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body'); - $this->body->innerHTML = $this->bodyCache; - - if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) { - $this->removeFlag(self::FLAG_STRIP_UNLIKELYS); - return $this->grabArticle($this->body); - } - else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) { - $this->removeFlag(self::FLAG_WEIGHT_CLASSES); - return $this->grabArticle($this->body); - } - else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) { - $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY); - return $this->grabArticle($this->body); - } - else { - return false; - } - } - return $articleContent; - } - - /** - * Remove script tags from document - * - * @param DOMElement - * @return void - */ - public function removeScripts($doc) { - $scripts = $doc->getElementsByTagName('script'); - for($i = $scripts->length-1; $i >= 0; $i--) - { - $scripts->item($i)->parentNode->removeChild($scripts->item($i)); - } - } - - /** - * Get the inner text of a node. - * This also strips out any excess whitespace to be found. - * - * @param DOMElement $ - * @param boolean $normalizeSpaces (default: true) - * @return string - **/ - public function getInnerText($e, $normalizeSpaces=true) { - $textContent = ''; - - if (!isset($e->textContent) || $e->textContent == '') { - return ''; - } - - $textContent = trim($e->textContent); - - if ($normalizeSpaces) { - return preg_replace($this->regexps['normalize'], ' ', $textContent); - } else { - return $textContent; - } - } - - /** - * Get the number of times a string $s appears in the node $e. - * - * @param DOMElement $e - * @param string - what to count. Default is "," - * @return number (integer) - **/ - public function getCharCount($e, $s=',') { - return substr_count($this->getInnerText($e), $s); - } - - /** - * Remove the style attribute on every $e and under. - * - * @param DOMElement $e - * @return void - */ - public function cleanStyles($e) { - if (!is_object($e)) return; - $elems = $e->getElementsByTagName('*'); - foreach ($elems as $elem) { - $elem->removeAttribute('style'); - } - } - - /** - * Get the density of links as a percentage of the content - * This is the amount of text that is inside a link divided by the total text in the node. - * - * @param DOMElement $e - * @return number (float) - */ - public function getLinkDensity($e) { - $links = $e->getElementsByTagName('a'); - $textLength = strlen($this->getInnerText($e)); - $linkLength = 0; - for ($i=0, $il=$links->length; $i < $il; $i++) - { - $linkLength += strlen($this->getInnerText($links->item($i))); - } - if ($textLength > 0) { - return $linkLength / $textLength; - } else { - return 0; - } - } - - /** - * Get an elements class/id weight. Uses regular expressions to tell if this - * element looks good or bad. - * - * @param DOMElement $e - * @return number (Integer) - */ - public function getClassWeight($e) { - if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) { - return 0; - } - - $weight = 0; - - /* Look for a special classname */ - if ($e->hasAttribute('class') && $e->getAttribute('class') != '') - { - if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) { - $weight -= 25; - } - if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) { - $weight += 25; - } - } - - /* Look for a special ID */ - if ($e->hasAttribute('id') && $e->getAttribute('id') != '') - { - if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) { - $weight -= 25; - } - if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) { - $weight += 25; - } - } - return $weight; - } - - /** - * Remove extraneous break tags from a node. - * - * @param DOMElement $node - * @return void - */ - public function killBreaks($node) { - $html = $node->innerHTML; - $html = preg_replace($this->regexps['killBreaks'], '
', $html); - $node->innerHTML = $html; - } - - /** - * Clean a node of all elements of type "tag". - * (Unless it's a youtube/vimeo video. People love movies.) - * - * @param DOMElement $e - * @param string $tag - * @return void - */ - public function clean($e, $tag) { - $targetList = $e->getElementsByTagName($tag); - $isEmbed = ($tag == 'object' || $tag == 'embed'); - - for ($y=$targetList->length-1; $y >= 0; $y--) { - /* Allow youtube and vimeo videos through as people usually want to see those. */ - if ($isEmbed) { - $attributeValues = ''; - for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) { - $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test) - } - - /* First, check the elements attributes to see if any of them contain youtube or vimeo */ - if (preg_match($this->regexps['video'], $attributeValues)) { - continue; - } - - /* Then check the elements inside this element for the same. */ - if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) { - continue; - } - } - $targetList->item($y)->parentNode->removeChild($targetList->item($y)); - } - } - - /** - * Clean an element of all tags of type "tag" if they look fishy. - * "Fishy" is an algorithm based on content length, classnames, - * link density, number of images & embeds, etc. - * - * @param DOMElement $e - * @param string $tag - * @return void - */ - public function cleanConditionally($e, $tag) { - if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) { - return; - } - - $tagsList = $e->getElementsByTagName($tag); - $curTagsLength = $tagsList->length; - - /** - * Gather counts for other typical elements embedded within. - * Traverse backwards so we can remove nodes at the same time without effecting the traversal. - * - * TODO: Consider taking into account original contentScore here. - */ - for ($i=$curTagsLength-1; $i >= 0; $i--) { - $weight = $this->getClassWeight($tagsList->item($i)); - $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0; - - $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : '')); - - if ($weight + $contentScore < 0) { - $tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); - } - else if ( $this->getCharCount($tagsList->item($i), ',') < 10) { - /** - * If there are not very many commas, and the number of - * non-paragraph elements is more than paragraphs or other ominous signs, remove the element. - **/ - $p = $tagsList->item($i)->getElementsByTagName('p')->length; - $img = $tagsList->item($i)->getElementsByTagName('img')->length; - $li = $tagsList->item($i)->getElementsByTagName('li')->length-100; - $input = $tagsList->item($i)->getElementsByTagName('input')->length; - - $embedCount = 0; - $embeds = $tagsList->item($i)->getElementsByTagName('embed'); - for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) { - if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) { - $embedCount++; - } - } - - $linkDensity = $this->getLinkDensity($tagsList->item($i)); - $contentLength = strlen($this->getInnerText($tagsList->item($i))); - $toRemove = false; - - if ( $img > $p ) { - $toRemove = true; - } else if ($li > $p && $tag != 'ul' && $tag != 'ol') { - $toRemove = true; - } else if ( $input > floor($p/3) ) { - $toRemove = true; - } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) { - $toRemove = true; - } else if($weight < 25 && $linkDensity > 0.2) { - $toRemove = true; - } else if($weight >= 25 && $linkDensity > 0.5) { - $toRemove = true; - } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) { - $toRemove = true; - } - - if ($toRemove) { - $tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); - } - } - } - } - - /** - * Clean out spurious headers from an Element. Checks things like classnames and link density. - * - * @param DOMElement $e - * @return void - */ - public function cleanHeaders($e) { - for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) { - $headers = $e->getElementsByTagName('h' . $headerIndex); - for ($i=$headers->length-1; $i >=0; $i--) { - if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) { - $headers->item($i)->parentNode->removeChild($headers->item($i)); - } - } - } - } - - public function flagIsActive($flag) { - return ($this->flags & $flag) > 0; - } - - public function addFlag($flag) { - $this->flags = $this->flags | $flag; - } - - public function removeFlag($flag) { - $this->flags = $this->flags & ~$flag; - } -} -?> \ No newline at end of file + public $version = '1.7.1-without-multi-page'; + public $convertLinksToFootnotes = false; + public $revertForcedParagraphElements = true; + public $articleTitle; + public $articleContent; + public $dom; + public $url = null; // optional - URL where HTML was retrieved + public $debug = false; + public $lightClean = true; // preserves more content (experimental) added 2012-09-19 + protected $body = null; // + protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later + protected $flags = 7; // 1 | 2 | 4; // Start with all flags set. + protected $success = false; // indicates whether we were able to extract or not + + /** + * All of the regular expressions in use within readability. + * Defined up here so we don't instantiate them repeatedly in loops. + **/ + public $regexps = array( + 'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i', + 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i', + 'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i', + 'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i', + 'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i', + 'replaceBrs' => '/(]*>[ \n\r\t]*){2,}/i', + 'replaceFonts' => '/<(\/?)font[^>]*>/i', + // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim() + 'normalize' => '/\s{2,}/', + 'killBreaks' => '/((\s| ?)*){1,}/', + 'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i', + 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i' + ); + + /* constants */ + const FLAG_STRIP_UNLIKELYS = 1; + const FLAG_WEIGHT_CLASSES = 2; + const FLAG_CLEAN_CONDITIONALLY = 4; + + /** + * Create instance of Readability + * @param string UTF-8 encoded string + * @param string (optional) URL associated with HTML (used for footnotes) + * @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib') + */ + function __construct($html, $url=null, $parser='libxml') + { + $this->url = $url; + /* Turn all double br's into p's */ + $html = preg_replace($this->regexps['replaceBrs'], '

', $html); + $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html); + $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8"); + if (trim($html) == '') $html = ''; + if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) { + // all good + } else { + $this->dom = new DOMDocument(); + $this->dom->preserveWhiteSpace = false; + @$this->dom->loadHTML($html); + } + $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement'); + } + + /** + * Get article title element + * @return DOMElement + */ + public function getTitle() { + return $this->articleTitle; + } + + /** + * Get article content element + * @return DOMElement + */ + public function getContent() { + return $this->articleContent; + } + + /** + * Runs readability. + * + * Workflow: + * 1. Prep the document by removing script tags, css, etc. + * 2. Build readability's DOM tree. + * 3. Grab the article content from the current dom tree. + * 4. Replace the current DOM tree with the new one. + * 5. Read peacefully. + * + * @return boolean true if we found content, false otherwise + **/ + public function init() + { + if (!isset($this->dom->documentElement)) return false; + $this->removeScripts($this->dom); + //die($this->getInnerHTML($this->dom->documentElement)); + + // Assume successful outcome + $this->success = true; + + $bodyElems = $this->dom->getElementsByTagName('body'); + if ($bodyElems->length > 0) { + if ($this->bodyCache == null) { + $this->bodyCache = $bodyElems->item(0)->innerHTML; + } + if ($this->body == null) { + $this->body = $bodyElems->item(0); + } + } + + $this->prepDocument(); + + //die($this->dom->documentElement->parentNode->nodeType); + //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement)); + //die($this->getInnerHTML($this->dom->documentElement)); + + /* Build readability's DOM tree */ + $overlay = $this->dom->createElement('div'); + $innerDiv = $this->dom->createElement('div'); + $articleTitle = $this->getArticleTitle(); + $articleContent = $this->grabArticle(); + + if (!$articleContent) { + $this->success = false; + $articleContent = $this->dom->createElement('div'); + $articleContent->setAttribute('id', 'readability-content'); + $articleContent->innerHTML = '

Sorry, Readability was unable to parse this page for content.

'; + } + + $overlay->setAttribute('id', 'readOverlay'); + $innerDiv->setAttribute('id', 'readInner'); + + /* Glue the structure of our document together. */ + $innerDiv->appendChild($articleTitle); + $innerDiv->appendChild($articleContent); + $overlay->appendChild($innerDiv); + + /* Clear the old HTML, insert the new content. */ + $this->body->innerHTML = ''; + $this->body->appendChild($overlay); + //document.body.insertBefore(overlay, document.body.firstChild); + $this->body->removeAttribute('style'); + + $this->postProcessContent($articleContent); + + // Set title and content instance variables + $this->articleTitle = $articleTitle; + $this->articleContent = $articleContent; + + return $this->success; + } + + /** + * Debug + */ + protected function dbg($msg) { + if ($this->debug) echo '* ',$msg, "\n"; + } + + /** + * Run any post-process modifications to article content as necessary. + * + * @param DOMElement + * @return void + */ + public function postProcessContent($articleContent) { + if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) { + $this->addFootnotes($articleContent); + } + } + + /** + * Get the article title as an H1. + * + * @return DOMElement + */ + protected function getArticleTitle() { + $curTitle = ''; + $origTitle = ''; + + try { + $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0)); + } catch(Exception $e) {} + + if (preg_match('/ [\|\-] /', $curTitle)) + { + $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle); + + if (count(explode(' ', $curTitle)) < 3) { + $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle); + } + } + else if (strpos($curTitle, ': ') !== false) + { + $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle); + + if (count(explode(' ', $curTitle)) < 3) { + $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle); + } + } + else if(strlen($curTitle) > 150 || strlen($curTitle) < 15) + { + $hOnes = $this->dom->getElementsByTagName('h1'); + if($hOnes->length == 1) + { + $curTitle = $this->getInnerText($hOnes->item(0)); + } + } + + $curTitle = trim($curTitle); + + if (count(explode(' ', $curTitle)) <= 4) { + $curTitle = $origTitle; + } + + $articleTitle = $this->dom->createElement('h1'); + $articleTitle->innerHTML = $curTitle; + + return $articleTitle; + } + + /** + * Prepare the HTML document for readability to scrape it. + * This includes things like stripping javascript, CSS, and handling terrible markup. + * + * @return void + **/ + protected function prepDocument() { + /** + * In some cases a body element can't be found (if the HTML is totally hosed for example) + * so we create a new body node and append it to the document. + */ + if ($this->body == null) + { + $this->body = $this->dom->createElement('body'); + $this->dom->documentElement->appendChild($this->body); + } + $this->body->setAttribute('id', 'readabilityBody'); + + /* Remove all style tags in head */ + $styleTags = $this->dom->getElementsByTagName('style'); + for ($i = $styleTags->length-1; $i >= 0; $i--) + { + $styleTags->item($i)->parentNode->removeChild($styleTags->item($i)); + } + + /* Turn all double br's into p's */ + /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */ + //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '

').replace(readability.regexps.replaceFonts, '<$1span>'); + // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree. + // Manipulating innerHTML as it's done in JS is not possible in PHP. + } + + /** + * For easier reading, convert this document to have footnotes at the bottom rather than inline links. + * @see http://www.roughtype.com/archives/2010/05/experiments_in.php + * + * @return void + **/ + public function addFootnotes($articleContent) { + $footnotesWrapper = $this->dom->createElement('div'); + $footnotesWrapper->setAttribute('id', 'readability-footnotes'); + $footnotesWrapper->innerHTML = '

References

'; + + $articleFootnotes = $this->dom->createElement('ol'); + $articleFootnotes->setAttribute('id', 'readability-footnotes-list'); + $footnotesWrapper->appendChild($articleFootnotes); + + $articleLinks = $articleContent->getElementsByTagName('a'); + + $linkCount = 0; + for ($i = 0; $i < $articleLinks->length; $i++) + { + $articleLink = $articleLinks->item($i); + $footnoteLink = $articleLink->cloneNode(true); + $refLink = $this->dom->createElement('a'); + $footnote = $this->dom->createElement('li'); + $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST); + if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST); + //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host, + $linkText = $this->getInnerText($articleLink); + + if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) { + continue; + } + + $linkCount++; + + /** Add a superscript reference after the article link */ + $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount); + $refLink->innerHTML = '[' . $linkCount . ']'; + $refLink->setAttribute('class', 'readability-DoNotFootnote'); + $refLink->setAttribute('style', 'color: inherit;'); + + //TODO: does this work or should we use DOMNode.isSameNode()? + if ($articleLink->parentNode->lastChild == $articleLink) { + $articleLink->parentNode->appendChild($refLink); + } else { + $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling); + } + + $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;'); + $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount); + + $footnote->innerHTML = '^ '; + + $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText); + $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount); + + $footnote->appendChild($footnoteLink); + if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . ' (' . $linkDomain . ')'; + + $articleFootnotes->appendChild($footnote); + } + + if ($linkCount > 0) { + $articleContent->appendChild($footnotesWrapper); + } + } + + /** + * Reverts P elements with class 'readability-styled' + * to text nodes - which is what they were before. + * + * @param DOMElement + * @return void + */ + function revertReadabilityStyledElements($articleContent) { + $xpath = new DOMXPath($articleContent->ownerDocument); + $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent); + //$elems = $articleContent->getElementsByTagName('p'); + for ($i = $elems->length-1; $i >= 0; $i--) { + $e = $elems->item($i); + $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e); + //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') { + // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e); + //} + } + } + + /** + * Prepare the article node for display. Clean out any inline styles, + * iframes, forms, strip extraneous

tags, etc. + * + * @param DOMElement + * @return void + */ + function prepArticle($articleContent) { + $this->cleanStyles($articleContent); + $this->killBreaks($articleContent); + if ($this->revertForcedParagraphElements) { + $this->revertReadabilityStyledElements($articleContent); + } + + /* Clean out junk from the article content */ + $this->cleanConditionally($articleContent, 'form'); + $this->clean($articleContent, 'object'); + $this->clean($articleContent, 'h1'); + + /** + * If there is only one h2, they are probably using it + * as a header and not a subheader, so remove it since we already have a header. + ***/ + if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) { + $this->clean($articleContent, 'h2'); + } + $this->clean($articleContent, 'iframe'); + + $this->cleanHeaders($articleContent); + + /* Do these last as the previous stuff may have removed junk that will affect these */ + $this->cleanConditionally($articleContent, 'table'); + $this->cleanConditionally($articleContent, 'ul'); + $this->cleanConditionally($articleContent, 'div'); + + /* Remove extra paragraphs */ + $articleParagraphs = $articleContent->getElementsByTagName('p'); + for ($i = $articleParagraphs->length-1; $i >= 0; $i--) + { + $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length; + $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length; + $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length; + $iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length; + + if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '') + { + $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i)); + } + } + + try { + $articleContent->innerHTML = preg_replace('/]*>\s*

innerHTML); + //articleContent.innerHTML = articleContent.innerHTML.replace(/]*>\s*

dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e); + } + } + + /** + * Initialize a node with the readability object. Also checks the + * className/id for special names to add to its score. + * + * @param Element + * @return void + **/ + protected function initializeNode($node) { + $readability = $this->dom->createAttribute('readability'); + $readability->value = 0; // this is our contentScore + $node->setAttributeNode($readability); + + switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case + case 'DIV': + $readability->value += 5; + break; + + case 'PRE': + case 'TD': + case 'BLOCKQUOTE': + $readability->value += 3; + break; + + case 'ADDRESS': + case 'OL': + case 'UL': + case 'DL': + case 'DD': + case 'DT': + case 'LI': + case 'FORM': + $readability->value -= 3; + break; + + case 'H1': + case 'H2': + case 'H3': + case 'H4': + case 'H5': + case 'H6': + case 'TH': + $readability->value -= 5; + break; + } + $readability->value += $this->getClassWeight($node); + } + + /*** + * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is + * most likely to be the stuff a user wants to read. Then return it wrapped up in a div. + * + * @return DOMElement + **/ + protected function grabArticle($page=null) { + $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS); + if (!$page) $page = $this->dom; + $allElements = $page->getElementsByTagName('*'); + /** + * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs + * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.) + * + * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5 + * TODO: Shouldn't this be a reverse traversal? + **/ + $node = null; + $nodesToScore = array(); + for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) { + //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) { + //$node = $targetList->item($nodeIndex); + $tagName = strtoupper($node->tagName); + /* Remove unlikely candidates */ + if ($stripUnlikelyCandidates) { + $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id'); + if ( + preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) && + !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) && + $tagName != 'BODY' + ) + { + $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString); + //$nodesToRemove[] = $node; + $node->parentNode->removeChild($node); + $nodeIndex--; + continue; + } + } + + if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') { + $nodesToScore[] = $node; + } + + /* Turn all divs that don't have children block level elements into p's */ + if ($tagName == 'DIV') { + if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) { + //$this->dbg('Altering div to p'); + $newNode = $this->dom->createElement('p'); + try { + $newNode->innerHTML = $node->innerHTML; + //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node); + $node->parentNode->replaceChild($newNode, $node); + $nodeIndex--; + $nodesToScore[] = $node; // or $newNode? + } + catch(Exception $e) { + $this->dbg('Could not alter div to p, reverting back to div.: ' . $e); + } + } + else + { + /* EXPERIMENTAL */ + // TODO: change these p elements back to text nodes after processing + for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) { + $childNode = $node->childNodes->item($i); + if ($childNode->nodeType == 3) { // XML_TEXT_NODE + //$this->dbg('replacing text node with a p tag with the same content.'); + $p = $this->dom->createElement('p'); + $p->innerHTML = $childNode->nodeValue; + $p->setAttribute('style', 'display: inline;'); + $p->setAttribute('class', 'readability-styled'); + $childNode->parentNode->replaceChild($p, $childNode); + } + } + } + } + } + + /** + * Loop through all paragraphs, and assign a score to them based on how content-y they look. + * Then add their score to their parent node. + * + * A score is determined by things like number of commas, class names, etc. Maybe eventually link density. + **/ + $candidates = array(); + for ($pt=0; $pt < count($nodesToScore); $pt++) { + $parentNode = $nodesToScore[$pt]->parentNode; + // $grandParentNode = $parentNode ? $parentNode->parentNode : null; + $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null); + $innerText = $this->getInnerText($nodesToScore[$pt]); + + if (!$parentNode || !isset($parentNode->tagName)) { + continue; + } + + /* If this paragraph is less than 25 characters, don't even count it. */ + if(strlen($innerText) < 25) { + continue; + } + + /* Initialize readability data for the parent. */ + if (!$parentNode->hasAttribute('readability')) + { + $this->initializeNode($parentNode); + $candidates[] = $parentNode; + } + + /* Initialize readability data for the grandparent. */ + if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName)) + { + $this->initializeNode($grandParentNode); + $candidates[] = $grandParentNode; + } + + $contentScore = 0; + + /* Add a point for the paragraph itself as a base. */ + $contentScore++; + + /* Add points for any commas within this paragraph */ + $contentScore += count(explode(',', $innerText)); + + /* For every 100 characters in this paragraph, add another point. Up to 3 points. */ + $contentScore += min(floor(strlen($innerText) / 100), 3); + + /* Add the score to the parent. The grandparent gets half. */ + $parentNode->getAttributeNode('readability')->value += $contentScore; + + if ($grandParentNode) { + $grandParentNode->getAttributeNode('readability')->value += $contentScore/2; + } + } + + /** + * After we've calculated scores, loop through all of the possible candidate nodes we found + * and find the one with the highest score. + **/ + $topCandidate = null; + for ($c=0, $cl=count($candidates); $c < $cl; $c++) + { + /** + * Scale the final candidates score based on link density. Good content should have a + * relatively small link density (5% or less) and be mostly unaffected by this operation. + **/ + $readability = $candidates[$c]->getAttributeNode('readability'); + $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c])); + + $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value); + + if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) { + $topCandidate = $candidates[$c]; + } + } + + /** + * If we still have no top candidate, just use the body as a last resort. + * We also have to copy the body node so it is something we can modify. + **/ + if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY') + { + $topCandidate = $this->dom->createElement('div'); + if ($page instanceof DOMDocument) { + if (!isset($page->documentElement)) { + // we don't have a body either? what a mess! :) + } else { + $topCandidate->innerHTML = $page->documentElement->innerHTML; + $page->documentElement->innerHTML = ''; + $page->documentElement->appendChild($topCandidate); + } + } else { + $topCandidate->innerHTML = $page->innerHTML; + $page->innerHTML = ''; + $page->appendChild($topCandidate); + } + $this->initializeNode($topCandidate); + } + + /** + * Now that we have the top candidate, look through its siblings for content that might also be related. + * Things like preambles, content split by ads that we removed, etc. + **/ + $articleContent = $this->dom->createElement('div'); + $articleContent->setAttribute('id', 'readability-content'); + $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2); + $siblingNodes = $topCandidate->parentNode->childNodes; + if (!isset($siblingNodes)) { + $siblingNodes = new stdClass; + $siblingNodes->length = 0; + } + + for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++) + { + $siblingNode = $siblingNodes->item($s); + $append = false; + + $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : '')); + + //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown')); + + if ($siblingNode === $topCandidate) + // or if ($siblingNode->isSameNode($topCandidate)) + { + $append = true; + } + + $contentBonus = 0; + /* Give a bonus if sibling nodes and top candidates have the example same classname */ + if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') { + $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2; + } + + if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) + { + $append = true; + } + + if (strtoupper($siblingNode->nodeName) == 'P') { + $linkDensity = $this->getLinkDensity($siblingNode); + $nodeContent = $this->getInnerText($siblingNode); + $nodeLength = strlen($nodeContent); + + if ($nodeLength > 80 && $linkDensity < 0.25) + { + $append = true; + } + else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent)) + { + $append = true; + } + } + + if ($append) + { + $this->dbg('Appending node: ' . $siblingNode->nodeName); + + $nodeToAppend = null; + $sibNodeName = strtoupper($siblingNode->nodeName); + if ($sibNodeName != 'DIV' && $sibNodeName != 'P') { + /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */ + + $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.'); + $nodeToAppend = $this->dom->createElement('div'); + try { + $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id')); + $nodeToAppend->innerHTML = $siblingNode->innerHTML; + } + catch(Exception $e) + { + $this->dbg('Could not alter siblingNode to div, reverting back to original.'); + $nodeToAppend = $siblingNode; + $s--; + $sl--; + } + } else { + $nodeToAppend = $siblingNode; + $s--; + $sl--; + } + + /* To ensure a node does not interfere with readability styles, remove its classnames */ + $nodeToAppend->removeAttribute('class'); + + /* Append sibling and subtract from our list because it removes the node when you append to another node */ + $articleContent->appendChild($nodeToAppend); + } + } + + /** + * So we have all of the content that we need. Now we clean it up for presentation. + **/ + $this->prepArticle($articleContent); + + /** + * Now that we've gone through the full algorithm, check to see if we got any meaningful content. + * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher + * likelihood of finding the content, and the sieve approach gives us a higher likelihood of + * finding the -right- content. + **/ + if (strlen($this->getInnerText($articleContent, false)) < 250) + { + // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7 + // in the meantime, we check and create an empty element if it's not there. + if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body'); + $this->body->innerHTML = $this->bodyCache; + + if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) { + $this->removeFlag(self::FLAG_STRIP_UNLIKELYS); + return $this->grabArticle($this->body); + } + else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) { + $this->removeFlag(self::FLAG_WEIGHT_CLASSES); + return $this->grabArticle($this->body); + } + else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) { + $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY); + return $this->grabArticle($this->body); + } + else { + return false; + } + } + return $articleContent; + } + + /** + * Remove script tags from document + * + * @param DOMElement + * @return void + */ + public function removeScripts($doc) { + $scripts = $doc->getElementsByTagName('script'); + for($i = $scripts->length-1; $i >= 0; $i--) + { + $scripts->item($i)->parentNode->removeChild($scripts->item($i)); + } + } + + /** + * Get the inner text of a node. + * This also strips out any excess whitespace to be found. + * + * @param DOMElement $ + * @param boolean $normalizeSpaces (default: true) + * @return string + **/ + public function getInnerText($e, $normalizeSpaces=true) { + $textContent = ''; + + if (!isset($e->textContent) || $e->textContent == '') { + return ''; + } + + $textContent = trim($e->textContent); + + if ($normalizeSpaces) { + return preg_replace($this->regexps['normalize'], ' ', $textContent); + } else { + return $textContent; + } + } + + /** + * Get the number of times a string $s appears in the node $e. + * + * @param DOMElement $e + * @param string - what to count. Default is "," + * @return number (integer) + **/ + public function getCharCount($e, $s=',') { + return substr_count($this->getInnerText($e), $s); + } + + /** + * Remove the style attribute on every $e and under. + * + * @param DOMElement $e + * @return void + */ + public function cleanStyles($e) { + if (!is_object($e)) return; + $elems = $e->getElementsByTagName('*'); + foreach ($elems as $elem) { + $elem->removeAttribute('style'); + } + } + + /** + * Get the density of links as a percentage of the content + * This is the amount of text that is inside a link divided by the total text in the node. + * + * @param DOMElement $e + * @return number (float) + */ + public function getLinkDensity($e) { + $links = $e->getElementsByTagName('a'); + $textLength = strlen($this->getInnerText($e)); + $linkLength = 0; + for ($i=0, $il=$links->length; $i < $il; $i++) + { + $linkLength += strlen($this->getInnerText($links->item($i))); + } + if ($textLength > 0) { + return $linkLength / $textLength; + } else { + return 0; + } + } + + /** + * Get an elements class/id weight. Uses regular expressions to tell if this + * element looks good or bad. + * + * @param DOMElement $e + * @return number (Integer) + */ + public function getClassWeight($e) { + if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) { + return 0; + } + + $weight = 0; + + /* Look for a special classname */ + if ($e->hasAttribute('class') && $e->getAttribute('class') != '') + { + if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) { + $weight -= 25; + } + if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) { + $weight += 25; + } + } + + /* Look for a special ID */ + if ($e->hasAttribute('id') && $e->getAttribute('id') != '') + { + if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) { + $weight -= 25; + } + if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) { + $weight += 25; + } + } + return $weight; + } + + /** + * Remove extraneous break tags from a node. + * + * @param DOMElement $node + * @return void + */ + public function killBreaks($node) { + $html = $node->innerHTML; + $html = preg_replace($this->regexps['killBreaks'], '
', $html); + $node->innerHTML = $html; + } + + /** + * Clean a node of all elements of type "tag". + * (Unless it's a youtube/vimeo video. People love movies.) + * + * Updated 2012-09-18 to preserve youtube/vimeo iframes + * + * @param DOMElement $e + * @param string $tag + * @return void + */ + public function clean($e, $tag) { + $targetList = $e->getElementsByTagName($tag); + $isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed'); + + for ($y=$targetList->length-1; $y >= 0; $y--) { + /* Allow youtube and vimeo videos through as people usually want to see those. */ + if ($isEmbed) { + $attributeValues = ''; + for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) { + $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test) + } + + /* First, check the elements attributes to see if any of them contain youtube or vimeo */ + if (preg_match($this->regexps['video'], $attributeValues)) { + continue; + } + + /* Then check the elements inside this element for the same. */ + if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) { + continue; + } + } + $targetList->item($y)->parentNode->removeChild($targetList->item($y)); + } + } + + /** + * Clean an element of all tags of type "tag" if they look fishy. + * "Fishy" is an algorithm based on content length, classnames, + * link density, number of images & embeds, etc. + * + * @param DOMElement $e + * @param string $tag + * @return void + */ + public function cleanConditionally($e, $tag) { + if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) { + return; + } + + $tagsList = $e->getElementsByTagName($tag); + $curTagsLength = $tagsList->length; + + /** + * Gather counts for other typical elements embedded within. + * Traverse backwards so we can remove nodes at the same time without effecting the traversal. + * + * TODO: Consider taking into account original contentScore here. + */ + for ($i=$curTagsLength-1; $i >= 0; $i--) { + $weight = $this->getClassWeight($tagsList->item($i)); + $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0; + + $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : '')); + + if ($weight + $contentScore < 0) { + $tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); + } + else if ( $this->getCharCount($tagsList->item($i), ',') < 10) { + /** + * If there are not very many commas, and the number of + * non-paragraph elements is more than paragraphs or other ominous signs, remove the element. + **/ + $p = $tagsList->item($i)->getElementsByTagName('p')->length; + $img = $tagsList->item($i)->getElementsByTagName('img')->length; + $li = $tagsList->item($i)->getElementsByTagName('li')->length-100; + $input = $tagsList->item($i)->getElementsByTagName('input')->length; + $a = $tagsList->item($i)->getElementsByTagName('a')->length; + + $embedCount = 0; + $embeds = $tagsList->item($i)->getElementsByTagName('embed'); + for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) { + if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) { + $embedCount++; + } + } + $embeds = $tagsList->item($i)->getElementsByTagName('iframe'); + for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) { + if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) { + $embedCount++; + } + } + + $linkDensity = $this->getLinkDensity($tagsList->item($i)); + $contentLength = strlen($this->getInnerText($tagsList->item($i))); + $toRemove = false; + + if ($this->lightClean) { + $this->dbg('Light clean...'); + if ( ($img > $p) && ($img > 4) ) { + $this->dbg(' more than 4 images and more image elements than paragraph elements'); + $toRemove = true; + } else if ($li > $p && $tag != 'ul' && $tag != 'ol') { + $this->dbg(' too many

  • elements, and parent is not
      or
        '); + $toRemove = true; + } else if ( $input > floor($p/3) ) { + $this->dbg(' too many elements'); + $toRemove = true; + } else if ($contentLength < 25 && ($embedCount === 0 && ($img === 0 || $img > 2))) { + $this->dbg(' content length less than 25 chars, 0 embeds and either 0 images or more than 2 images'); + $toRemove = true; + } else if($weight < 25 && $linkDensity > 0.2) { + $this->dbg(' weight smaller than 25 and link density above 0.2'); + $toRemove = true; + } else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) { + $this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5'); + $toRemove = true; + } else if($embedCount > 3) { + $this->dbg(' more than 3 embeds'); + $toRemove = true; + } + } else { + $this->dbg('Standard clean...'); + if ( $img > $p ) { + $this->dbg(' more image elements than paragraph elements'); + $toRemove = true; + } else if ($li > $p && $tag != 'ul' && $tag != 'ol') { + $this->dbg(' too many
      1. elements, and parent is not
          or
            '); + $toRemove = true; + } else if ( $input > floor($p/3) ) { + $this->dbg(' too many elements'); + $toRemove = true; + } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) { + $this->dbg(' content length less than 25 chars and 0 images, or more than 2 images'); + $toRemove = true; + } else if($weight < 25 && $linkDensity > 0.2) { + $this->dbg(' weight smaller than 25 and link density above 0.2'); + $toRemove = true; + } else if($weight >= 25 && $linkDensity > 0.5) { + $this->dbg(' weight above 25 but link density greater than 0.5'); + $toRemove = true; + } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) { + $this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed'); + $toRemove = true; + } + } + + if ($toRemove) { + //$this->dbg('Removing: '.$tagsList->item($i)->innerHTML); + $tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); + } + } + } + } + + /** + * Clean out spurious headers from an Element. Checks things like classnames and link density. + * + * @param DOMElement $e + * @return void + */ + public function cleanHeaders($e) { + for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) { + $headers = $e->getElementsByTagName('h' . $headerIndex); + for ($i=$headers->length-1; $i >=0; $i--) { + if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) { + $headers->item($i)->parentNode->removeChild($headers->item($i)); + } + } + } + } + + public function flagIsActive($flag) { + return ($this->flags & $flag) > 0; + } + + public function addFlag($flag) { + $this->flags = $this->flags | $flag; + } + + public function removeFlag($flag) { + $this->flags = $this->flags & ~$flag; + } +} \ No newline at end of file From 494e21b4da955912b9e521787650e269818b1147 Mon Sep 17 00:00:00 2001 From: nicosomb Date: Tue, 23 Apr 2013 16:03:52 +0200 Subject: [PATCH 09/15] add tidy call to clean html output --- inc/functions.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/inc/functions.php b/inc/functions.php index 7db9e90..c2a149c 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -127,6 +127,12 @@ function prepare_url($url) $html = Encoding::toUTF8(get_external_file($url,15)); } + if (function_exists('tidy_parse_string')) { + $tidy = tidy_parse_string($html, array(), 'UTF8'); + $tidy->cleanRepair(); + $html = $tidy->value; + } + if (isset($html) and strlen($html) > 0) { $r = new Readability($html, $url); @@ -279,7 +285,13 @@ function display_view($view, $id = 0, $full_head = 'yes') $tpl->assign('id', $entry['id']); $tpl->assign('url', $entry['url']); $tpl->assign('title', $entry['title']); - $tpl->assign('content', $entry['content']); + $content = $entry['content']; + if (function_exists('tidy_parse_string')) { + $tidy = tidy_parse_string($content, array('indent'=>true, 'show-body-only' => true), 'UTF8'); + $tidy->cleanRepair(); + $content = $tidy->value; + } + $tpl->assign('content', $content); $tpl->assign('is_fav', $entry['is_fav']); $tpl->assign('is_read', $entry['is_read']); $tpl->assign('load_all_js', 0); From baa8617364c3888fd28c3ffe83cfeb5345914356 Mon Sep 17 00:00:00 2001 From: nicosomb Date: Wed, 24 Apr 2013 08:27:55 +0200 Subject: [PATCH 10/15] update JSLikeHTMLElement --- inc/JSLikeHTMLElement.php | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/inc/JSLikeHTMLElement.php b/inc/JSLikeHTMLElement.php index 0557205..dfcc1be 100644 --- a/inc/JSLikeHTMLElement.php +++ b/inc/JSLikeHTMLElement.php @@ -4,7 +4,7 @@ * * This class extends PHP's DOMElement to allow * users to get and set the innerHTML property of -* HTML elements in the same way it's done in +* HTML elements in the same way it's done in * JavaScript. * * Example usage: @@ -15,16 +15,16 @@ * $doc->registerNodeClass('DOMElement', 'JSLikeHTMLElement'); * $doc->loadHTML('

            Para 1

            Para 2

            '); * $elem = $doc->getElementsByTagName('div')->item(0); -* +* * // print innerHTML * echo $elem->innerHTML; // prints '

            Para 1

            Para 2

            ' * echo "\n\n"; -* +* * // set innerHTML * $elem->innerHTML = 'FiveFilters.org'; * echo $elem->innerHTML; // prints 'FiveFilters.org' * echo "\n\n"; -* +* * // print document (with our changes) * echo $doc->saveXML(); * @endcode @@ -59,7 +59,7 @@ class JSLikeHTMLElement extends DOMElement $value = mb_convert_encoding($value, 'HTML-ENTITIES', 'UTF-8'); // Using will generate a warning, but so will bad HTML // (and by this point, bad HTML is what we've got). - // We use it (and suppress the warning) because an HTML fragment will + // We use it (and suppress the warning) because an HTML fragment will // be wrapped around tags which we don't really want to keep. // Note: despite the warning, if loadHTML succeeds it will return true. $result = @$f->loadHTML(''.$value.''); @@ -86,7 +86,7 @@ class JSLikeHTMLElement extends DOMElement * @code * $string = $div->innerHTML; * @endcode - */ + */ public function __get($name) { if ($name == 'innerHTML') { @@ -106,5 +106,4 @@ class JSLikeHTMLElement extends DOMElement { return '['.$this->tagName.']'; } -} -?> \ No newline at end of file +} \ No newline at end of file From aa8c9f2a32ea75278d52c86c6a3a39d34bce5cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 31 May 2013 22:55:52 +0200 Subject: [PATCH 11/15] Installation mode --- inc/Session.class.php | 2 +- inc/config.php | 25 ++++++++++++++++--- inc/functions.php | 7 ++++-- inc/store/sqlite.class.php | 51 +++++++++++++++++++++++++++++++++++++- inc/store/store.class.php | 8 ++++++ index.php | 9 +++++-- tpl/install.html | 30 ++++++++++++++++++++++ tpl/login.html | 2 +- 8 files changed, 124 insertions(+), 10 deletions(-) create mode 100644 tpl/install.html diff --git a/inc/Session.class.php b/inc/Session.class.php index ee12b3d..eff924c 100644 --- a/inc/Session.class.php +++ b/inc/Session.class.php @@ -93,7 +93,7 @@ class Session // Force logout public static function logout() { - unset($_SESSION['uid'],$_SESSION['info'],$_SESSION['expires_on'],$_SESSION['tokens']); + unset($_SESSION['uid'],$_SESSION['info'],$_SESSION['expires_on'],$_SESSION['tokens'], $_SESSION['login'], $_SESSION['pass']); } // Make sure user is logged in. diff --git a/inc/config.php b/inc/config.php index 737f521..2de725f 100644 --- a/inc/config.php +++ b/inc/config.php @@ -18,6 +18,7 @@ define ('ABS_PATH', 'assets/'); define ('CONVERT_LINKS_FOOTNOTES', TRUE); define ('REVERT_FORCED_PARAGRAPH_ELEMENTS',FALSE); define ('DOWNLOAD_PICTURES', TRUE); +define ('SALT', '464v54gLLw928uz4zUBqkRJeiPY68zCX'); $storage_type = 'sqlite'; # sqlite or file include 'functions.php'; @@ -33,9 +34,7 @@ require_once 'class.messages.php'; Session::init(); -$store = new $storage_type(); -$msg = new Messages(); - +$store = new $storage_type(); # initialisation de RainTPL raintpl::$tpl_dir = './tpl/'; raintpl::$cache_dir = './cache/'; @@ -43,4 +42,24 @@ raintpl::$base_url = get_poche_url(); raintpl::configure('path_replace', false); raintpl::configure('debug', false); $tpl = new raintpl(); + +if(!$store->isInstalled()) +{ + logm('poche still not installed'); + $tpl->draw('install'); + if (isset($_GET['install'])) { + if (($_POST['password'] == $_POST['password_repeat']) + && $_POST['password'] != "" && $_POST['login'] != "") { + $store->install($_POST['login'], encode_string($_POST['password'] . $_POST['login'])); + Session::logout(); + MyTool::redirect(); + } + } + exit(); +} + +$_SESSION['login'] = (isset ($_SESSION['login'])) ? $_SESSION['login'] : $store->getLogin(); +$_SESSION['pass'] = (isset ($_SESSION['pass'])) ? $_SESSION['pass'] : $store->getPassword(); + +$msg = new Messages(); $tpl->assign('msg', $msg); \ No newline at end of file diff --git a/inc/functions.php b/inc/functions.php index c2a149c..73e591c 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -23,6 +23,11 @@ function get_poche_url() return $protocol . "://" . $_SERVER['HTTP_HOST'] . $_SERVER['REQUEST_URI']; } +function encode_string($string) +{ + return sha1($string . SALT); +} + // function define to retrieve url content function get_external_file($url) { @@ -375,12 +380,10 @@ function action_to_do($action, $url, $id = 0) break; case 'toggle_fav' : $store->favoriteById($id); - $msg->add('s', 'the favorite toggle has been done successfully'); logm('mark as favorite link #' . $id); break; case 'toggle_archive' : $store->archiveById($id); - $msg->add('s', 'the archive toggle has been done successfully'); logm('archive link #' . $id); break; default: diff --git a/inc/store/sqlite.class.php b/inc/store/sqlite.class.php index cda412e..4bfbb29 100644 --- a/inc/store/sqlite.class.php +++ b/inc/store/sqlite.class.php @@ -17,7 +17,6 @@ class Sqlite extends Store { parent::__construct(); $this->handle = new PDO(self::$db_path); - $this->handle->exec('CREATE TABLE IF NOT EXISTS "entries" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE , "title" VARCHAR, "url" VARCHAR UNIQUE , "is_read" INTEGER DEFAULT 0, "is_fav" INTEGER DEFAULT 0, "content" BLOB)'); $this->handle->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); } @@ -25,6 +24,56 @@ class Sqlite extends Store { return $this->handle; } + public function isInstalled() { + $sql = "SELECT name FROM sqlite_sequence WHERE name=?"; + $query = $this->executeQuery($sql, array('config')); + $hasConfig = $query->fetchAll(); + + if (count($hasConfig) == 0) + return FALSE; + + if (!$this->getLogin() || !$this->getPassword()) + return FALSE; + + return TRUE; + } + + public function install($login, $password) { + $this->getHandle()->exec('CREATE TABLE IF NOT EXISTS "config" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE , "name" VARCHAR UNIQUE, "value" BLOB)'); + + $this->handle->exec('CREATE TABLE IF NOT EXISTS "entries" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE , "title" VARCHAR, "url" VARCHAR UNIQUE , "is_read" INTEGER DEFAULT 0, "is_fav" INTEGER DEFAULT 0, "content" BLOB)'); + + if (!$this->getLogin()) { + $sql_login = 'INSERT INTO config ( name, value ) VALUES (?, ?)'; + $params_login = array('login', $login); + $query = $this->executeQuery($sql_login, $params_login); + } + + if (!$this->getPassword()) { + $sql_pass = 'INSERT INTO config ( name, value ) VALUES (?, ?)'; + $params_pass = array('password', $password); + $query = $this->executeQuery($sql_pass, $params_pass); + } + + return TRUE; + } + + public function getLogin() { + $sql = "SELECT value FROM config WHERE name=?"; + $query = $this->executeQuery($sql, array('login')); + $login = $query->fetchAll(); + + return isset($login[0]['value']) ? $login[0]['value'] : FALSE; + } + + public function getPassword() { + $sql = "SELECT value FROM config WHERE name=?"; + $query = $this->executeQuery($sql, array('password')); + $pass = $query->fetchAll(); + + return isset($pass[0]['value']) ? $pass[0]['value'] : FALSE; + } + private function executeQuery($sql, $params) { try { diff --git a/inc/store/store.class.php b/inc/store/store.class.php index 360ff7c..dd7d4cf 100644 --- a/inc/store/store.class.php +++ b/inc/store/store.class.php @@ -13,6 +13,14 @@ class Store { } + public function getLogin() { + + } + + public function getPassword() { + + } + public function add() { } diff --git a/index.php b/index.php index 829d551..1522a2e 100644 --- a/index.php +++ b/index.php @@ -25,9 +25,14 @@ $ref = empty($_SERVER['HTTP_REFERER']) ? '' : $_SERVER['HTTP_REFERER']; if (isset($_GET['login'])) { // Login if (!empty($_POST['login']) && !empty($_POST['password'])) { - if (Session::login('poche', 'poche', $_POST['login'], $_POST['password'])) { +// echo $_SESSION['login']."
            "; +// echo $_SESSION['pass']."
            "; +// echo $_POST['login']."
            "; +// echo encode_string($_POST['password'] . $_POST['login']); +// die; + if (Session::login($_SESSION['login'], $_SESSION['pass'], $_POST['login'], encode_string($_POST['password'] . $_POST['login']))) { logm('login successful'); - $msg->add('s', 'welcome in your pocket!'); + $msg->add('s', 'welcome in your poche!'); if (!empty($_POST['longlastingsession'])) { $_SESSION['longlastingsession'] = 31536000; $_SESSION['expires_on'] = time() + $_SESSION['longlastingsession']; diff --git a/tpl/install.html b/tpl/install.html new file mode 100644 index 0000000..d11a781 --- /dev/null +++ b/tpl/install.html @@ -0,0 +1,30 @@ +{include="head"} + +
            +

            logo pochepoche

            +
            +
            +
            +
            +

            install your poche

            +
            + + +
            +
            + + +
            +
            + + +
            +
            + +
            +
            + + +
            + +{include="footer"} diff --git a/tpl/login.html b/tpl/login.html index ebe4b5e..69c17a5 100644 --- a/tpl/login.html +++ b/tpl/login.html @@ -12,7 +12,7 @@
            - +
            From da368cc84f4d79afc30052a209502fa8fd17241e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Sat, 1 Jun 2013 08:23:37 +0200 Subject: [PATCH 12/15] update password in config screen --- index.php | 18 +++++++++++++----- tpl/config.html | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/index.php b/index.php index 1522a2e..91d0b30 100644 --- a/index.php +++ b/index.php @@ -25,11 +25,6 @@ $ref = empty($_SERVER['HTTP_REFERER']) ? '' : $_SERVER['HTTP_REFERER']; if (isset($_GET['login'])) { // Login if (!empty($_POST['login']) && !empty($_POST['password'])) { -// echo $_SESSION['login']."
            "; -// echo $_SESSION['pass']."
            "; -// echo $_POST['login']."
            "; -// echo encode_string($_POST['password'] . $_POST['login']); -// die; if (Session::login($_SESSION['login'], $_SESSION['pass'], $_POST['login'], encode_string($_POST['password'] . $_POST['login']))) { logm('login successful'); $msg->add('s', 'welcome in your poche!'); @@ -55,6 +50,19 @@ elseif (isset($_GET['logout'])) { Session::logout(); MyTool::redirect(); } +elseif (isset($_GET['config'])) { + if (isset($_POST['password']) && isset($_POST['password_repeat'])) { + if ($_POST['password'] == $_POST['password_repeat'] && $_POST['password'] != "") { + logm('password updated'); + $store->updatePassword(encode_string($_POST['password'] . $_SESSION['login'])); + $msg->add('s', 'your password has been updated'); + } + else + $msg->add('e', 'your password can\'t be empty and you have to repeat it in the second field'); + } + else + $msg->add('e', 'error in your password update'); +} # Traitement des paramètres et déclenchement des actions $view = (isset ($_REQUEST['view'])) ? htmlentities($_REQUEST['view']) : 'index'; diff --git a/tpl/config.html b/tpl/config.html index 7d1c6af..1100d45 100644 --- a/tpl/config.html +++ b/tpl/config.html @@ -3,6 +3,25 @@

            Thanks to the bookmarklet, you will be able to easily add a link to your poche. If you don't know how use a bookmarklet, have a look here.

            Drag & drop this link to your bookmarks bar and have fun with poche.

            poche it !

            + +

            Password

            +
            +
            +
            + + +
            +
            + + +
            +
            + +
            +
            + + +

            Export

            Click here to export your poche datas.

            \ No newline at end of file From 6499b26ae8e148fa5481db5b7a4e9fc0e55aad94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Sat, 1 Jun 2013 08:29:37 +0200 Subject: [PATCH 13/15] demo mode (you can't update password in demo mode --- img/messages/close.png | Bin img/messages/cross.png | Bin img/messages/help.png | Bin img/messages/tick.png | Bin img/messages/warning.png | Bin inc/config.php | 1 + inc/store/sqlite.class.php | 7 +++++++ index.php | 11 +++++++---- 8 files changed, 15 insertions(+), 4 deletions(-) mode change 100755 => 100644 img/messages/close.png mode change 100755 => 100644 img/messages/cross.png mode change 100755 => 100644 img/messages/help.png mode change 100755 => 100644 img/messages/tick.png mode change 100755 => 100644 img/messages/warning.png diff --git a/img/messages/close.png b/img/messages/close.png old mode 100755 new mode 100644 diff --git a/img/messages/cross.png b/img/messages/cross.png old mode 100755 new mode 100644 diff --git a/img/messages/help.png b/img/messages/help.png old mode 100755 new mode 100644 diff --git a/img/messages/tick.png b/img/messages/tick.png old mode 100755 new mode 100644 diff --git a/img/messages/warning.png b/img/messages/warning.png old mode 100755 new mode 100644 diff --git a/inc/config.php b/inc/config.php index 2de725f..bd9287f 100644 --- a/inc/config.php +++ b/inc/config.php @@ -14,6 +14,7 @@ if (!is_dir('db/')) { @mkdir('db/',0705); } +define ('MODE_DEMO', FALSE); define ('ABS_PATH', 'assets/'); define ('CONVERT_LINKS_FOOTNOTES', TRUE); define ('REVERT_FORCED_PARAGRAPH_ELEMENTS',FALSE); diff --git a/inc/store/sqlite.class.php b/inc/store/sqlite.class.php index 4bfbb29..2108160 100644 --- a/inc/store/sqlite.class.php +++ b/inc/store/sqlite.class.php @@ -74,6 +74,13 @@ class Sqlite extends Store { return isset($pass[0]['value']) ? $pass[0]['value'] : FALSE; } + public function updatePassword($password) + { + $sql_update = "UPDATE config SET value=? WHERE name='password'"; + $params_update = array($password); + $query = $this->executeQuery($sql_update, $params_update); + } + private function executeQuery($sql, $params) { try { diff --git a/index.php b/index.php index 91d0b30..0a778d0 100644 --- a/index.php +++ b/index.php @@ -54,14 +54,17 @@ elseif (isset($_GET['config'])) { if (isset($_POST['password']) && isset($_POST['password_repeat'])) { if ($_POST['password'] == $_POST['password_repeat'] && $_POST['password'] != "") { logm('password updated'); - $store->updatePassword(encode_string($_POST['password'] . $_SESSION['login'])); - $msg->add('s', 'your password has been updated'); + if (!DEMO) { + $store->updatePassword(encode_string($_POST['password'] . $_SESSION['login'])); + $msg->add('s', 'your password has been updated'); + } + else { + $msg->add('i', 'in demo mode, you can\'t update password'); + } } else $msg->add('e', 'your password can\'t be empty and you have to repeat it in the second field'); } - else - $msg->add('e', 'error in your password update'); } # Traitement des paramètres et déclenchement des actions From 66ce09ba038baf7ab729384f05dfcb2bdda03ea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Wed, 31 Jul 2013 11:45:10 +0200 Subject: [PATCH 14/15] google groups link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 70a0764..7ce5d13 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The website of poche is [inthepoche.com](http://inthepoche.com). To test poche, a demo website is online : [demo.inthepoche.com](http://demo.inthepoche.com) (login poche, password poche). -To get news from poche, [follow us on twitter](http://twitter.com/getpoche) or [read the poche blog](http://inthepoche.com/blog). +To get news from poche, [follow us on twitter](http://twitter.com/getpoche) or [read the poche blog](http://inthepoche.com/blog). A Google Group is also available : [https://groups.google.com/forum/#!forum/poche-users](https://groups.google.com/forum/#!forum/poche-users) [![flattr](http://api.flattr.com/button/flattr-badge-large.png)](http://flattr.com/thing/1265480/poche-a-read-it-later-open-source-system) From 6d37de5f828b700843567a91418671919eed4be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Wed, 31 Jul 2013 11:50:38 +0200 Subject: [PATCH 15/15] typo in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ce5d13..b44e7d3 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The website of poche is [inthepoche.com](http://inthepoche.com). To test poche, a demo website is online : [demo.inthepoche.com](http://demo.inthepoche.com) (login poche, password poche). -To get news from poche, [follow us on twitter](http://twitter.com/getpoche) or [read the poche blog](http://inthepoche.com/blog). A Google Group is also available : [https://groups.google.com/forum/#!forum/poche-users](https://groups.google.com/forum/#!forum/poche-users) +To get news from poche, [follow us on twitter](http://twitter.com/getpoche) or [read the poche blog](http://inthepoche.com/blog). A Google Group is also available : [poche-users](https://groups.google.com/forum/#!forum/poche-users). [![flattr](http://api.flattr.com/button/flattr-badge-large.png)](http://flattr.com/thing/1265480/poche-a-read-it-later-open-source-system)