[add] cron to fetch content on imported entries

cron.php

@@ -0,0 +1,53 @@
+<?php
+
+include_once 'inc/poche/global.inc.php';
+include_once 'inc/poche/config.inc.php';
+
+if (php_sapi_name() === 'cli') {
+    $options_cli = getopt('', array(
+        'limit::',
+        'user-id::',
+        'token::',
+    ));
+}
+else {
+    $options_cli = $_GET;
+}
+
+$limit = ! empty($options_cli['limit']) && ctype_digit($options_cli['limit']) ? (int) $options_cli['limit'] : 10;
+$user_id = ! empty($options_cli['user-id']) && ctype_digit($options_cli['user-id']) ? (int) $options_cli['user-id'] : null;
+$token = ! empty($options_cli['token']) ? $options_cli['token'] : null;
+
+if (is_null($user_id)) {
+    die('You must give a user id');
+}
+
+if (is_null($token)) {
+    die('You must give a token');
+}
+
+$store = new Database();
+$config = $store->getConfigUser($user_id);
+
+if ($token != $config['token']) {
+    die(_('Uh, there is a problem with the cron.'));
+}
+
+$items = $store->retrieveUnfetchedEntries($user_id, $limit);
+
+foreach ($items as $item) {
+    $url = new Url(base64_encode($item['url']));
+    $content = Tools::getPageContent($url);
+
+    $title = ($content['rss']['channel']['item']['title'] != '') ? $content['rss']['channel']['item']['title'] : _('Untitled');
+    $body = $content['rss']['channel']['item']['description'];
+
+    // // clean content from prevent xss attack
+    // $config = HTMLPurifier_Config::createDefault();
+    // $purifier = new HTMLPurifier($config);
+    // $title = $purifier->purify($title);
+    // $body = $purifier->purify($body);
+
+
+    $store->updateContentAndTitle($item['id'], $title, $body, $user_id);
+}

inc/poche/Database.class.php

@@ -230,8 +230,30 @@ class Database {
         }
     }
 
+    public function updateContentAndTitle($id, $title, $body, $user_id) {
+        $sql_action = 'UPDATE entries SET content = ?, title = ? WHERE id=? AND user_id=?';
+        $params_action = array($body, $title, $id, $user_id);
+        $query = $this->executeQuery($sql_action, $params_action);
+
+        return $query;
+    }
+
+    public function retrieveUnfetchedEntries($user_id, $limit) {
+
+        $sql_limit = "LIMIT 0,".$limit;
+        if (STORAGE == 'postgres') {
+            $sql_limit = "LIMIT ".$limit." OFFSET 0";
+        }
+
+        $sql        = "SELECT * FROM entries WHERE (content = '' OR content IS NULL) AND user_id=? ORDER BY id " . $sql_limit;
+        $query      = $this->executeQuery($sql, array($user_id));
+        $entries    = $query->fetchAll();
+
+        return $entries;
+    }
+
     public function retrieveAll($user_id) {
-        $sql        = "SELECT * FROM entries WHERE user_id=? ORDER BY id";
+        $sql        = "SELECT * FROM entries WHERE content <> '' AND user_id=? ORDER BY id";
         $query      = $this->executeQuery($sql, array($user_id));
         $entries    = $query->fetchAll();
 
@@ -250,7 +272,7 @@ class Database {
 
     public function retrieveOneByURL($url, $user_id) {
         $entry  = NULL;
-        $sql    = "SELECT * FROM entries WHERE url=? AND user_id=?";
+        $sql    = "SELECT * FROM entries WHERE content <> '' AND url=? AND user_id=?";
         $params = array($url, $user_id);
         $query  = $this->executeQuery($sql, $params);
         $entry  = $query->fetchAll();
@@ -267,21 +289,22 @@ class Database {
     public function getEntriesByView($view, $user_id, $limit = '', $tag_id = 0) {
         switch ($view) {
             case 'archive':
-                $sql    = "SELECT * FROM entries WHERE user_id=? AND is_read=? ";
+                $sql    = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
                 $params = array($user_id, 1);
                 break;
             case 'fav' :
-                $sql    = "SELECT * FROM entries WHERE user_id=? AND is_fav=? ";
+                $sql    = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_fav=? ";
                 $params = array($user_id, 1);
                 break;
             case 'tag' :
                 $sql    = "SELECT entries.* FROM entries
                 LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id
-                WHERE entries.user_id=? AND tags_entries.tag_id = ? ";
+                WHERE entries.content <> '' AND
+                entries.user_id=? AND tags_entries.tag_id = ? ";
                 $params = array($user_id, $tag_id);
                 break;
             default:
-                $sql    = "SELECT * FROM entries WHERE user_id=? AND is_read=? ";
+                $sql    = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
                 $params = array($user_id, 0);
                 break;
         }
@@ -294,24 +317,25 @@ class Database {
                 return $entries;
         }
 
-    public function getEntriesByViewCount($view, $user_id, $tag_id = 0) {
+    public function getEntriesByViewCount($view, $user_id, $tag_id = 0) {
-        switch ($view) {
+        switch ($view) {
             case 'archive':
-                    $sql    = "SELECT count(*) FROM entries WHERE user_id=? AND is_read=? ";
+                    $sql    = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
                 $params = array($user_id, 1);
                 break;
             case 'fav' :
-                    $sql    = "SELECT count(*) FROM entries WHERE user_id=? AND is_fav=? ";
+                    $sql    = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_fav=? ";
                 $params = array($user_id, 1);
                 break;
-            case 'tag' :
+            case 'tag' :
-                $sql    = "SELECT count(*) FROM entries
+                $sql    = "SELECT count(*) FROM entries
-                    LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id
+                    LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id
-                    WHERE entries.user_id=? AND tags_entries.tag_id = ? ";
+                    WHERE entries.content <> '' AND
-                $params = array($user_id, $tag_id);
+                    entries.user_id=? AND tags_entries.tag_id = ? ";
-                break;
+                $params = array($user_id, $tag_id);
+                break;
             default:
-                $sql    = "SELECT count(*) FROM entries WHERE user_id=? AND is_read=? ";
+                $sql    = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
                 $params = array($user_id, 0);
                 break;
         }
@@ -319,7 +343,7 @@ class Database {
         $query = $this->executeQuery($sql, $params);
         list($count) = $query->fetch();
 
-        return $count;
+        return $count;
     }
 
     public function updateContent($id, $content, $user_id) {
@@ -369,7 +393,7 @@ class Database {
         $sql = "SELECT DISTINCT tags.* FROM tags
           LEFT JOIN tags_entries ON tags_entries.tag_id=tags.id
           LEFT JOIN entries ON tags_entries.entry_id=entries.id
-          WHERE entries.user_id=?";
+          WHERE entries.content <> '' AND entries.user_id=?";
         $query = $this->executeQuery($sql, array($user_id));
         $tags = $query->fetchAll();
 
@@ -381,7 +405,7 @@ class Database {
         $sql    = "SELECT DISTINCT tags.* FROM tags
           LEFT JOIN tags_entries ON tags_entries.tag_id=tags.id
           LEFT JOIN entries ON tags_entries.entry_id=entries.id
-          WHERE tags.id=? AND entries.user_id=?";
+          WHERE entries.content <> '' AND tags.id=? AND entries.user_id=?";
         $params = array(intval($id), $user_id);
         $query  = $this->executeQuery($sql, $params);
         $tag  = $query->fetchAll();
@@ -393,7 +417,8 @@ class Database {
         $sql = 
             "SELECT entries.* FROM entries
             LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id
-            WHERE tags_entries.tag_id = ? AND entries.user_id=?";
+            WHERE entries.content <> '' AND
+            tags_entries.tag_id = ? AND entries.user_id=?";
         $query = $this->executeQuery($sql, array($tag_id, $user_id));
         $entries = $query->fetchAll();
 

inc/poche/Poche.class.php

@@ -362,60 +362,6 @@ class Poche
         );
     }
 
-    protected function getPageContent(Url $url)
-    {
-        // Saving and clearing context
-        $REAL = array();
-        foreach( $GLOBALS as $key => $value ) {
-            if( $key != 'GLOBALS' && $key != '_SESSION' && $key != 'HTTP_SESSION_VARS' ) {
-                $GLOBALS[$key] = array();
-                $REAL[$key] = $value;
-            }
-        }
-        // Saving and clearing session
-        $REAL_SESSION = array();
-        foreach( $_SESSION as $key => $value ) {
-            $REAL_SESSION[$key] = $value;
-            unset($_SESSION[$key]);
-        }
-
-        // Running code in different context
-        $scope = function() {
-            extract( func_get_arg(1) );
-            $_GET = $_REQUEST = array(
-                        "url" => $url->getUrl(),
-                        "max" => 5,
-                        "links" => "preserve",
-                        "exc" => "",
-                        "format" => "json",
-                        "submit" => "Create Feed"
-            );
-            ob_start();
-            require func_get_arg(0);
-            $json = ob_get_flush();
-            return $json;
-        };
-        $json = $scope( "inc/3rdparty/makefulltextfeed.php", array("url" => $url) );
-
-        // Clearing and restoring context
-        foreach( $GLOBALS as $key => $value ) {
-            if( $key != "GLOBALS" && $key != "_SESSION" ) {
-                unset($GLOBALS[$key]);
-            }
-        }
-        foreach( $REAL as $key => $value ) {
-            $GLOBALS[$key] = $value;
-        }
-        // Clearing and restoring session
-        foreach( $_SESSION as $key => $value ) {
-            unset($_SESSION[$key]);
-        }
-        foreach( $REAL_SESSION as $key => $value ) {
-            $_SESSION[$key] = $value;
-        }
-        return json_decode($json, true);
-    }
-
     /**
      * Call action (mark as fav, archive, delete, etc.)
      */
@@ -424,15 +370,21 @@ class Poche
         switch ($action)
         {
             case 'add':
-                $content = $this->getPageContent($url);
+                if (!$import) {
-                $title = ($content['rss']['channel']['item']['title'] != '') ? $content['rss']['channel']['item']['title'] : _('Untitled');
+                    $content = Tools::getPageContent($url);
-                $body = $content['rss']['channel']['item']['description'];
+                    $title = ($content['rss']['channel']['item']['title'] != '') ? $content['rss']['channel']['item']['title'] : _('Untitled');
+                    $body = $content['rss']['channel']['item']['description'];
 
-                // clean content from prevent xss attack
+                    // clean content from prevent xss attack
-                $config = HTMLPurifier_Config::createDefault();
+                    $config = HTMLPurifier_Config::createDefault();
-                $purifier = new HTMLPurifier($config);
+                    $purifier = new HTMLPurifier($config);
-                $title = $purifier->purify($title);
+                    $title = $purifier->purify($title);
-                $body = $purifier->purify($body);
+                    $body = $purifier->purify($body);
+                }
+                else {
+                    $title = '';
+                    $body = '';
+                }
 
                 //search for possible duplicate if not in import mode
                 if (!$import) {
@@ -897,7 +849,7 @@ class Poche
             # the second <ol> is for read links
             $read = 1;
         }
-        $this->messages->add('s', _('import from instapaper completed'));
+        $this->messages->add('s', _('import from instapaper completed. You have to execute the cron to fetch content.'));
         Tools::logm('import from instapaper completed');
         Tools::redirect();
     }
@@ -941,7 +893,7 @@ class Poche
             # the second <ul> is for read links
             $read = 1;
         }
-        $this->messages->add('s', _('import from pocket completed'));
+        $this->messages->add('s', _('import from pocket completed. You have to execute the cron to fetch content.'));
         Tools::logm('import from pocket completed');
         Tools::redirect();
     }
@@ -997,7 +949,7 @@ class Poche
                 }
             }
         }
-        $this->messages->add('s', _('import from Readability completed. ' . $count . ' new links.'));
+        $this->messages->add('s', _('import from Readability completed. You have to execute the cron to fetch content.'));
         Tools::logm('import from Readability completed');
         Tools::redirect();
     }
@@ -1043,7 +995,7 @@ class Poche
             }
             
         }
-        $this->messages->add('s', _('import from Poche completed. ' . $count . ' new links.'));
+        $this->messages->add('s', _('import from Poche completed. You have to execute the cron to fetch content.'));
         Tools::logm('import from Poche completed');
         Tools::redirect();
     }

inc/poche/Tools.class.php

@@ -193,7 +193,7 @@ class Tools
 
     public static function logm($message)
     {
-        if (DEBUG_POCHE) {
+        if (DEBUG_POCHE && php_sapi_name() != 'cli') {
             $t = strval(date('Y/m/d_H:i:s')) . ' - ' . $_SERVER["REMOTE_ADDR"] . ' - ' . strval($message) . "\n";
             file_put_contents(CACHE . '/log.txt', $t, FILE_APPEND);
             error_log('DEBUG POCHE : ' . $message);
@@ -251,4 +251,58 @@ class Tools
 
         exit;
     }
+
+    public static function getPageContent(Url $url)
+    {
+        // Saving and clearing context
+        $REAL = array();
+        foreach( $GLOBALS as $key => $value ) {
+            if( $key != 'GLOBALS' && $key != '_SESSION' && $key != 'HTTP_SESSION_VARS' ) {
+                $GLOBALS[$key] = array();
+                $REAL[$key] = $value;
+            }
+        }
+        // Saving and clearing session
+        $REAL_SESSION = array();
+        foreach( $_SESSION as $key => $value ) {
+            $REAL_SESSION[$key] = $value;
+            unset($_SESSION[$key]);
+        }
+
+        // Running code in different context
+        $scope = function() {
+            extract( func_get_arg(1) );
+            $_GET = $_REQUEST = array(
+                        "url" => $url->getUrl(),
+                        "max" => 5,
+                        "links" => "preserve",
+                        "exc" => "",
+                        "format" => "json",
+                        "submit" => "Create Feed"
+            );
+            ob_start();
+            require func_get_arg(0);
+            $json = ob_get_flush();
+            return $json;
+        };
+        $json = $scope( "inc/3rdparty/makefulltextfeed.php", array("url" => $url) );
+
+        // Clearing and restoring context
+        foreach( $GLOBALS as $key => $value ) {
+            if( $key != "GLOBALS" && $key != "_SESSION" ) {
+                unset($GLOBALS[$key]);
+            }
+        }
+        foreach( $REAL as $key => $value ) {
+            $GLOBALS[$key] = $value;
+        }
+        // Clearing and restoring session
+        foreach( $_SESSION as $key => $value ) {
+            unset($_SESSION[$key]);
+        }
+        foreach( $REAL_SESSION as $key => $value ) {
+            $_SESSION[$key] = $value;
+        }
+        return json_decode($json, true);
+    }
 }

themes/baggy/config.twig

@@ -125,6 +125,14 @@
                 <li><a href="./?import&amp;from=poche">{% trans "Import from wallabag" %}</a> {{ '(after uploaded %s file)'|trans|format(constant('POCHE_FILE')) }}</li>
             </ul>
 
+            {% if token == '' %}
+                <p>{% trans "3. Your feed token is currently empty and must first be generated to fetch content. Click <a href='?feed&amp;action=generate'>here to generate it</a>." %}</p>
+            {% else %}
+            <p>3. {% trans "You can fetch content for imported items." %} <a href="cron.php?limit=10&amp;user-id={{ user_id }}&amp;token={{token}}" target="_blank">Click here</a> to fetch content for 10 articles.</p>
+            <p>{% trans "You can also create a cron task:" %}</p>
+            <pre><code>0 */4 * * *  cd /path/to/wallabag && php cron.php --limit=10 --user-id={{user_id}} --token={{token}} >/dev/null 2>&1</code></pre>
+            {% endif %}
+
             <h2>{% trans "Export your wallabag data" %}</h2>
             {% if constant('STORAGE') == 'sqlite' %}
             <p><a href="?download" target="_blank">{% trans "Click here" %}</a> {% trans "to download your database." %}</p>{% endif %}