import without cron

inc/3rdparty/libraries/feedwriter/FeedWriter.php

@@ -90,15 +90,15 @@ define('JSONP', 3, true);
 	public function genarateFeed()
 	{
 		if ($this->version == RSS2) {
-			header('Content-type: text/xml; charset=UTF-8');
+// 			header('Content-type: text/xml; charset=UTF-8');
 			// this line prevents Chrome 20 from prompting download
 			// used by Google: https://news.google.com/news/feeds?ned=us&topic=b&output=rss
-			header('X-content-type-options: nosniff');
+// 			header('X-content-type-options: nosniff');
 		} elseif ($this->version == JSON) {
-			header('Content-type: application/json; charset=UTF-8');
+// 			header('Content-type: application/json; charset=UTF-8');
 			$this->json = new stdClass();
 		} elseif ($this->version == JSONP) {
-			header('Content-type: application/javascript; charset=UTF-8');
+// 			header('Content-type: application/javascript; charset=UTF-8');
 			$this->json = new stdClass();
 		}
 		$this->printHead();

inc/poche/Database.class.php

@@ -252,8 +252,16 @@ class Database {
         return $entries;
     }
 
+    public function retrieveUnfetchedEntriesCount($user_id) {
+      $sql        = "SELECT count(*) FROM entries WHERE (content = '' OR content IS NULL) AND user_id=?";
+      $query      = $this->executeQuery($sql, array($user_id));
+      list($count) = $query->fetch();
+
+      return $count;
+    }
+
     public function retrieveAll($user_id) {
-        $sql        = "SELECT * FROM entries WHERE content <> '' AND user_id=? ORDER BY id";
+        $sql        = "SELECT * FROM entries WHERE user_id=? ORDER BY id";
         $query      = $this->executeQuery($sql, array($user_id));
         $entries    = $query->fetchAll();
 
@@ -272,7 +280,7 @@ class Database {
 
     public function retrieveOneByURL($url, $user_id) {
         $entry  = NULL;
-        $sql    = "SELECT * FROM entries WHERE content <> '' AND url=? AND user_id=?";
+        $sql    = "SELECT * FROM entries WHERE url=? AND user_id=?";
         $params = array($url, $user_id);
         $query  = $this->executeQuery($sql, $params);
         $entry  = $query->fetchAll();
@@ -289,22 +297,21 @@ class Database {
     public function getEntriesByView($view, $user_id, $limit = '', $tag_id = 0) {
         switch ($view) {
             case 'archive':
-                $sql    = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
+                $sql    = "SELECT * FROM entries WHERE user_id=? AND is_read=? ";
                 $params = array($user_id, 1);
                 break;
             case 'fav' :
-                $sql    = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_fav=? ";
+                $sql    = "SELECT * FROM entries WHERE user_id=? AND is_fav=? ";
                 $params = array($user_id, 1);
                 break;
             case 'tag' :
                 $sql    = "SELECT entries.* FROM entries
                 LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id
-                WHERE entries.content <> '' AND
+                WHERE entries.user_id=? AND tags_entries.tag_id = ? ";
-                entries.user_id=? AND tags_entries.tag_id = ? ";
                 $params = array($user_id, $tag_id);
                 break;
             default:
-                $sql    = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
+                $sql    = "SELECT * FROM entries WHERE user_id=? AND is_read=? ";
                 $params = array($user_id, 0);
                 break;
         }
@@ -320,22 +327,21 @@ class Database {
     public function getEntriesByViewCount($view, $user_id, $tag_id = 0) {
         switch ($view) {
             case 'archive':
-                    $sql    = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
+                    $sql    = "SELECT count(*) FROM entries WHERE user_id=? AND is_read=? ";
                 $params = array($user_id, 1);
                 break;
             case 'fav' :
-                    $sql    = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_fav=? ";
+                    $sql    = "SELECT count(*) FROM entries WHERE user_id=? AND is_fav=? ";
                 $params = array($user_id, 1);
                 break;
             case 'tag' :
                 $sql    = "SELECT count(*) FROM entries
                     LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id
-                    WHERE entries.content <> '' AND
+                    WHERE entries.user_id=? AND tags_entries.tag_id = ? ";
-                    entries.user_id=? AND tags_entries.tag_id = ? ";
                 $params = array($user_id, $tag_id);
                 break;
             default:
-                $sql    = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
+                $sql    = "SELECT count(*) FROM entries WHERE user_id=? AND is_read=? ";
                 $params = array($user_id, 0);
                 break;
         }
@@ -353,11 +359,24 @@ class Database {
         return $query;
     }
 
-    public function add($url, $title, $content, $user_id) {
+    /**
-        $sql_action = 'INSERT INTO entries ( url, title, content, user_id ) VALUES (?, ?, ?, ?)';
+     *
-        $params_action = array($url, $title, $content, $user_id);
+     * @param string $url
-        $query = $this->executeQuery($sql_action, $params_action);
+     * @param string $title
-        return $query;
+     * @param string $content
+     * @param integer $user_id
+     * @return integer $id of inserted record
+     */
+    public function add($url, $title, $content, $user_id, $isFavorite=0, $isRead=0) {
+        $sql_action = 'INSERT INTO entries ( url, title, content, user_id, is_fav, is_read ) VALUES (?, ?, ?, ?, ?, ?)';
+        $params_action = array($url, $title, $content, $user_id, $isFavorite, $isRead);
+        if ( !$this->executeQuery($sql_action, $params_action) ) {
+          $id = null;
+        }
+        else {
+          $id = intval($this->getLastId( (STORAGE == 'postgres') ? 'users_id_seq' : '' ));
+        }
+        return $id;
     }
 
     public function deleteById($id, $user_id) {
@@ -401,7 +420,7 @@ class Database {
         $sql = "SELECT DISTINCT tags.*, count(entries.id) AS entriescount FROM tags
           LEFT JOIN tags_entries ON tags_entries.tag_id=tags.id
           LEFT JOIN entries ON tags_entries.entry_id=entries.id
-          WHERE entries.content <> '' AND entries.user_id=?
+          WHERE entries.user_id=?
             ". (($term) ? "AND lower(tags.value) LIKE ?" : '') ."
           GROUP BY tags.id, tags.value
           ORDER BY tags.value";
@@ -416,7 +435,7 @@ class Database {
         $sql    = "SELECT DISTINCT tags.* FROM tags
           LEFT JOIN tags_entries ON tags_entries.tag_id=tags.id
           LEFT JOIN entries ON tags_entries.entry_id=entries.id
-          WHERE entries.content <> '' AND tags.id=? AND entries.user_id=?";
+          WHERE tags.id=? AND entries.user_id=?";
         $params = array(intval($id), $user_id);
         $query  = $this->executeQuery($sql, $params);
         $tag  = $query->fetchAll();
@@ -428,8 +447,7 @@ class Database {
         $sql =
             "SELECT entries.* FROM entries
             LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id
-            WHERE entries.content <> '' AND
+            WHERE tags_entries.tag_id = ? AND entries.user_id=?";
-            tags_entries.tag_id = ? AND entries.user_id=?";
         $query = $this->executeQuery($sql, array($tag_id, $user_id));
         $entries = $query->fetchAll();
 

inc/poche/Poche.class.php

@@ -385,19 +385,15 @@ class Poche
                     $body = '';
                 }
 
-                //search for possible duplicate if not in import mode
+                //search for possible duplicate
                 $duplicate = NULL;
                 if (!$import) {
                   $duplicate = $this->store->retrieveOneByURL($url->getUrl(), $this->user->getId());
                 }
 
-                if ($this->store->add($url->getUrl(), $title, $body, $this->user->getId())) {
+                $last_id = $this->store->add($url->getUrl(), $title, $body, $this->user->getId());
+                if ( $last_id && !$import ) {
                     Tools::logm('add link ' . $url->getUrl());
-                    $sequence = '';
-                    if (STORAGE == 'postgres') {
-                        $sequence = 'entries_id_seq';
-                    }
-                    $last_id = $this->store->getLastId($sequence);
                     if (DOWNLOAD_PICTURES) {
                         $content = filtre_picture($body, $url->getUrl(), $last_id);
                         Tools::logm('updating content article');
@@ -417,10 +413,8 @@ class Poche
                         }
                     }
 
-                    if (!$import) {
                     $this->messages->add('s', _('the link has been added successfully'));
                 }
-                }
                 else {
                     if (!$import) {
                         $this->messages->add('e', _('error during insertion : the link wasn\'t added'));
@@ -1041,32 +1035,110 @@ class Poche
 
     /**
      * import datas into your poche
-     * @param  string $from name of the service to import : pocket, instapaper or readability
-     * @todo add the return value
      * @return boolean
      */
-    public function import($from)
+    public function import() {
-    {
-        $providers = array(
-            'pocket' => 'importFromPocket',
-            'readability' => 'importFromReadability',
-            'instapaper' => 'importFromInstapaper',
-            'poche' => 'importFromPoche',
-        );
 
-        if (! isset($providers[$from])) {
+      if ( isset($_FILES['file']) ) {
-            $this->messages->add('e', _('Unknown import provider.'));
+        // assume, that file is in json format
-            Tools::redirect();
+        $str_data = file_get_contents($_FILES['file']['tmp_name']);
+        $data = json_decode($str_data, true);
+
+        if ( $data === null ) {
+          //not json - assume html
+          $html = new simple_html_dom();
+          $html->load_file($_FILES['file']['tmp_name']);
+          $data = array();
+          $read = 0;
+          foreach (array('ol','ul') as $list) {
+            foreach ($html->find($list) as $ul) {
+            	foreach ($ul->find('li') as $li) {
+            	  $tmpEntry = array();
+            		$a = $li->find('a');
+            		$tmpEntry['url'] = $a[0]->href;
+            		$tmpEntry['tags'] = $a[0]->tags;
+            		$tmpEntry['is_read'] = $read;
+            		if ($tmpEntry['url']) {
+            		  $data[] = $tmpEntry;
+            		}
+            	}
+            	# the second <ol/ul> is for read links
+            	$read = ((sizeof($data) && $read)?0:1);
+            }
+          }
         }
 
-        $targetFile = CACHE . '/' . constant(strtoupper($from) . '_FILE');
+        $i = 0; //counter for articles inserted
+        foreach ($data as $record) {
+          //echo '<pre>';
+          //var_dump($record);
+  //         foreach ($record as $key=>$val) {
+  //           echo "\n=================\n$i: $key: $val\n";
+  //         }
+  //         exit;
 
-        if (! file_exists($targetFile)) {
+          $url = trim($record['url']);
-            $this->messages->add('e', _('Could not find required "' . $targetFile . '" import file.'));
+          if ( $url ) {
-            Tools::redirect();
+            $title = (isset($record['title']) ? $record['title'] :  _('Untitled - Import - ').'</a> <a href="./?import">'._('click to finish import').'</a><a>');
+            $body = (isset($record['content']) ? $record['content'] : '');
+            $isRead = (isset($record['is_read']) ? intval($record['is_read']) : 0);
+            $isFavorite = (isset($record['is_fav']) ? intval($record['is_fav']) : 0);
+            //insert new record
+            $id = $this->store->add($url, $title, $body, $this->user->getId(), $isFavorite, $isRead);
+            if ( $id ) {
+              //increment no of records inserted
+              $i++;
+              if ( isset($record['tags']) && trim($record['tags']) ) {
+              	//@TODO: set tags
+
+              }
+            }
+          }
         }
 
-        $this->$providers[$from]($targetFile);
+        if ( $i > 0 ) {
+          $this->messages->add('s', _('Articles inserted: ').$i._('. Please note, that some may be marked as "read".'));
+        }
+      }
+      //file parsing finished here
+
+      //now download article contents if any
+
+      //check if we need to download any content
+      $recordsDownloadRequired = $this->store->retrieveUnfetchedEntriesCount($this->user->getId());
+      if ( $recordsDownloadRequired == 0 ) {
+        //nothing to download
+        $this->messages->add('s', _('Import finished.'));
+        Tools::redirect();
+      }
+      else {
+        //if just inserted - don't download anything, download will start in next reload
+        if ( !isset($_FILES['file']) ) {
+          //download next batch
+          $items = $this->store->retrieveUnfetchedEntries($this->user->getId(), IMPORT_LIMIT);
+
+          $config = HTMLPurifier_Config::createDefault();
+          $config->set('Cache.SerializerPath', CACHE);
+          $purifier = new HTMLPurifier($config);
+
+          foreach ($items as $item) {
+          	$url = new Url(base64_encode($item['url']));
+          	$content = Tools::getPageContent($url);
+
+          	$title = (($content['rss']['channel']['item']['title'] != '') ? $content['rss']['channel']['item']['title'] : _('Untitled'));
+          	$body = (($content['rss']['channel']['item']['description'] != '') ? $content['rss']['channel']['item']['description'] : _('Undefined'));
+
+          	//clean content to prevent xss attack
+          	$title = $purifier->purify($title);
+          	$body = $purifier->purify($body);
+
+          	$this->store->updateContentAndTitle($item['id'], $title, $body, $this->user->getId());
+          }
+
+        }
+      }
+
+      return array('includeImport'=>true, 'import'=>array('recordsDownloadRequired'=>$recordsDownloadRequired, 'recordsUnderDownload'=> IMPORT_LIMIT, 'delay'=> IMPORT_DELAY * 1000) );
     }
 
     public function uploadFile() {

inc/poche/config.inc.php.new

@@ -52,12 +52,8 @@ define ('CACHE', ROOT . '/cache');
 
 define ('PAGINATION', '10');
 
-define ('POCKET_FILE', '/ril_export.html');
+//limit for download of articles during import
-define ('READABILITY_FILE', '/readability');
+define ('IMPORT_LIMIT', 5);
-define ('INSTAPAPER_FILE', '/instapaper-export.html');
+//delay between downloads (in sec)
-define ('POCHE_FILE', '/poche-export');
+define ('IMPORT_DELAY', 5);
 
-define ('IMPORT_POCKET_FILE', ROOT . POCKET_FILE);
-define ('IMPORT_READABILITY_FILE', ROOT . READABILITY_FILE);
-define ('IMPORT_INSTAPAPER_FILE', ROOT . INSTAPAPER_FILE);
-define ('IMPORT_POCHE_FILE', ROOT . POCHE_FILE);

index.php

@@ -67,7 +67,8 @@ if (isset($_GET['login'])) {
     # Update password
     $poche->updatePassword();
 } elseif (isset($_GET['import'])) {
-    $import = $poche->import($_GET['from']);
+    $import = $poche->import();
+    $tpl_vars = array_merge($tpl_vars, $import);
 } elseif (isset($_GET['download'])) {
     Tools::download_db();
 } elseif (isset($_GET['empty-cache'])) {

themes/default/_import.twig

@@ -0,0 +1,15 @@
+<script type="text/javascript">
+<!--
+  $(document).ready(function() {
+  	$("body").css("cursor", "wait");
+  	
+    setTimeout(function(){
+      window.location = '/?import';
+    }, {{ import.delay }} );
+  });
+//-->
+</script>
+<div class="messages warning">
+  <p>{% trans "Download required for " %} {{ import.recordsDownloadRequired }} {% trans "records" %}.</p>
+  <p>{% trans "Downloading next " %} {{ import.recordsUnderDownload }} {% trans "articles, please wait" %}...</p>
+</div>

themes/default/config.twig

@@ -104,36 +104,19 @@
             {% endif %}
 
             <h2>{% trans "Import" %}</h2>
-            <p>{% trans "Importing from other services can be quite long, and webservers default configuration often prevents long scripts execution time, so it must be done in multiple parts." %}</p>
+            <p>{% trans "You can import your Pocket, Readability, Instapaper, Wallabag or any data in appropriate json or html format." %}</p>
-            <p>1. {% trans "First, select the export file on your computer and upload it." %}</p>
+            <p>{% trans "Please select export file on your computer:" %}</p>
-            <form method="post" action="?uploadfile" name="uploadfile" enctype="multipart/form-data">
+            <form method="post" action="?import" name="uploadfile" enctype="multipart/form-data">
                 <fieldset class="w500p">
                     <div class="row">
                         <label class="col w150p" for="file">{% trans "File:" %}</label>
                         <input class="col" type="file" id="file" name="file" tabindex="4">
                     </div>
                     <div class="row mts txtcenter">
-                        <button class="bouton" type="submit" tabindex="4">{% trans "Upload" %}</button>
+                        <button class="bouton" type="submit" tabindex="4">{% trans "Import" %}</button>
                     </div>
                 </fieldset>
-                <input type="hidden" name="MAX_FILE_SIZE" value="1048576">
-                <input type="hidden" name="returnurl" value="{{ referer }}">
             </form>
-            <p>2. {% trans "Then, click on the right link below." %}</p>
-            <ul>
-                <li><a href="./?import&amp;from=pocket">{% trans "Import from Pocket" %}</a> {{ '(after uploaded %s file)'|trans|format(constant('POCKET_FILE')) }}</li>
-                <li><a href="./?import&amp;from=readability">{% trans "Import from Readability" %}</a> {{ '(after uploaded %s file)'|trans|format(constant('READABILITY_FILE')) }}</li>
-                <li><a href="./?import&amp;from=instapaper">{% trans "Import from Instapaper" %}</a> {{ '(after uploaded %s file)'|trans|format(constant('INSTAPAPER_FILE')) }}</li>
-                <li><a href="./?import&amp;from=poche">{% trans "Import from wallabag" %}</a> {{ '(after uploaded %s file)'|trans|format(constant('POCHE_FILE')) }}</li>
-            </ul>
-
-            {% if token == '' %}
-                <p>{% trans "3. Your feed token is currently empty and must first be generated to fetch content. Click <a href='?feed&amp;action=generate'>here to generate it</a>." %}</p>
-            {% else %}
-            <p>3. {% trans "Finally, you have to fetch content for imported items." %} <a href="cron.php?limit=10&amp;user-id={{ user_id }}&amp;token={{token}}" target="_blank">{% trans "Click here" %}</a> {% trans "to fetch content for 10 articles" %}.</p>
-            <p>{% trans "If you have console access to your server, you can also create a cron task:" %}</p>
-            <pre><code>0 */4 * * *  cd /path/to/wallabag && php cron.php --limit=10 --user-id={{user_id}} --token={{token}} >/dev/null 2>&1</code></pre>
-            {% endif %}
 
             <h2>{% trans "Export your wallabag data" %}</h2>
             {% if constant('STORAGE') == 'sqlite' %}

themes/default/home.twig

@@ -16,6 +16,11 @@
 {% include '_sorting.twig' %}
 {% endblock %}
 {% block content %}
+
+						{% if includeImport %}
+								{% include '_import.twig' %}
+						{% endif %}
+
             {% if tag %}
                 <h3>{% trans "Tag" %}: <b>{{ tag.value }}</b></h3>
             {% endif %}