mirror of
https://github.com/moparisthebest/wallabag
synced 2024-11-27 11:22:17 -05:00
import without cron
This commit is contained in:
Maryana Rozhankivska
parent
22db488d21
commit
182faf2696
8
inc/3rdparty/libraries/feedwriter/FeedWriter.php
vendored
Normal file → Executable file
8
inc/3rdparty/libraries/feedwriter/FeedWriter.php
vendored
Normal file → Executable file
@ -90,15 +90,15 @@ define('JSONP', 3, true);
|
||||
public function genarateFeed()
|
||||
{
|
||||
if ($this->version == RSS2) {
|
||||
header('Content-type: text/xml; charset=UTF-8');
|
||||
// header('Content-type: text/xml; charset=UTF-8');
|
||||
// this line prevents Chrome 20 from prompting download
|
||||
// used by Google: https://news.google.com/news/feeds?ned=us&topic=b&output=rss
|
||||
header('X-content-type-options: nosniff');
|
||||
// header('X-content-type-options: nosniff');
|
||||
} elseif ($this->version == JSON) {
|
||||
header('Content-type: application/json; charset=UTF-8');
|
||||
// header('Content-type: application/json; charset=UTF-8');
|
||||
$this->json = new stdClass();
|
||||
} elseif ($this->version == JSONP) {
|
||||
header('Content-type: application/javascript; charset=UTF-8');
|
||||
// header('Content-type: application/javascript; charset=UTF-8');
|
||||
$this->json = new stdClass();
|
||||
}
|
||||
$this->printHead();
|
||||
|
||||
@ -252,8 +252,16 @@ class Database {
|
||||
return $entries;
|
||||
}
|
||||
|
||||
public function retrieveUnfetchedEntriesCount($user_id) {
|
||||
$sql = "SELECT count(*) FROM entries WHERE (content = '' OR content IS NULL) AND user_id=?";
|
||||
$query = $this->executeQuery($sql, array($user_id));
|
||||
list($count) = $query->fetch();
|
||||
|
||||
return $count;
|
||||
}
|
||||
|
||||
public function retrieveAll($user_id) {
|
||||
$sql = "SELECT * FROM entries WHERE content <> '' AND user_id=? ORDER BY id";
|
||||
$sql = "SELECT * FROM entries WHERE user_id=? ORDER BY id";
|
||||
$query = $this->executeQuery($sql, array($user_id));
|
||||
$entries = $query->fetchAll();
|
||||
|
||||
@ -272,7 +280,7 @@ class Database {
|
||||
|
||||
public function retrieveOneByURL($url, $user_id) {
|
||||
$entry = NULL;
|
||||
$sql = "SELECT * FROM entries WHERE content <> '' AND url=? AND user_id=?";
|
||||
$sql = "SELECT * FROM entries WHERE url=? AND user_id=?";
|
||||
$params = array($url, $user_id);
|
||||
$query = $this->executeQuery($sql, $params);
|
||||
$entry = $query->fetchAll();
|
||||
@ -289,22 +297,21 @@ class Database {
|
||||
public function getEntriesByView($view, $user_id, $limit = '', $tag_id = 0) {
|
||||
switch ($view) {
|
||||
case 'archive':
|
||||
$sql = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
|
||||
$sql = "SELECT * FROM entries WHERE user_id=? AND is_read=? ";
|
||||
$params = array($user_id, 1);
|
||||
break;
|
||||
case 'fav' :
|
||||
$sql = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_fav=? ";
|
||||
$sql = "SELECT * FROM entries WHERE user_id=? AND is_fav=? ";
|
||||
$params = array($user_id, 1);
|
||||
break;
|
||||
case 'tag' :
|
||||
$sql = "SELECT entries.* FROM entries
|
||||
LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id
|
||||
WHERE entries.content <> '' AND
|
||||
entries.user_id=? AND tags_entries.tag_id = ? ";
|
||||
WHERE entries.user_id=? AND tags_entries.tag_id = ? ";
|
||||
$params = array($user_id, $tag_id);
|
||||
break;
|
||||
default:
|
||||
$sql = "SELECT * FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
|
||||
$sql = "SELECT * FROM entries WHERE user_id=? AND is_read=? ";
|
||||
$params = array($user_id, 0);
|
||||
break;
|
||||
}
|
||||
@ -320,22 +327,21 @@ class Database {
|
||||
public function getEntriesByViewCount($view, $user_id, $tag_id = 0) {
|
||||
switch ($view) {
|
||||
case 'archive':
|
||||
$sql = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
|
||||
$sql = "SELECT count(*) FROM entries WHERE user_id=? AND is_read=? ";
|
||||
$params = array($user_id, 1);
|
||||
break;
|
||||
case 'fav' :
|
||||
$sql = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_fav=? ";
|
||||
$sql = "SELECT count(*) FROM entries WHERE user_id=? AND is_fav=? ";
|
||||
$params = array($user_id, 1);
|
||||
break;
|
||||
case 'tag' :
|
||||
$sql = "SELECT count(*) FROM entries
|
||||
LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id
|
||||
WHERE entries.content <> '' AND
|
||||
entries.user_id=? AND tags_entries.tag_id = ? ";
|
||||
WHERE entries.user_id=? AND tags_entries.tag_id = ? ";
|
||||
$params = array($user_id, $tag_id);
|
||||
break;
|
||||
default:
|
||||
$sql = "SELECT count(*) FROM entries WHERE content <> '' AND user_id=? AND is_read=? ";
|
||||
$sql = "SELECT count(*) FROM entries WHERE user_id=? AND is_read=? ";
|
||||
$params = array($user_id, 0);
|
||||
break;
|
||||
}
|
||||
@ -353,11 +359,24 @@ class Database {
|
||||
return $query;
|
||||
}
|
||||
|
||||
public function add($url, $title, $content, $user_id) {
|
||||
$sql_action = 'INSERT INTO entries ( url, title, content, user_id ) VALUES (?, ?, ?, ?)';
|
||||
$params_action = array($url, $title, $content, $user_id);
|
||||
$query = $this->executeQuery($sql_action, $params_action);
|
||||
return $query;
|
||||
/**
|
||||
*
|
||||
* @param string $url
|
||||
* @param string $title
|
||||
* @param string $content
|
||||
* @param integer $user_id
|
||||
* @return integer $id of inserted record
|
||||
*/
|
||||
public function add($url, $title, $content, $user_id, $isFavorite=0, $isRead=0) {
|
||||
$sql_action = 'INSERT INTO entries ( url, title, content, user_id, is_fav, is_read ) VALUES (?, ?, ?, ?, ?, ?)';
|
||||
$params_action = array($url, $title, $content, $user_id, $isFavorite, $isRead);
|
||||
if ( !$this->executeQuery($sql_action, $params_action) ) {
|
||||
$id = null;
|
||||
}
|
||||
else {
|
||||
$id = intval($this->getLastId( (STORAGE == 'postgres') ? 'users_id_seq' : '' ));
|
||||
}
|
||||
return $id;
|
||||
}
|
||||
|
||||
public function deleteById($id, $user_id) {
|
||||
@ -401,7 +420,7 @@ class Database {
|
||||
$sql = "SELECT DISTINCT tags.*, count(entries.id) AS entriescount FROM tags
|
||||
LEFT JOIN tags_entries ON tags_entries.tag_id=tags.id
|
||||
LEFT JOIN entries ON tags_entries.entry_id=entries.id
|
||||
WHERE entries.content <> '' AND entries.user_id=?
|
||||
WHERE entries.user_id=?
|
||||
". (($term) ? "AND lower(tags.value) LIKE ?" : '') ."
|
||||
GROUP BY tags.id, tags.value
|
||||
ORDER BY tags.value";
|
||||
@ -416,7 +435,7 @@ class Database {
|
||||
$sql = "SELECT DISTINCT tags.* FROM tags
|
||||
LEFT JOIN tags_entries ON tags_entries.tag_id=tags.id
|
||||
LEFT JOIN entries ON tags_entries.entry_id=entries.id
|
||||
WHERE entries.content <> '' AND tags.id=? AND entries.user_id=?";
|
||||
WHERE tags.id=? AND entries.user_id=?";
|
||||
$params = array(intval($id), $user_id);
|
||||
$query = $this->executeQuery($sql, $params);
|
||||
$tag = $query->fetchAll();
|
||||
@ -428,8 +447,7 @@ class Database {
|
||||
$sql =
|
||||
"SELECT entries.* FROM entries
|
||||
LEFT JOIN tags_entries ON tags_entries.entry_id=entries.id
|
||||
WHERE entries.content <> '' AND
|
||||
tags_entries.tag_id = ? AND entries.user_id=?";
|
||||
WHERE tags_entries.tag_id = ? AND entries.user_id=?";
|
||||
$query = $this->executeQuery($sql, array($tag_id, $user_id));
|
||||
$entries = $query->fetchAll();
|
||||
|
||||
|
||||
@ -385,19 +385,15 @@ class Poche
|
||||
$body = '';
|
||||
}
|
||||
|
||||
//search for possible duplicate if not in import mode
|
||||
//search for possible duplicate
|
||||
$duplicate = NULL;
|
||||
if (!$import) {
|
||||
$duplicate = $this->store->retrieveOneByURL($url->getUrl(), $this->user->getId());
|
||||
}
|
||||
|
||||
if ($this->store->add($url->getUrl(), $title, $body, $this->user->getId())) {
|
||||
$last_id = $this->store->add($url->getUrl(), $title, $body, $this->user->getId());
|
||||
if ( $last_id && !$import ) {
|
||||
Tools::logm('add link ' . $url->getUrl());
|
||||
$sequence = '';
|
||||
if (STORAGE == 'postgres') {
|
||||
$sequence = 'entries_id_seq';
|
||||
}
|
||||
$last_id = $this->store->getLastId($sequence);
|
||||
if (DOWNLOAD_PICTURES) {
|
||||
$content = filtre_picture($body, $url->getUrl(), $last_id);
|
||||
Tools::logm('updating content article');
|
||||
@ -417,10 +413,8 @@ class Poche
|
||||
}
|
||||
}
|
||||
|
||||
if (!$import) {
|
||||
$this->messages->add('s', _('the link has been added successfully'));
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (!$import) {
|
||||
$this->messages->add('e', _('error during insertion : the link wasn\'t added'));
|
||||
@ -1041,32 +1035,110 @@ class Poche
|
||||
|
||||
/**
|
||||
* import datas into your poche
|
||||
* @param string $from name of the service to import : pocket, instapaper or readability
|
||||
* @todo add the return value
|
||||
* @return boolean
|
||||
*/
|
||||
public function import($from)
|
||||
{
|
||||
$providers = array(
|
||||
'pocket' => 'importFromPocket',
|
||||
'readability' => 'importFromReadability',
|
||||
'instapaper' => 'importFromInstapaper',
|
||||
'poche' => 'importFromPoche',
|
||||
);
|
||||
public function import() {
|
||||
|
||||
if (! isset($providers[$from])) {
|
||||
$this->messages->add('e', _('Unknown import provider.'));
|
||||
Tools::redirect();
|
||||
if ( isset($_FILES['file']) ) {
|
||||
// assume, that file is in json format
|
||||
$str_data = file_get_contents($_FILES['file']['tmp_name']);
|
||||
$data = json_decode($str_data, true);
|
||||
|
||||
if ( $data === null ) {
|
||||
//not json - assume html
|
||||
$html = new simple_html_dom();
|
||||
$html->load_file($_FILES['file']['tmp_name']);
|
||||
$data = array();
|
||||
$read = 0;
|
||||
foreach (array('ol','ul') as $list) {
|
||||
foreach ($html->find($list) as $ul) {
|
||||
foreach ($ul->find('li') as $li) {
|
||||
$tmpEntry = array();
|
||||
$a = $li->find('a');
|
||||
$tmpEntry['url'] = $a[0]->href;
|
||||
$tmpEntry['tags'] = $a[0]->tags;
|
||||
$tmpEntry['is_read'] = $read;
|
||||
if ($tmpEntry['url']) {
|
||||
$data[] = $tmpEntry;
|
||||
}
|
||||
}
|
||||
# the second <ol/ul> is for read links
|
||||
$read = ((sizeof($data) && $read)?0:1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$targetFile = CACHE . '/' . constant(strtoupper($from) . '_FILE');
|
||||
$i = 0; //counter for articles inserted
|
||||
foreach ($data as $record) {
|
||||
//echo '<pre>';
|
||||
//var_dump($record);
|
||||
// foreach ($record as $key=>$val) {
|
||||
// echo "\n=================\n$i: $key: $val\n";
|
||||
// }
|
||||
// exit;
|
||||
|
||||
if (! file_exists($targetFile)) {
|
||||
$this->messages->add('e', _('Could not find required "' . $targetFile . '" import file.'));
|
||||
Tools::redirect();
|
||||
$url = trim($record['url']);
|
||||
if ( $url ) {
|
||||
$title = (isset($record['title']) ? $record['title'] : _('Untitled - Import - ').'</a> <a href="./?import">'._('click to finish import').'</a><a>');
|
||||
$body = (isset($record['content']) ? $record['content'] : '');
|
||||
$isRead = (isset($record['is_read']) ? intval($record['is_read']) : 0);
|
||||
$isFavorite = (isset($record['is_fav']) ? intval($record['is_fav']) : 0);
|
||||
//insert new record
|
||||
$id = $this->store->add($url, $title, $body, $this->user->getId(), $isFavorite, $isRead);
|
||||
if ( $id ) {
|
||||
//increment no of records inserted
|
||||
$i++;
|
||||
if ( isset($record['tags']) && trim($record['tags']) ) {
|
||||
//@TODO: set tags
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$this->$providers[$from]($targetFile);
|
||||
if ( $i > 0 ) {
|
||||
$this->messages->add('s', _('Articles inserted: ').$i._('. Please note, that some may be marked as "read".'));
|
||||
}
|
||||
}
|
||||
//file parsing finished here
|
||||
|
||||
//now download article contents if any
|
||||
|
||||
//check if we need to download any content
|
||||
$recordsDownloadRequired = $this->store->retrieveUnfetchedEntriesCount($this->user->getId());
|
||||
if ( $recordsDownloadRequired == 0 ) {
|
||||
//nothing to download
|
||||
$this->messages->add('s', _('Import finished.'));
|
||||
Tools::redirect();
|
||||
}
|
||||
else {
|
||||
//if just inserted - don't download anything, download will start in next reload
|
||||
if ( !isset($_FILES['file']) ) {
|
||||
//download next batch
|
||||
$items = $this->store->retrieveUnfetchedEntries($this->user->getId(), IMPORT_LIMIT);
|
||||
|
||||
$config = HTMLPurifier_Config::createDefault();
|
||||
$config->set('Cache.SerializerPath', CACHE);
|
||||
$purifier = new HTMLPurifier($config);
|
||||
|
||||
foreach ($items as $item) {
|
||||
$url = new Url(base64_encode($item['url']));
|
||||
$content = Tools::getPageContent($url);
|
||||
|
||||
$title = (($content['rss']['channel']['item']['title'] != '') ? $content['rss']['channel']['item']['title'] : _('Untitled'));
|
||||
$body = (($content['rss']['channel']['item']['description'] != '') ? $content['rss']['channel']['item']['description'] : _('Undefined'));
|
||||
|
||||
//clean content to prevent xss attack
|
||||
$title = $purifier->purify($title);
|
||||
$body = $purifier->purify($body);
|
||||
|
||||
$this->store->updateContentAndTitle($item['id'], $title, $body, $this->user->getId());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return array('includeImport'=>true, 'import'=>array('recordsDownloadRequired'=>$recordsDownloadRequired, 'recordsUnderDownload'=> IMPORT_LIMIT, 'delay'=> IMPORT_DELAY * 1000) );
|
||||
}
|
||||
|
||||
public function uploadFile() {
|
||||
|
||||
@ -52,12 +52,8 @@ define ('CACHE', ROOT . '/cache');
|
||||
|
||||
define ('PAGINATION', '10');
|
||||
|
||||
define ('POCKET_FILE', '/ril_export.html');
|
||||
define ('READABILITY_FILE', '/readability');
|
||||
define ('INSTAPAPER_FILE', '/instapaper-export.html');
|
||||
define ('POCHE_FILE', '/poche-export');
|
||||
//limit for download of articles during import
|
||||
define ('IMPORT_LIMIT', 5);
|
||||
//delay between downloads (in sec)
|
||||
define ('IMPORT_DELAY', 5);
|
||||
|
||||
define ('IMPORT_POCKET_FILE', ROOT . POCKET_FILE);
|
||||
define ('IMPORT_READABILITY_FILE', ROOT . READABILITY_FILE);
|
||||
define ('IMPORT_INSTAPAPER_FILE', ROOT . INSTAPAPER_FILE);
|
||||
define ('IMPORT_POCHE_FILE', ROOT . POCHE_FILE);
|
||||
3
index.php
Normal file → Executable file
3
index.php
Normal file → Executable file
@ -67,7 +67,8 @@ if (isset($_GET['login'])) {
|
||||
# Update password
|
||||
$poche->updatePassword();
|
||||
} elseif (isset($_GET['import'])) {
|
||||
$import = $poche->import($_GET['from']);
|
||||
$import = $poche->import();
|
||||
$tpl_vars = array_merge($tpl_vars, $import);
|
||||
} elseif (isset($_GET['download'])) {
|
||||
Tools::download_db();
|
||||
} elseif (isset($_GET['empty-cache'])) {
|
||||
|
||||
15
themes/default/_import.twig
Executable file
15
themes/default/_import.twig
Executable file
@ -0,0 +1,15 @@
|
||||
<script type="text/javascript">
|
||||
<!--
|
||||
$(document).ready(function() {
|
||||
$("body").css("cursor", "wait");
|
||||
|
||||
setTimeout(function(){
|
||||
window.location = '/?import';
|
||||
}, {{ import.delay }} );
|
||||
});
|
||||
//-->
|
||||
</script>
|
||||
<div class="messages warning">
|
||||
<p>{% trans "Download required for " %} {{ import.recordsDownloadRequired }} {% trans "records" %}.</p>
|
||||
<p>{% trans "Downloading next " %} {{ import.recordsUnderDownload }} {% trans "articles, please wait" %}...</p>
|
||||
</div>
|
||||
25
themes/default/config.twig
Normal file → Executable file
25
themes/default/config.twig
Normal file → Executable file
@ -104,36 +104,19 @@
|
||||
{% endif %}
|
||||
|
||||
<h2>{% trans "Import" %}</h2>
|
||||
<p>{% trans "Importing from other services can be quite long, and webservers default configuration often prevents long scripts execution time, so it must be done in multiple parts." %}</p>
|
||||
<p>1. {% trans "First, select the export file on your computer and upload it." %}</p>
|
||||
<form method="post" action="?uploadfile" name="uploadfile" enctype="multipart/form-data">
|
||||
<p>{% trans "You can import your Pocket, Readability, Instapaper, Wallabag or any data in appropriate json or html format." %}</p>
|
||||
<p>{% trans "Please select export file on your computer:" %}</p>
|
||||
<form method="post" action="?import" name="uploadfile" enctype="multipart/form-data">
|
||||
<fieldset class="w500p">
|
||||
<div class="row">
|
||||
<label class="col w150p" for="file">{% trans "File:" %}</label>
|
||||
<input class="col" type="file" id="file" name="file" tabindex="4">
|
||||
</div>
|
||||
<div class="row mts txtcenter">
|
||||
<button class="bouton" type="submit" tabindex="4">{% trans "Upload" %}</button>
|
||||
<button class="bouton" type="submit" tabindex="4">{% trans "Import" %}</button>
|
||||
</div>
|
||||
</fieldset>
|
||||
<input type="hidden" name="MAX_FILE_SIZE" value="1048576">
|
||||
<input type="hidden" name="returnurl" value="{{ referer }}">
|
||||
</form>
|
||||
<p>2. {% trans "Then, click on the right link below." %}</p>
|
||||
<ul>
|
||||
<li><a href="./?import&from=pocket">{% trans "Import from Pocket" %}</a> {{ '(after uploaded %s file)'|trans|format(constant('POCKET_FILE')) }}</li>
|
||||
<li><a href="./?import&from=readability">{% trans "Import from Readability" %}</a> {{ '(after uploaded %s file)'|trans|format(constant('READABILITY_FILE')) }}</li>
|
||||
<li><a href="./?import&from=instapaper">{% trans "Import from Instapaper" %}</a> {{ '(after uploaded %s file)'|trans|format(constant('INSTAPAPER_FILE')) }}</li>
|
||||
<li><a href="./?import&from=poche">{% trans "Import from wallabag" %}</a> {{ '(after uploaded %s file)'|trans|format(constant('POCHE_FILE')) }}</li>
|
||||
</ul>
|
||||
|
||||
{% if token == '' %}
|
||||
<p>{% trans "3. Your feed token is currently empty and must first be generated to fetch content. Click <a href='?feed&action=generate'>here to generate it</a>." %}</p>
|
||||
{% else %}
|
||||
<p>3. {% trans "Finally, you have to fetch content for imported items." %} <a href="cron.php?limit=10&user-id={{ user_id }}&token={{token}}" target="_blank">{% trans "Click here" %}</a> {% trans "to fetch content for 10 articles" %}.</p>
|
||||
<p>{% trans "If you have console access to your server, you can also create a cron task:" %}</p>
|
||||
<pre><code>0 */4 * * * cd /path/to/wallabag && php cron.php --limit=10 --user-id={{user_id}} --token={{token}} >/dev/null 2>&1</code></pre>
|
||||
{% endif %}
|
||||
|
||||
<h2>{% trans "Export your wallabag data" %}</h2>
|
||||
{% if constant('STORAGE') == 'sqlite' %}
|
||||
|
||||
@ -16,6 +16,11 @@
|
||||
{% include '_sorting.twig' %}
|
||||
{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
{% if includeImport %}
|
||||
{% include '_import.twig' %}
|
||||
{% endif %}
|
||||
|
||||
{% if tag %}
|
||||
<h3>{% trans "Tag" %}: <b>{{ tag.value }}</b></h3>
|
||||
{% endif %}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user