Implement TotalDownload option.

Setting this option will change the download progress to show the amount
downloaded, download rate, ETA, and download percent of the entire
download list rather than per each individual file.

The progress bar is still based on the completion of the current file
regardless if the TotalDownload option is set.

This closes FS#7205.

Signed-off-by: Nathan Jones <nathanj@insightbb.com>
Signed-off-by: Dan McGee <dan@archlinux.org>

lib/libalpm/alpm.h

@@ -81,7 +81,8 @@ int alpm_logaction(char *fmt, ...);
  * Downloading
  */
 
-typedef void (*alpm_cb_download)(const char *filename, int xfered, int total);
+typedef void (*alpm_cb_download)(const char *filename, int file_xfered,
+		int file_total, int list_xfered, int list_total);
 
 /*
  * Options

lib/libalpm/db.c

@@ -257,7 +257,8 @@ int SYMEXPORT alpm_db_update(int force, pmdb_t *db)
 
 	dbpath = alpm_option_get_dbpath();
 
-	ret = _alpm_downloadfiles_forreal(db->servers, dbpath, files, lastupdate, newmtime);
+	ret = _alpm_downloadfiles_forreal(db->servers, dbpath, files, lastupdate,
+			newmtime, NULL, 0);
 	FREELIST(files);
 	if(ret == 1) {
 		/* mtimes match, do nothing */

lib/libalpm/server.c

@@ -133,13 +133,19 @@ static struct url *url_for_file(pmserver_t *server, const char *filename)
 /*
  * Download a list of files from a list of servers
  *   - if one server fails, we try the next one in the list
+ *   - if *dl_total is non-NULL, then it will be used as the starting
+ *     download amount when TotalDownload is set. It will also be
+ *     set to the final download amount for the calling function to use.
+ *   - totalsize is the total download size for use when TotalDownload
+ *     is set. Use 0 if the total download size is not known.
  *
  * RETURN:  0 for successful download, 1 on error
  */
 int _alpm_downloadfiles(alpm_list_t *servers, const char *localpath,
-		alpm_list_t *files)
+		alpm_list_t *files, int *dl_total, unsigned long totalsize)
 {
-	return(_alpm_downloadfiles_forreal(servers, localpath, files, NULL, NULL));
+	return(_alpm_downloadfiles_forreal(servers, localpath, files, NULL, NULL,
+				dl_total, totalsize));
 }
 
 /*
@@ -150,15 +156,21 @@ int _alpm_downloadfiles(alpm_list_t *servers, const char *localpath,
  *     "YYYYMMDDHHMMSS" to match the form of ftplib's FtpModDate() function.
  *   - if *mtime2 is non-NULL, then it will be filled with the mtime
  *     of the remote file (from MDTM FTP cmd or Last-Modified HTTP header).
+ *   - if *dl_total is non-NULL, then it will be used as the starting
+ *     download amount when TotalDownload is set. It will also be
+ *     set to the final download amount for the calling function to use.
+ *   - totalsize is the total download size for use when TotalDownload
+ *     is set. Use 0 if the total download size is not known.
  * 
  * RETURN:  0 for successful download
  *          1 if the mtimes are identical
  *         -1 on error
  */
 int _alpm_downloadfiles_forreal(alpm_list_t *servers, const char *localpath,
-	alpm_list_t *files, const char *mtime1, char *mtime2)
+	alpm_list_t *files, const char *mtime1, char *mtime2, int *dl_total,
+	unsigned long totalsize)
 {
-	int dltotal_bytes = 0;
+	int dl_thisfile = 0;
 	alpm_list_t *lp;
 	int done = 0;
 	alpm_list_t *complete = NULL;
@@ -206,12 +218,15 @@ int _alpm_downloadfiles_forreal(alpm_list_t *servers, const char *localpath,
 				if(stat(output, &st) == 0 && st.st_size > 0) {
 					_alpm_log(PM_LOG_DEBUG, "existing file found, using it\n");
 					fileurl->offset = (off_t)st.st_size;
-					dltotal_bytes = st.st_size;
+					dl_thisfile = st.st_size;
+					if (dl_total != NULL) {
+						*dl_total += st.st_size;
+					}
 					localf = fopen(output, "a");
 					chk_resume = 1;
 				} else {
 					fileurl->offset = (off_t)0;
-					dltotal_bytes = 0;
+					dl_thisfile = 0;
 				}
 				
 				/* libdownload does not reset the error code, reset it in the case of previous errors */
@@ -267,7 +282,7 @@ int _alpm_downloadfiles_forreal(alpm_list_t *servers, const char *localpath,
 				if(localf == NULL) {
 					_alpm_rmrf(output);
 					fileurl->offset = (off_t)0;
-					dltotal_bytes = 0;
+					dl_thisfile = 0;
 					localf = fopen(output, "w");
 					if(localf == NULL) { /* still null? */
 						_alpm_log(PM_LOG_ERROR, _("cannot write to file '%s'\n"), output);
@@ -280,7 +295,10 @@ int _alpm_downloadfiles_forreal(alpm_list_t *servers, const char *localpath,
 				}
 
 				/* Progress 0 - initialize */
-				if(handle->dlcb) handle->dlcb(pkgname, 0, ust.size);
+				if(handle->dlcb) {
+					handle->dlcb(pkgname, 0, ust.size, dl_total ? *dl_total : 0,
+							totalsize);
+				}
 
 				int nread = 0;
 				char buffer[PM_DLBUF_LEN];
@@ -310,10 +328,17 @@ int _alpm_downloadfiles_forreal(alpm_list_t *servers, const char *localpath,
 					if(nwritten != nread) {
 						
 					}
-					dltotal_bytes += nread;
+					dl_thisfile += nread;
-
+					if (dl_total != NULL) {
-					if(handle->dlcb) handle->dlcb(pkgname, dltotal_bytes, ust.size);
+						*dl_total += nread;
 					}
+
+					if(handle->dlcb) {
+						handle->dlcb(pkgname, dl_thisfile, ust.size,
+								dl_total ? *dl_total : 0, totalsize);
+					}
+				}
+
 				downloadFreeURL(fileurl);
 				fclose(localf);
 				fclose(dlf);
@@ -429,7 +454,7 @@ char SYMEXPORT *alpm_fetch_pkgurl(const char *url)
 	alpm_list_t *files = alpm_list_add(NULL, filename);
 
 	/* download the file */
-	if(_alpm_downloadfiles(servers, cachedir, files)) {
+	if(_alpm_downloadfiles(servers, cachedir, files, NULL, 0)) {
 		_alpm_log(PM_LOG_WARNING, _("failed to download %s\n"), url);
 		return(NULL);
 	}

lib/libalpm/server.h

@@ -37,9 +37,11 @@ struct __pmserver_t {
 
 pmserver_t *_alpm_server_new(const char *url);
 void _alpm_server_free(pmserver_t *server);
-int _alpm_downloadfiles(alpm_list_t *servers, const char *localpath, alpm_list_t *files);
+int _alpm_downloadfiles(alpm_list_t *servers, const char *localpath,
+		alpm_list_t *files, int *dl_total, unsigned long totalsize);
 int _alpm_downloadfiles_forreal(alpm_list_t *servers, const char *localpath,
-	alpm_list_t *files, const char *mtime1, char *mtime2);
+	alpm_list_t *files, const char *mtime1, char *mtime2,
+	int *dl_total, unsigned long totalsize);
 
 #endif /* _ALPM_SERVER_H */
 

lib/libalpm/sync.c

@@ -990,6 +990,7 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
 	pmtrans_t *tr = NULL;
 	int replaces = 0, retval = 0;
 	const char *cachedir = NULL;
+	int dltotal = 0, dl = 0;
 
 	ALPM_LOG_FUNC;
 
@@ -998,6 +999,15 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
 
 	cachedir = _alpm_filecache_setup();
 	trans->state = STATE_DOWNLOADING;
+
+	/* Sum up the download sizes. This has to be in its own loop because
+	 * the download loop is grouped by db. */
+	for(j = trans->packages; j; j = j->next) {
+		pmsyncpkg_t *sync = j->data;
+		pmpkg_t *spkg = sync->pkg;
+		dltotal += alpm_pkg_download_size(spkg, db_local);
+	}
+
 	/* group sync records by repository and download */
 	for(i = handle->dbs_sync; i; i = i->next) {
 		pmdb_t *current = i->data;
@@ -1061,7 +1071,7 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
 
 		if(files) {
 			EVENT(trans, PM_TRANS_EVT_RETRIEVE_START, current->treename, NULL);
-			if(_alpm_downloadfiles(current->servers, cachedir, files)) {
+			if(_alpm_downloadfiles(current->servers, cachedir, files, &dl, dltotal)) {
 				_alpm_log(PM_LOG_WARNING, _("failed to retrieve some files from %s\n"),
 						current->treename);
 				RET_ERR(PM_ERR_RETRIEVE, -1);

src/pacman/callback.c

@@ -85,16 +85,17 @@ static float get_update_timediff(int first_call)
 }
 
 /* refactored from cb_trans_progress */
-static void fill_progress(const int percent, const int proglen)
+static void fill_progress(const int graph_percent, const int display_percent,
+		const int proglen)
 {
 	const unsigned int hashlen = proglen - 8;
-	const unsigned int hash = percent * hashlen / 100;
+	const unsigned int hash = graph_percent * hashlen / 100;
 	static unsigned int lasthash = 0, mouth = 0;
 	unsigned int i;
 
 	/* printf("\ndebug: proglen: %i\n", proglen); DEBUG*/
 
-	if(percent == 0) {
+	if(graph_percent == 0) {
 		lasthash = 0;
 		mouth = 0;
 	}
@@ -139,10 +140,10 @@ static void fill_progress(const int percent, const int proglen)
 	}
 	/* print percent after progress bar */
 	if(proglen > 5) {
-		printf(" %3d%%", percent);
+		printf(" %3d%%", display_percent);
 	}
 
-	if(percent == 100) {
+	if(graph_percent == 100) {
 		printf("\n");
 	} else {
 		printf("\r");
@@ -432,34 +433,55 @@ void cb_trans_progress(pmtransprog_t event, const char *pkgname, int percent,
 	free(wcopr);
 
 	/* call refactored fill progress function */
-	fill_progress(percent, getcols() - infolen);
+	fill_progress(percent, percent, getcols() - infolen);
 }
 
 /* callback to handle display of download progress */
-void cb_dl_progress(const char *filename, int xfered, int total)
+void cb_dl_progress(const char *filename, int file_xfered, int file_total,
+		int list_xfered, int list_total)
 {
 	const int infolen = 50;
 	char *fname, *p; 
 
 	float rate = 0.0, timediff = 0.0, f_xfered = 0.0;
 	unsigned int eta_h = 0, eta_m = 0, eta_s = 0;
-	int percent;
+	int graph_percent = 0, display_percent = 0;
 	char rate_size = 'K', xfered_size = 'K';
+	int xfered = 0, total = 0;
+
+	/* Need this variable when TotalDownload is set to know if we should
+	 * reset xfered_last and rate_last. */
+	static int has_init = 0;
 
 	if(config->noprogressbar) {
 		return;
 	}
 
-	/* this is basically a switch on xferred: 0, total, and anything else */
+	/* Choose how to display the amount downloaded, rate, ETA, and
-	if(xfered == 0) {
+	 * percentage depending on the TotalDownload option. */
-		/* set default starting values */
+	if (config->totaldownload && list_total > 0) {
+		xfered = list_xfered;
+		total = list_total;
+	} else {
+		xfered = file_xfered;
+		total = file_total;
+	}
+
+	/* this is basically a switch on file_xferred: 0, file_total, and
+	 * anything else */
+	if(file_xfered == 0) {
+		/* set default starting values, but only once for TotalDownload */
+		if (!(config->totaldownload && list_total > 0) ||
+				(config->totaldownload && list_total > 0 && !has_init)) {
 			gettimeofday(&initial_time, NULL);
+			timediff = get_update_timediff(1);
 			xfered_last = 0;
 			rate_last = 0.0;
-		timediff = get_update_timediff(1);
+			has_init = 1;
+		}
 		rate = 0.0;
 		eta_s = 0;
-	} else if(xfered == total) {
+	} else if(file_xfered == file_total) {
 		/* compute final values */
 		struct timeval current_time;
 		float diff_sec, diff_usec;
@@ -468,7 +490,7 @@ void cb_dl_progress(const char *filename, int xfered, int total)
 		diff_sec = current_time.tv_sec - initial_time.tv_sec;
 		diff_usec = current_time.tv_usec - initial_time.tv_usec;
 		timediff = diff_sec + (diff_usec / 1000000.0);
-		rate = total / (timediff * 1024.0);
+		rate = xfered / (timediff * 1024.0);
 
 		/* round elapsed time to the nearest second */
 		eta_s = (int)(timediff + 0.5);
@@ -488,8 +510,6 @@ void cb_dl_progress(const char *filename, int xfered, int total)
 		xfered_last = xfered;
 	}
 
-	percent = (int)((float)xfered) / ((float)total) * 100;
-
 	/* fix up time for display */
 	eta_h = eta_s / 3600;
 	eta_s -= eta_h * 3600;
@@ -537,7 +557,11 @@ void cb_dl_progress(const char *filename, int xfered, int total)
 
 	free(fname);
 	
-	fill_progress(percent, getcols() - infolen);
+	/* The progress bar is based on the file percent regardless of the
+	 * TotalDownload option. */
+	graph_percent = (int)((float)file_xfered) / ((float)file_total) * 100;
+	display_percent = (int)((float)xfered) / ((float)total) * 100;
+	fill_progress(graph_percent, display_percent, getcols() - infolen);
 	return;
 }
 

src/pacman/callback.h

@@ -35,7 +35,8 @@ void cb_trans_progress(pmtransprog_t event, const char *pkgname, int percent,
                    int howmany, int remain);
 
 /* callback to handle display of download progress */
-void cb_dl_progress(const char *filename, int xfered, int total);
+void cb_dl_progress(const char *filename, int file_xfered, int file_total,
+		int list_xfered, int list_total);
 
 /* callback to handle messages/notifications from pacman library */
 void cb_log(pmloglevel_t level, char *fmt, va_list args);