Download delta files if UseDelta is set.

Delta files will be used if the size is smaller than a percent
(MAX_DELTA_RATIO) of the package size.

Signed-off-by: Nathan Jones <nathanj@insightbb.com>
Signed-off-by: Dan McGee <dan@archlinux.org>

doc/pacman.conf.5.txt

@@ -105,6 +105,10 @@ Options
 *ShowSize*::
 	Display the size of individual packages for '\--sync' and '\--query' modes.
 
+*UseDelta*::
+	Download delta files instead of complete packages if possible.  Requires
+	the xdelta program to be installed.
+
 
 Repository Sections
 -------------------

lib/libalpm/alpm.h

@@ -138,6 +138,7 @@ void alpm_option_set_xfercommand(const char *cmd);
 
 unsigned short alpm_option_get_nopassiveftp();
 void alpm_option_set_nopassiveftp(unsigned short nopasv);
+void alpm_option_set_usedelta(unsigned short usedelta);
 
 pmdb_t *alpm_option_get_localdb();
 alpm_list_t *alpm_option_get_syncdbs();
@@ -294,6 +295,13 @@ typedef enum _pmtransevt_t {
 	PM_TRANS_EVT_EXTRACT_DONE,
 	PM_TRANS_EVT_INTEGRITY_START,
 	PM_TRANS_EVT_INTEGRITY_DONE,
+	PM_TRANS_EVT_DELTA_INTEGRITY_START,
+	PM_TRANS_EVT_DELTA_INTEGRITY_DONE,
+	PM_TRANS_EVT_DELTA_PATCHES_START,
+	PM_TRANS_EVT_DELTA_PATCHES_DONE,
+	PM_TRANS_EVT_DELTA_PATCH_START,
+	PM_TRANS_EVT_DELTA_PATCH_DONE,
+	PM_TRANS_EVT_DELTA_PATCH_FAILED,
 	PM_TRANS_EVT_PRINTURI,
 	PM_TRANS_EVT_RETRIEVE_START,
 } pmtransevt_t;
@@ -442,6 +450,9 @@ enum _pmerrno_t {
 	PM_ERR_PKG_INVALID_NAME,
 	PM_ERR_PKG_CORRUPTED,
 	PM_ERR_PKG_REPO_NOT_FOUND,
+	/* Deltas */
+	PM_ERR_DLT_CORRUPTED,
+	PM_ERR_DLT_PATCHFAILED,
 	/* Groups */
 	PM_ERR_GRP_NOT_FOUND,
 	/* Dependencies */

lib/libalpm/error.c

@@ -123,6 +123,11 @@ const char SYMEXPORT *alpm_strerror(int err)
 			return _("corrupted package");
 		case PM_ERR_PKG_REPO_NOT_FOUND:
 			return _("no such repository");
+		/* Deltas */
+		case PM_ERR_DLT_CORRUPTED:
+			return _("corrupted delta");
+		case PM_ERR_DLT_PATCHFAILED:
+			return _("delta patch failed");
 		/* Groups */
 		case PM_ERR_GRP_NOT_FOUND:
 			return _("group not found");

lib/libalpm/handle.c

@@ -64,6 +64,7 @@ pmhandle_t *_alpm_handle_new()
 	handle->cachedirs = NULL;
 	handle->lockfile = NULL;
 	handle->logfile = NULL;
+	handle->usedelta = 0;
 
 	return(handle);
 }
@@ -496,4 +497,9 @@ void SYMEXPORT alpm_option_set_nopassiveftp(unsigned short nopasv)
 	handle->nopassiveftp = nopasv;
 }
 
+void SYMEXPORT alpm_option_set_usedelta(unsigned short usedelta)
+{
+	handle->usedelta = usedelta;
+}
+
 /* vim: set ts=2 sw=2 noet: */

lib/libalpm/handle.h

@@ -61,6 +61,7 @@ typedef struct _pmhandle_t {
 	unsigned short nopassiveftp; /* Don't use PASV ftp connections */
 	time_t upgradedelay;      /* Time to wait before upgrading a package */
 	char *xfercommand;        /* External download command */
+	unsigned short usedelta;     /* Download deltas if possible */
 } pmhandle_t;
 
 extern pmhandle_t *handle;

lib/libalpm/sync.c

@@ -48,6 +48,7 @@
 #include "handle.h"
 #include "alpm.h"
 #include "server.h"
+#include "delta.h"
 
 pmsyncpkg_t *_alpm_sync_new(int type, pmpkg_t *spkg, void *data)
 {
@@ -700,6 +701,148 @@ cleanup:
 	return(ret);
 }
 
+/** Returns a list of deltas that should be downloaded instead of the
+ * package.
+ *
+ * It first tests if a delta path exists between the currently installed
+ * version (if any) and the version to upgrade to. If so, the delta path
+ * is used if its size is below a set percentage (MAX_DELTA_RATIO) of
+ * the package size, Otherwise, an empty list is returned.
+ *
+ * @param newpkg the new package to upgrade to
+ * @param db_local the local database
+ *
+ * @return the list of pmdelta_t * objects. NULL (the empty list) is
+ * returned if the package should be downloaded instead of deltas.
+ */
+static alpm_list_t *pkg_upgrade_delta_path(pmpkg_t *newpkg, pmdb_t *db_local)
+{
+	pmpkg_t *oldpkg = alpm_db_get_pkg(db_local, newpkg->name);
+	alpm_list_t *ret = NULL;
+
+	if(oldpkg) {
+		const char *oldname = alpm_pkg_get_filename(oldpkg);
+		char *oldpath = _alpm_filecache_find(oldname);
+
+		if(oldpath) {
+			alpm_list_t *deltas = _alpm_shortest_delta_path(
+					alpm_pkg_get_deltas(newpkg),
+					alpm_pkg_get_version(oldpkg),
+					alpm_pkg_get_version(newpkg));
+
+			if(deltas) {
+				unsigned long dltsize = _alpm_delta_path_size(deltas);
+				unsigned long pkgsize = alpm_pkg_get_size(newpkg);
+
+				if(dltsize < pkgsize * MAX_DELTA_RATIO) {
+					ret = deltas;
+				} else {
+					ret = NULL;
+					alpm_list_free(deltas);
+				}
+			}
+
+			FREE(oldpath);
+		}
+	}
+
+	return(ret);
+}
+
+/** Applies delta files to create an upgraded package file.
+ *
+ * All intermediate files are deleted, leaving only the starting and
+ * ending package files.
+ *
+ * @param trans the transaction
+ * @param patches A list of alternating pmpkg_t * and pmdelta_t *
+ * objects. The patch command will be built using the pmpkg_t, pmdelta_t
+ * pair.
+ *
+ * @return 0 if all delta files were able to be applied, 1 otherwise.
+ */
+static int apply_deltas(pmtrans_t *trans, alpm_list_t *patches)
+{
+	/* keep track of the previous package in the loop to decide if a
+	 * package file should be deleted */
+	pmpkg_t *lastpkg = NULL;
+	int lastpkg_failed = 0;
+	int ret = 0;
+	const char *cachedir = _alpm_filecache_setup();
+
+	alpm_list_t *p = patches;
+	while(p) {
+		pmpkg_t *pkg;
+		pmdelta_t *d;
+		char command[PATH_MAX], fname[PATH_MAX];
+		char pkgfilename[PKG_FILENAME_LEN];
+
+		pkg = alpm_list_getdata(p);
+		p = alpm_list_next(p);
+
+		d = alpm_list_getdata(p);
+		p = alpm_list_next(p);
+
+		/* if patching fails, ignore the rest of that package's deltas */
+		if(lastpkg_failed) {
+			if(pkg == lastpkg) {
+				continue;
+			} else {
+				lastpkg_failed = 0;
+			}
+		}
+
+		/* an example of the patch command: (using /cache for cachedir)
+		 * xdelta patch /cache/pacman_3.0.0-1_to_3.0.1-1-i686.delta \
+		 *              /cache/pacman-3.0.0-1-i686.pkg.tar.gz       \
+		 *              /cache/pacman-3.0.1-1-i686.pkg.tar.gz
+		 */
+
+		/* build the patch command */
+		snprintf(command, PATH_MAX,
+				"xdelta patch"         /* the command */
+				" %s/%s"               /* the delta */
+				" %s/%s-%s-%s" PKGEXT  /* the 'from' package */
+				" %s/%s-%s-%s" PKGEXT, /* the 'to' package */
+				cachedir, d->filename,
+				cachedir, pkg->name, d->from, pkg->arch,
+				cachedir, pkg->name, d->to, pkg->arch);
+
+		_alpm_log(PM_LOG_DEBUG, _("command: %s\n"), command);
+
+		snprintf(pkgfilename, PKG_FILENAME_LEN, "%s-%s-%s" PKGEXT,
+				pkg->name, d->to, pkg->arch);
+
+		EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_START, pkgfilename, d->filename);
+
+		if(system(command) == 0) {
+			EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_DONE, NULL, NULL);
+
+			/* delete the delta file */
+			snprintf(fname, PATH_MAX, "%s/%s", cachedir, d->filename);
+			unlink(fname);
+
+			/* Delete the 'from' package but only if it is an intermediate
+			 * package. The starting 'from' package should be kept, just
+			 * as if deltas were not used. Delete the package file if the
+			 * previous iteration of the loop used the same package. */
+			if(pkg == lastpkg) {
+				snprintf(fname, PATH_MAX, "%s/%s-%s-%s" PKGEXT,
+						cachedir, pkg->name, d->from, pkg->arch);
+				unlink(fname);
+			} else {
+				lastpkg = pkg;
+			}
+		} else {
+			EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_FAILED, NULL, NULL);
+			lastpkg_failed = 1;
+			ret = 1;
+		}
+	}
+
+	return(ret);
+}
+
 /** Compares the md5sum of a file to the expected value.
  *
  * If the md5sum does not match, the user is asked whether the file
@@ -762,6 +905,29 @@ static int test_md5sum(pmtrans_t *trans, const char *filename,
 	return(ret);
 }
 
+/** Compares the md5sum of a delta to the expected value.
+ *
+ * @param trans the transaction
+ * @param delta the delta to test
+ * @param data data to write the error messages to
+ *
+ * @return 0 if the md5sum matched, 1 otherwise
+ */
+static int test_delta_md5sum(pmtrans_t *trans, pmdelta_t *delta,
+		alpm_list_t **data)
+{
+	const char *filename;
+	char *md5sum;
+	int ret = 0;
+
+	filename = alpm_delta_get_filename(delta);
+	md5sum = alpm_delta_get_md5sum(delta);
+
+	ret = test_md5sum(trans, filename, md5sum, data);
+
+	return(ret);
+}
+
 /** Compares the md5sum of a package to the expected value.
  *
  * @param trans the transaction
@@ -787,6 +953,7 @@ static int test_pkg_md5sum(pmtrans_t *trans, pmpkg_t *pkg, alpm_list_t **data)
 int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
 {
 	alpm_list_t *i, *j, *files = NULL;
+	alpm_list_t *patches = NULL, *deltas = NULL;
 	pmtrans_t *tr = NULL;
 	int replaces = 0, retval = 0;
 	const char *cachedir = NULL;
@@ -817,8 +984,42 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
 				} else {
 					char *fpath = _alpm_filecache_find(fname);
 					if(!fpath) {
-						/* file is not in the cache dir, so add it to the list */
-						files = alpm_list_add(files, strdup(fname));
+						if(handle->usedelta) {
+							alpm_list_t *delta_path = pkg_upgrade_delta_path(spkg, db_local);
+
+							if(delta_path) {
+								alpm_list_t *dlts = NULL;
+
+								for(dlts = delta_path; dlts; dlts = alpm_list_next(dlts)) {
+									pmdelta_t *d = (pmdelta_t *)alpm_list_getdata(dlts);
+									char *fpath2 = _alpm_filecache_find(d->filename);
+
+									if(!fpath2) {
+										/* add the delta filename to the download list if
+										 * it's not in the cache*/
+										files = alpm_list_add(files, strdup(d->filename));
+									}
+
+									/* save the package and delta so that the xdelta patch
+									 * command can be run after the downloads finish */
+									patches = alpm_list_add(patches, spkg);
+									patches = alpm_list_add(patches, d);
+
+									/* keep a list of the delta files for md5sums */
+									deltas = alpm_list_add(deltas, d);
+								}
+
+								alpm_list_free(delta_path);
+								delta_path = NULL;
+							} else {
+								/* no deltas to download, so add the file to the
+								 * download list */
+								files = alpm_list_add(files, strdup(fname));
+							}
+						} else {
+							/* not using deltas, so add the file to the download list */
+							files = alpm_list_add(files, strdup(fname));
+						}
 					}
 					FREE(fpath);
 				}
@@ -839,7 +1040,48 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
 		return(0);
 	}
 
-	/* Check integrity of files */
+	if(handle->usedelta) {
+		int ret = 0;
+
+		/* only output if there are deltas to work with */
+		if(deltas) {
+			/* Check integrity of deltas */
+			EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_START, NULL, NULL);
+
+			for(i = deltas; i; i = i->next) {
+				pmdelta_t *d = alpm_list_getdata(i);
+
+				ret = test_delta_md5sum(trans, d, data);
+
+				if(ret == 1) {
+					retval = 1;
+				} else if(ret == -1) { /* -1 is for serious errors */
+					RET_ERR(pm_errno, -1);
+				}
+			}
+			if(retval) {
+				pm_errno = PM_ERR_DLT_CORRUPTED;
+				goto error;
+			}
+			EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_DONE, NULL, NULL);
+
+			/* Use the deltas to generate the packages */
+			EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_START, NULL, NULL);
+			ret = apply_deltas(trans, patches);
+			EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_DONE, NULL, NULL);
+
+			alpm_list_free(patches);
+			patches = NULL;
+			alpm_list_free(deltas);
+			deltas = NULL;
+		}
+		if(ret) {
+			pm_errno = PM_ERR_DLT_PATCHFAILED;
+			goto error;
+		}
+	}
+
+	/* Check integrity of packages */
 	EVENT(trans, PM_TRANS_EVT_INTEGRITY_START, NULL, NULL);
 
 	for(i = trans->packages; i; i = i->next) {

lib/libalpm/util.h

@@ -67,6 +67,9 @@ char *strsep(char **, const char *);
 #define SYMEXPORT __attribute__((visibility("default")))
 #define SYMHIDDEN __attribute__((visibility("internal")))
 
+/* max percent of package size to download deltas */
+#define MAX_DELTA_RATIO 0.7
+
 #endif /* _ALPM_UTIL_H */
 
 /* vim: set ts=2 sw=2 noet: */

src/pacman/callback.c

@@ -236,6 +236,27 @@ void cb_trans_evt(pmtransevt_t event, void *data1, void *data2)
 		case PM_TRANS_EVT_INTEGRITY_DONE:
 			printf(_("done.\n"));
 			break;
+		case PM_TRANS_EVT_DELTA_INTEGRITY_START:
+			printf(_("checking delta integrity... "));
+			break;
+		case PM_TRANS_EVT_DELTA_INTEGRITY_DONE:
+			printf(_("done.\n"));
+			break;
+		case PM_TRANS_EVT_DELTA_PATCHES_START:
+			printf(_("applying deltas...\n"));
+			break;
+		case PM_TRANS_EVT_DELTA_PATCHES_DONE:
+			/* nothing */
+			break;
+		case PM_TRANS_EVT_DELTA_PATCH_START:
+			printf(_("generating %s with %s... "), (char *)data1, (char *)data2);
+			break;
+		case PM_TRANS_EVT_DELTA_PATCH_DONE:
+			printf(_("done.\n"));
+			break;
+		case PM_TRANS_EVT_DELTA_PATCH_FAILED:
+			printf(_("failed.\n"));
+			break;
 		case PM_TRANS_EVT_PRINTURI:
 			printf("%s/%s\n", (char*)data1, (char*)data2);
 			break;
@@ -309,7 +330,7 @@ void cb_trans_conv(pmtransconv_t event, void *data1, void *data2,
 			break;
 		case PM_TRANS_CONV_CORRUPTED_PKG:
 			if(!config->noconfirm) {
-				snprintf(str, LOG_STR_LEN, _(":: Archive %s is corrupted. Do you want to delete it? [Y/n] "),
+				snprintf(str, LOG_STR_LEN, _(":: File %s is corrupted. Do you want to delete it? [Y/n] "),
 						(char *)data1);
 				*response = yesno(str);
 			} else {

src/pacman/pacman.c

@@ -548,6 +548,9 @@ static int _parseconfig(const char *file, const char *givensection,
 				} else if(strcmp(key, "ShowSize") == 0 || strcmp(upperkey, "SHOWSIZE") == 0) {
 					config->showsize = 1;
 					pm_printf(PM_LOG_DEBUG, "config: showsize\n");
+				} else if(strcmp(key, "UseDelta") == 0 || strcmp(upperkey, "USEDELTA") == 0) {
+					alpm_option_set_usedelta(1);
+					pm_printf(PM_LOG_DEBUG, "config: usedelta\n");
 				} else {
 					pm_printf(PM_LOG_ERROR, _("config file %s, line %d: directive '%s' not recognized.\n"),
 							file, linenum, key);