1
0
mirror of https://github.com/moparisthebest/pacman synced 2024-11-17 23:05:00 -05:00
pacman/scripts/repo-add.sh.in
Dan McGee 8bbaf045b9 repo-add: use bsdtar optimization for better performance
When unzipping packages and the database archives, we don't need to look
through the entire archive to do what we need to do. For packages, .PKGINFO
should only be found once and should be the first file in the package. For
the database check, we only really need to look for one desc file.

The bsdtar -q option is very similar to the GNU tar --occurrence=1 option.

Example of speedup:

$ time repo-add junkdb.db.tar.gz *.pkg.tar.gz >/dev/null
real	0m16.159s
user	0m14.836s
sys	0m2.277s

$ time ./scripts/repo-add junkdb.db.tar.gz *.pkg.tar.gz >/dev/null
real	0m4.949s
user	0m3.730s
sys	0m2.093s

Signed-off-by: Dan McGee <dan@archlinux.org>
2009-06-23 21:51:42 -05:00

499 lines
13 KiB
Bash

#!/bin/bash
#
# repo-add - add a package to a given repo database file
# repo-remove - remove a package entry from a given repo database file
# @configure_input@
#
# Copyright (c) 2006-2008 Aaron Griffin <aaron@archlinux.org>
# Copyright (c) 2007-2008 Dan McGee <dan@archlinux.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# gettext initialization
export TEXTDOMAIN='pacman'
export TEXTDOMAINDIR='@localedir@'
myver='@PACKAGE_VERSION@'
confdir='@sysconfdir@'
QUIET=0
REPO_DB_FILE=
LOCKFILE=
CLEAN_LOCK=0
startdir="$PWD"
# ensure we have a sane umask set
umask 0022
msg() {
local mesg=$1; shift
printf "==> ${mesg}\n" "$@" >&1
}
msg2() {
[ $QUIET -ne 0 ] && return
local mesg=$1; shift
printf " -> ${mesg}\n" "$@" >&1
}
warning() {
local mesg=$1; shift
printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2
}
error() {
local mesg=$1; shift
printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2
}
# print usage instructions
usage() {
printf "repo-add, repo-remove (pacman) %s\n\n" "$myver"
printf "$(gettext "Usage: repo-add [-q] <path-to-db> <package|delta> ...\n")"
printf "$(gettext "Usage: repo-remove [-q] <path-to-db> <packagename|delta> ...\n\n")"
printf "$(gettext "\
repo-add will update a package database by reading a package file.\n\
Multiple packages to add can be specified on the command line.\n\n")"
printf "$(gettext "\
repo-remove will update a package database by removing the package name\n\
specified on the command line from the given repo database. Multiple\n\
packages to remove can be specified on the command line.\n\n")"
printf "$(gettext "\
Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\
and errors\n\n")"
echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")"
echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")"
}
version() {
printf "repo-add, repo-remove (pacman) %s\n\n" "$myver"
printf "$(gettext "\
Copyright (C) 2006-2008 Aaron Griffin <aaron@archlinux.org>.\n\
Copyright (c) 2007-2008 Dan McGee <dan@archlinux.org>.\n\n\
This is free software; see the source for copying conditions.\n\
There is NO WARRANTY, to the extent permitted by law.\n")"
}
# write a list entry
# arg1 - Entry name
# arg2 - List
# arg3 - File to write to
write_list_entry() {
if [ -n "$2" ]; then
echo "%$1%" >>$3
echo -e $2 >>$3
fi
}
find_pkgentry()
{
local pkgname=$1
local pkgentry
for pkgentry in $tmpdir/$pkgname*; do
name=${pkgentry##*/}
if [ "${name%-*-*}" = "$pkgname" ]; then
echo $pkgentry
return 0
fi
done
return 1
}
# Get the package name from the delta filename
get_delta_pkgname() {
local tmp
tmp=${1##*/}
echo ${tmp%-*-*_to*}
}
# write a delta entry
# arg1 - path to delta file
db_write_delta()
{
deltafile="$1"
pkgname="$(get_delta_pkgname $deltafile)"
pkgentry=$(find_pkgentry $pkgname)
if [ -z "$pkgentry" ]; then
return 1
fi
deltas="$pkgentry/deltas"
# create deltas file if it does not already exist
if [ ! -f "$deltas" ]; then
msg2 "$(gettext "Creating 'deltas' db entry...")"
echo -e "%DELTAS%" >>$deltas
fi
# get md5sum and compressed size of package
md5sum="$(openssl dgst -md5 "$deltafile" | awk '{print $NF}')"
csize=$(@SIZECMD@ "$deltafile")
oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//')
newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//')
if grep -q "$oldfile.*$newfile" $deltas; then
warning "$(gettext "An entry for '%s' already existed")" "$deltafile"
sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup
msg2 "$(gettext "Removing existing entry '%s'...")" "$deltafile"
fi
echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas
return 0
} # end db_write_delta
# remove a delta entry
# arg1 - path to delta file
db_remove_delta()
{
deltafile="$1"
filename=${deltafile##*/}
pkgname="$(get_delta_pkgname $deltafile)"
pkgentry=$(find_pkgentry $pkgname)
if [ -z "$pkgentry" ]; then
return 1
fi
deltas="$pkgentry/deltas"
if [ ! -f "$deltas" ]; then
return 1
fi
if grep -q "$filename" $deltas; then
sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup
msg2 "$(gettext "Removing existing entry '%s'...")" "$filename"
return 0
fi
return 1
} # end db_remove_delta
# write an entry to the pacman database
# arg1 - path to package
db_write_entry()
{
# blank out all variables
local pkgfile="$1"
local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager force \
_groups _licenses _replaces _depends _conflicts _provides _optdepends
local OLDIFS="$IFS"
# IFS (field separator) is only the newline character
IFS="
"
# read info from the zipped package
local line var val
for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO |
grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do
# bash awesomeness here- var is always one word, val is everything else
var=${line%% *}
val=${line#* }
declare $var="$val"
case "$var" in
group) _groups="$_groups$group\n" ;;
license) _licenses="$_licenses$license\n" ;;
replaces) _replaces="$_replaces$replaces\n" ;;
depend) _depends="$_depends$depend\n" ;;
conflict) _conflicts="$_conflicts$conflict\n" ;;
provides) _provides="$_provides$provides\n" ;;
optdepend) _optdepends="$_optdepends$optdepend\n" ;;
esac
done
IFS=$OLDIFS
# get md5sum and compressed size of package
md5sum="$(openssl dgst -md5 "$pkgfile" | awk '{print $NF}')"
csize=$(@SIZECMD@ "$pkgfile")
# ensure $pkgname and $pkgver variables were found
if [ -z "$pkgname" -o -z "$pkgver" ]; then
error "$(gettext "Invalid package file '%s'.")" "$pkgfile"
return 1
fi
cd "$tmpdir"
if [ -d "$pkgname-$pkgver" ]; then
warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver"
fi
# remove an existing entry if it exists, ignore failures
db_remove_entry "$pkgname"
# create package directory
mkdir "$pkgname-$pkgver"
cd "$pkgname-$pkgver"
# restore an eventual deltas file
[ -f "../$pkgname.deltas" ] && mv "../$pkgname.deltas" deltas
# create desc entry
msg2 "$(gettext "Creating 'desc' db entry...")"
echo -e "%FILENAME%\n$(basename "$1")\n" >>desc
echo -e "%NAME%\n$pkgname\n" >>desc
echo -e "%VERSION%\n$pkgver\n" >>desc
[ -n "$pkgdesc" ] && echo -e "%DESC%\n$pkgdesc\n" >>desc
write_list_entry "GROUPS" "$_groups" "desc"
[ -n "$csize" ] && echo -e "%CSIZE%\n$csize\n" >>desc
[ -n "$size" ] && echo -e "%ISIZE%\n$size\n" >>desc
# compute checksums
msg2 "$(gettext "Computing md5 checksums...")"
echo -e "%MD5SUM%\n$md5sum\n" >>desc
[ -n "$url" ] && echo -e "%URL%\n$url\n" >>desc
write_list_entry "LICENSE" "$_licenses" "desc"
[ -n "$arch" ] && echo -e "%ARCH%\n$arch\n" >>desc
[ -n "$builddate" ] && echo -e "%BUILDDATE%\n$builddate\n" >>desc
[ -n "$packager" ] && echo -e "%PACKAGER%\n$packager\n" >>desc
write_list_entry "REPLACES" "$_replaces" "desc"
[ -n "$force" ] && echo -e "%FORCE%\n" >>desc
# create depends entry
msg2 "$(gettext "Creating 'depends' db entry...")"
# create the file even if it will remain empty
touch "depends"
write_list_entry "DEPENDS" "$_depends" "depends"
write_list_entry "CONFLICTS" "$_conflicts" "depends"
write_list_entry "PROVIDES" "$_provides" "depends"
write_list_entry "OPTDEPENDS" "$_optdepends" "depends"
cd "$startdir"
return 0
} # end db_write_entry
# remove existing entries from the DB
# arg1 - package name
db_remove_entry() {
local pkgname=$1
local notfound=1
local pkgentry=$(find_pkgentry $pkgname)
while [ -n "$pkgentry" ]; do
notfound=0
if [ -f "$pkgentry/deltas" ]; then
mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas"
fi
msg2 "$(gettext "Removing existing entry '%s'...")" \
"$(basename $pkgentry)"
rm -rf $pkgentry
pkgentry=$(find_pkgentry $pkgname)
done
return $notfound
} # end db_remove_entry
check_repo_db()
{
# check lock file
if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then
CLEAN_LOCK=1
else
error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE"
[ -f "$LOCKFILE" ] && error "$(gettext "Held by %s")" "$(cat $LOCKFILE)"
exit 1
fi
if [ -f "$REPO_DB_FILE" ]; then
if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' 2>&1 >/dev/null; then
error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE"
exit 1
fi
msg "$(gettext "Extracting database to a temporary location...")"
bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir"
else
case "$cmd" in
repo-remove)
error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE"
exit 1
;;
repo-add)
# check if the file can be created (write permission, directory existence, etc)
if ! touch "$REPO_DB_FILE"; then
error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE"
exit 1
fi
rm -f "$REPO_DB_FILE"
;;
esac
fi
}
add()
{
if [ ! -f "$1" ]; then
error "$(gettext "File '%s' not found.")" "$1"
return 1
fi
if [ "${1##*.}" == "delta" ]; then
deltafile=$1
msg "$(gettext "Adding delta '%s'")" "$deltafile"
if [ ! "$(type -p xdelta3)" ]; then
error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")"
exit 1
fi
if db_write_delta "$deltafile"; then
return 0
else
return 1
fi
fi
pkgfile=$1
if ! bsdtar -tqf "$pkgfile" .PKGINFO 2>&1 >/dev/null; then
error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile"
return 1
fi
msg "$(gettext "Adding package '%s'")" "$pkgfile"
db_write_entry "$pkgfile"
}
remove()
{
if [ "${1##*.}" == "delta" ]; then
deltafile=$1
msg "$(gettext "Searching for delta '%s'...")" "$deltafile"
if db_remove_delta "$deltafile"; then
return 0
else
error "$(gettext "Delta matching '%s' not found.")" "$deltafile"
return 1
fi
fi
pkgname=$1
msg "$(gettext "Searching for package '%s'...")" "$pkgname"
if db_remove_entry "$pkgname"; then
rm -f "$tmpdir/$pkgname.deltas"
return 0
else
error "$(gettext "Package matching '%s' not found.")" "$pkgname"
return 1
fi
}
trap_exit()
{
echo
error "$@"
exit 1
}
clean_up() {
local exit_code=$?
cd "$startdir"
[ -d "$tmpdir" ] && rm -rf "$tmpdir"
[ $CLEAN_LOCK -eq 1 -a -f "$LOCKFILE" ] && rm -f "$LOCKFILE"
exit $exit_code
}
# PROGRAM START
# determine whether we have gettext; make it a no-op if we do not
if [ ! $(type -t gettext) ]; then
gettext() {
echo "$@"
}
fi
case "$1" in
-h|--help) usage; exit 0;;
-V|--version) version; exit 0;;
esac
# check for correct number of args
if [ $# -lt 2 ]; then
usage
exit 1
fi
# figure out what program we are
cmd="$(basename $0)"
if [ "$cmd" != "repo-add" -a "$cmd" != "repo-remove" ]; then
error "$(gettext "Invalid command name '%s' specified.")" "$cmd"
exit 1
fi
tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\
error "$(gettext "Cannot create temp directory for database building.")"; \
exit 1)
trap 'clean_up' EXIT
trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR
success=0
# parse arguments
for arg in "$@"; do
case "$arg" in
-q|--quiet) QUIET=1;;
-f|--force)
warning "$(gettext "the -f and --force options are no longer recognized")"
msg2 "$(gettext "use options=(force) in the PKGBUILD instead")"
;;
*)
if [ -z "$REPO_DB_FILE" ]; then
REPO_DB_FILE="$arg"
LOCKFILE="$REPO_DB_FILE.lck"
check_repo_db
else
case "$cmd" in
repo-add) add $arg && success=1 ;;
repo-remove) remove $arg && success=1 ;;
esac
fi
;;
esac
done
# if at least one operation was a success, re-zip database
if [ $success -eq 1 ]; then
msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE"
case "$REPO_DB_FILE" in
*tar.gz) TAR_OPT="z" ;;
*tar.bz2) TAR_OPT="j" ;;
*tar.xz) TAR_OPT="J" ;;
*) warning "$(gettext "'%s' does not have a valid archive extension.")" \
"$REPO_DB_FILE" ;;
esac
filename=$(basename "$REPO_DB_FILE")
cd "$tmpdir"
if [ -n "$(ls)" ]; then
bsdtar -c${TAR_OPT}f "$filename" *
else
# the database will be moved to .old below, and there will be no new one to replace it
error "$(gettext "All packages have been removed from the database. Deleting '%s'.")" "$REPO_DB_FILE"
fi
cd "$startdir"
[ -f "$REPO_DB_FILE" ] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old"
[ -f "$tmpdir/$filename" ] && mv "$tmpdir/$filename" "$REPO_DB_FILE"
else
msg "$(gettext "No packages modified, nothing to do.")"
fi
exit 0
# vim: set ts=2 sw=2 noet: