db, solver, io: scan cache items at startup

It is faster to just scan the cache directory for existing packages
at startup than trying to faccessat() them on demand. It also makes
quite a few parts of the code more readable and simpler.
cute-signatures
Timo Teräs 2012-02-22 08:45:40 +02:00
parent 568d57336d
commit bf82e2e5fd
8 changed files with 210 additions and 157 deletions

View File

@ -18,26 +18,6 @@
#include "apk_package.h"
#include "apk_io.h"
/* default architecture for APK packages. */
#if defined(__x86_64__)
#define APK_DEFAULT_ARCH "x86_64"
#elif defined(__i386__)
#define APK_DEFAULT_ARCH "x86"
#elif defined(__powerpc__) && !defined(__powerpc64__)
#define APK_DEFAULT_ARCH "ppc"
#elif defined(__powerpc64__)
#define APK_DEFAULT_ARCH "ppc64"
#elif defined(__arm__)
#define APK_DEFAULT_ARCH "arm"
#else
#warning APK_DEFAULT_ARCH is not set for this architecture
#define APK_DEFAULT_ARCH "noarch"
#endif
#define APK_MAX_REPOS 32
#define APK_MAX_TAGS 16
#define APK_CACHE_CSUM_BYTES 4
extern const char * const apk_index_gz;
extern const char * const apkindex_tar_gz;
@ -219,6 +199,7 @@ int apk_db_add_repository(apk_database_t db, apk_blob_t repository);
struct apk_repository *apk_db_select_repo(struct apk_database *db,
struct apk_package *pkg);
int apk_repository_update(struct apk_database *db, struct apk_repository *repo);
int apk_repo_is_remote(struct apk_repository *repo);
int apk_repo_format_filename(char *buf, size_t len,
const char *repourl, apk_blob_t *arch,
const char *pkgfile);
@ -228,6 +209,11 @@ void apk_cache_format_index(apk_blob_t to, struct apk_repository *repo);
int apk_cache_download(struct apk_database *db, const char *url, apk_blob_t *arch,
const char *item, const char *cache_item, int verify);
typedef void (*apk_cache_item_cb)(struct apk_database *db,
const char *filename,
struct apk_package *pkg);
int apk_db_cache_foreach_item(struct apk_database *db, apk_cache_item_cb cb);
int apk_db_install_pkg(struct apk_database *db,
struct apk_package *oldpkg,
struct apk_package *newpkg,

View File

@ -64,6 +64,26 @@ extern char **apk_argv;
#define APK_NO_NETWORK 0x1000
#define APK_OVERLAY_FROM_STDIN 0x2000
/* default architecture for APK packages. */
#if defined(__x86_64__)
#define APK_DEFAULT_ARCH "x86_64"
#elif defined(__i386__)
#define APK_DEFAULT_ARCH "x86"
#elif defined(__powerpc__) && !defined(__powerpc64__)
#define APK_DEFAULT_ARCH "ppc"
#elif defined(__powerpc64__)
#define APK_DEFAULT_ARCH "ppc64"
#elif defined(__arm__)
#define APK_DEFAULT_ARCH "arm"
#else
#warning APK_DEFAULT_ARCH is not set for this architecture
#define APK_DEFAULT_ARCH "noarch"
#endif
#define APK_MAX_REPOS 31 /* see struct apk_package */
#define APK_MAX_TAGS 16 /* see solver; unsigned short */
#define APK_CACHE_CSUM_BYTES 4
static inline size_t apk_calc_installed_size(size_t size)
{
const size_t bsize = 4 * 1024;

View File

@ -106,10 +106,13 @@ size_t apk_ostream_write_string(struct apk_ostream *ostream, const char *string)
apk_blob_t apk_blob_from_istream(struct apk_istream *istream, size_t size);
apk_blob_t apk_blob_from_file(int atfd, const char *file);
typedef int apk_dir_file_cb(void *ctx, const char *entry);
#define APK_FI_NOFOLLOW 0x80000000
int apk_file_get_info(int atfd, const char *filename, unsigned int flags,
struct apk_file_info *fi);
int apk_dir_foreach_file(int dirfd, apk_dir_file_cb cb, void *ctx);
int apk_move_file(int atfd, const char *from, const char *to);
int apk_url_download(const char *url, int atfd, const char *file);
const char *apk_url_local_file(const char *url);

View File

@ -96,8 +96,9 @@ struct apk_package {
struct apk_dependency_array *depends, *install_if;
size_t installed_size, size;
time_t build_time;
unsigned repos;
unsigned int topology_hard;
unsigned in_cache : 1;
unsigned repos : APK_MAX_REPOS;
struct apk_checksum csum;
};
APK_ARRAY(apk_package_array, struct apk_package *);

View File

@ -31,10 +31,10 @@ static int cache_download(struct apk_database *db)
struct apk_package *pkg;
struct apk_repository *repo;
char item[PATH_MAX], cacheitem[PATH_MAX];
int i, r = 0;
int i, r, ret = 0;
r = apk_solver_solve(db, 0, db->world, NULL, &changeset);
if (r != 0) {
if (r < 0) {
apk_error("Unable to select packages. Run apk fix.");
return r;
}
@ -43,86 +43,53 @@ static int cache_download(struct apk_database *db)
change = &changeset.changes->item[i];
pkg = change->newpkg;
apk_pkg_format_cache(pkg, APK_BLOB_BUF(cacheitem));
if (faccessat(db->cache_fd, cacheitem, R_OK, 0) == 0)
if (pkg->in_cache)
continue;
repo = apk_db_select_repo(db, pkg);
if (repo == NULL || apk_url_local_file(repo->url) != NULL)
if (repo == NULL || !apk_repo_is_remote(repo))
continue;
apk_pkg_format_cache(pkg, APK_BLOB_BUF(cacheitem));
apk_pkg_format_plain(pkg, APK_BLOB_BUF(item));
r |= apk_cache_download(db, repo->url, pkg->arch,
r = apk_cache_download(db, repo->url, pkg->arch,
item, cacheitem,
APK_SIGN_VERIFY_IDENTITY);
if (r) {
apk_error("%s: %s", item, apk_error_str(r));
ret++;
}
}
return r;
return ret;
}
static void cache_clean_item(struct apk_database *db, const char *filename, struct apk_package *pkg)
{
char tmp[PATH_MAX];
apk_blob_t b;
int i;
if (pkg != NULL || strcmp(filename, "installed") == 0)
return;
b = APK_BLOB_STR(filename);
for (i = 0; i < db->num_repos; i++) {
/* Check if this is a valid index */
apk_cache_format_index(APK_BLOB_BUF(tmp), &db->repos[i]);
if (apk_blob_compare(b, APK_BLOB_STR(tmp)) == 0)
return;
}
if (apk_verbosity >= 2)
apk_message("deleting %s", filename);
if (!(apk_flags & APK_SIMULATE))
unlinkat(db->cache_fd, filename, 0);
}
static int cache_clean(struct apk_database *db)
{
char tmp[PATH_MAX];
DIR *dir;
struct dirent *de;
int delete, i;
apk_blob_t b, bname, bver;
struct apk_name *name;
dir = fdopendir(dup(db->cache_fd));
if (dir == NULL)
return -1;
while ((de = readdir(dir)) != NULL) {
if (de->d_name[0] == '.')
continue;
delete = TRUE;
do {
b = APK_BLOB_STR(de->d_name);
if (apk_blob_compare(b, APK_BLOB_STR("installed")) == 0) {
delete = FALSE;
break;
}
if (apk_pkg_parse_name(b, &bname, &bver) < 0) {
/* Index - check for matching repository */
for (i = 0; i < db->num_repos; i++) {
apk_cache_format_index(APK_BLOB_BUF(tmp), &db->repos[i]);
if (apk_blob_compare(b, APK_BLOB_STR(tmp)) != 0)
continue;
delete = 0;
break;
}
} else {
/* Package - search for it */
name = apk_db_get_name(db, bname);
if (name == NULL)
break;
for (i = 0; i < name->pkgs->num; i++) {
struct apk_package *pkg = name->pkgs->item[i];
apk_pkg_format_cache(pkg, APK_BLOB_BUF(tmp));
if (apk_blob_compare(b, APK_BLOB_STR(tmp)) != 0)
continue;
delete = 0;
break;
}
}
} while (0);
if (delete) {
if (apk_verbosity >= 2)
apk_message("deleting %s", de->d_name);
if (!(apk_flags & APK_SIMULATE))
unlinkat(db->cache_fd, de->d_name, 0);
}
}
closedir(dir);
return 0;
return apk_db_cache_foreach_item(db, cache_clean_item);
}
static int cache_main(void *ctx, struct apk_database *db, int argc, char **argv)

View File

@ -1091,9 +1091,10 @@ static int apk_db_index_write_nr_cache(struct apk_database *db)
ctx.os = os;
list_for_each_entry(ipkg, &db->installed.packages, installed_pkgs_list) {
if (ipkg->pkg->repos != 0)
struct apk_package *pkg = ipkg->pkg;
if (pkg->repos != 0 || !pkg->in_cache)
continue;
r = write_index_entry(ipkg->pkg, &ctx);
r = write_index_entry(pkg, &ctx);
if (r != 0)
return r;
}
@ -1223,6 +1224,14 @@ static void relocate_database(struct apk_database *db)
apk_move_file(db->root_fd, apk_installed_file_old, apk_installed_file);
}
static void mark_in_cache(struct apk_database *db, const char *name, struct apk_package *pkg)
{
if (pkg == NULL)
return;
pkg->in_cache = 1;
}
int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
{
const char *msg = NULL;
@ -1245,8 +1254,8 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
apk_hash_init(&db->available.names, &pkg_name_hash_ops, 4000);
apk_hash_init(&db->available.packages, &pkg_info_hash_ops, 10000);
apk_hash_init(&db->installed.dirs, &dir_hash_ops, 10000);
apk_hash_init(&db->installed.files, &file_hash_ops, 20000);
apk_hash_init(&db->installed.dirs, &dir_hash_ops, 5000);
apk_hash_init(&db->installed.files, &file_hash_ops, 100000);
list_init(&db->installed.packages);
list_init(&db->installed.triggers);
apk_dependency_array_init(&db->world);
@ -1400,6 +1409,9 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
apk_db_index_write_nr_cache(db);
}
if (apk_db_cache_active(db))
apk_db_cache_foreach_item(db, mark_in_cache);
if (db->compat_newfeatures) {
apk_warning("This apk-tools is OLD! Some packages %s.",
db->compat_notinstallable ?
@ -1613,6 +1625,48 @@ int apk_db_cache_active(struct apk_database *db)
return db->cache_dir != apk_static_cache_dir;
}
struct foreach_cache_item_ctx {
struct apk_database *db;
apk_cache_item_cb cb;
};
static int foreach_cache_file(void *pctx, const char *name)
{
struct foreach_cache_item_ctx *ctx = (struct foreach_cache_item_ctx *) pctx;
struct apk_database *db = ctx->db;
struct apk_package *pkg = NULL;
apk_blob_t b = APK_BLOB_STR(name), bname, bver;
int i;
if (apk_pkg_parse_name(b, &bname, &bver) == 0) {
/* Package - search for it */
struct apk_name *name = apk_db_get_name(db, bname);
char tmp[PATH_MAX];
if (name == NULL)
goto no_pkg;
for (i = 0; i < name->pkgs->num; i++) {
struct apk_package *pkg0 = name->pkgs->item[i];
apk_pkg_format_cache(pkg0, APK_BLOB_BUF(tmp));
if (apk_blob_compare(b, APK_BLOB_STR(tmp)) == 0) {
pkg = pkg0;
break;
}
}
}
no_pkg:
ctx->cb(db, name, pkg);
return 0;
}
int apk_db_cache_foreach_item(struct apk_database *db, apk_cache_item_cb cb)
{
struct foreach_cache_item_ctx ctx = { db, cb };
return apk_dir_foreach_file(dup(db->cache_fd), foreach_cache_file, &ctx);
}
int apk_db_permanent(struct apk_database *db)
{
return db->permanent;
@ -1685,7 +1739,7 @@ int apk_repo_format_filename(char *buf, size_t len,
return n;
}
static int apk_repo_is_remote(struct apk_repository *repo)
int apk_repo_is_remote(struct apk_repository *repo)
{
return repo->csum.type != APK_CHECKSUM_NONE;
}
@ -1722,32 +1776,31 @@ struct apk_repository *apk_db_select_repo(struct apk_database *db,
unsigned int repos = pkg->repos & ~(db->bad_repos);
int i;
/* Always prefer local repositories */
if ((repos & db->local_repos) != 0)
repos &= db->local_repos;
/* Uninstalled and unavailable? */
if (repos == 0 && pkg->ipkg == NULL)
return NULL;
/* Pick first repository providing this package */
for (i = 0; i < APK_MAX_REPOS; i++)
if (repos & BIT(i))
break;
/* If this is a remote repository, and we have no network,
* check that we have it in cache */
if ((i >= APK_MAX_REPOS) ||
((db->local_repos & BIT(i)) == 0 && (apk_flags & APK_NO_NETWORK))) {
char cacheitem[PATH_MAX];
apk_pkg_format_cache(pkg, APK_BLOB_BUF(cacheitem));
if (faccessat(db->cache_fd, cacheitem, R_OK, 0) != 0)
if ((repos & db->local_repos) == 0) {
/* Network repository (or installed and unavailable) */
if (apk_flags & APK_NO_NETWORK) {
if (pkg->in_cache)
return &cache_repo;
return NULL;
}
} else {
/* Local repositories; don't use network */
repos &= db->local_repos;
}
if (i >= APK_MAX_REPOS)
return &cache_repo;
/* Pick first repository providing this package */
for (i = 0; i < APK_MAX_REPOS; i++) {
if (repos & BIT(i))
return &db->repos[i];
}
return NULL;
}
int apk_repository_update(struct apk_database *db, struct apk_repository *repo)
{
char cacheitem[PATH_MAX];
@ -2347,11 +2400,13 @@ static int apk_db_unpack_pkg(struct apk_database *db,
tar->close(tar);
if (need_copy) {
if (r == 0)
if (r == 0) {
renameat(db->cachetmp_fd, item, db->cache_fd, item);
else
pkg->in_cache = 1;
} else {
unlinkat(db->cachetmp_fd, item, 0);
}
}
if (r != 0) {
apk_error("%s: %s", file, apk_error_str(r));

View File

@ -14,6 +14,7 @@
#include <fcntl.h>
#include <unistd.h>
#include <malloc.h>
#include <dirent.h>
#include <sys/mman.h>
#include <sys/wait.h>
#include <sys/stat.h>
@ -542,6 +543,32 @@ int apk_file_get_info(int atfd, const char *filename, unsigned int flags,
return 0;
}
int apk_dir_foreach_file(int dirfd, apk_dir_file_cb cb, void *ctx)
{
struct dirent *de;
DIR *dir;
if (dirfd < 0)
return -1;
dir = fdopendir(dirfd);
if (dir == NULL)
return -1;
/* We get called here with dup():ed fd. Since they all refer to
* same object, we need to rewind so subsequent calls work. */
rewinddir(dir);
while ((de = readdir(dir)) != NULL) {
if (de->d_name[0] == '.')
continue;
cb(ctx, de->d_name);
}
closedir(dir);
return 0;
}
struct apk_istream *apk_istream_from_file_gz(int atfd, const char *file)
{
return apk_bstream_gunzip(apk_bstream_from_file(atfd, file));

View File

@ -86,8 +86,6 @@ struct apk_package_state {
unsigned int topology_soft;
unsigned short conflicts;
unsigned char preference;
unsigned availability_checked : 1;
unsigned unavailable : 1;
unsigned handle_install_if : 1;
unsigned locked : 1;
};
@ -294,13 +292,24 @@ static struct apk_name_state *name_to_ns_alloc(struct apk_name *name)
return ns;
}
static inline int pkg_available(struct apk_database *db, struct apk_package *pkg)
static inline int pkg_available(struct apk_solver_state *ss, struct apk_package *pkg)
{
struct apk_database *db = ss->db;
struct apk_name_state *ns = name_to_ns(pkg->name);
/* virtual packages - only deps used; no real .apk */
if (pkg->installed_size == 0)
return TRUE;
if (pkg->filename != NULL)
/* obviously present */
if (pkg->in_cache || pkg->filename != NULL)
return TRUE;
if (apk_db_select_repo(db, pkg) != NULL)
/* can download */
if ((pkg->repos & ~db->bad_repos) && !(apk_flags & APK_NO_NETWORK))
return TRUE;
/* installed, and no reinstall needed */
if ((pkg->ipkg != NULL) &&
(ns->inherited_reinstall == 0) &&
((ns->solver_flags_local|ss->solver_flags) & APK_SOLVERF_REINSTALL) == 0)
return TRUE;
return FALSE;
}
@ -410,7 +419,7 @@ static void get_topology_score(
score.non_preferred_actions++;
if (ns->locked || (ns->allowed_pinning | ns->maybe_pinning) == ns->allowed_pinning) {
allowed_pinning = ns->allowed_pinning | preferred_pinning;
allowed_pinning = ns->allowed_pinning | preferred_pinning | APK_DEFAULT_PINNING_MASK;
allowed_repos = get_pinning_mask_repos(ss->db, allowed_pinning);
if (!(repos & allowed_repos))
score.non_preferred_actions+=2;
@ -492,6 +501,14 @@ static void calculate_pkg_preference(struct apk_package *pkg)
}
}
static void count_name(struct apk_solver_state *ss, struct apk_name_state *ns)
{
if (!ns->decision_counted) {
ss->max_decisions++;
ns->decision_counted = 1;
}
}
static void sort_hard_dependencies(struct apk_solver_state *ss, struct apk_package *pkg)
{
struct apk_package_state *ps;
@ -505,19 +522,15 @@ static void sort_hard_dependencies(struct apk_solver_state *ss, struct apk_packa
return;
pkg->topology_hard = -1;
ps->topology_soft = -1;
calculate_pkg_preference(pkg);
calculate_pkg_preference(pkg);
/* Consider hard dependencies only */
foreach_dependency_pkg(ss, pkg->depends, sort_hard_dependencies);
foreach_dependency_pkg(ss, pkg->install_if, sort_hard_dependencies);
ss->max_decisions++;
ns = name_to_ns_alloc(pkg->name);
if (!ns->decision_counted) {
ss->max_decisions++;
ns->decision_counted = 1;
}
count_name(ss, ns);
ps->topology_soft = pkg->topology_hard = ++ss->num_topology_positions;
dbg_printf(PKG_VER_FMT ": topology_hard=%d\n",
@ -587,6 +600,7 @@ static void sort_name(struct apk_solver_state *ss, struct apk_name *name)
for (i = 0; i < name->pkgs->num; i++)
sort_soft_dependencies(ss, name->pkgs->item[i]);
count_name(ss, ns);
recalculate_maybe(ss, name,
ns->solver_flags_local & ns->solver_flags_local_mask,
ns->maybe_pinning);
@ -609,36 +623,16 @@ static int install_if_missing(struct apk_solver_state *ss, struct apk_package *p
struct apk_dependency *dep = &pkg->install_if->item[i];
ns = name_to_ns(dep->name);
if (!ns->locked || !apk_dep_is_satisfied(dep, ns->chosen))
/* ns can be NULL, if the install_if has a name with
* no packages */
if (ns == NULL || !ns->locked || !apk_dep_is_satisfied(dep, ns->chosen))
missing++;
}
return missing;
}
static int check_if_package_unavailable(struct apk_solver_state *ss, struct apk_package *pkg, int do_check)
{
struct apk_name *name = pkg->name;
struct apk_package_state *ps = pkg_to_ps(pkg);
struct apk_name_state *ns = name_to_ns(name);
/* installed and no-reinstall required? no check needed. */
if ((pkg->ipkg != NULL) && (ns->inherited_reinstall == 0) &&
((ns->solver_flags_local|ss->solver_flags) & APK_SOLVERF_REINSTALL) == 0)
return 0;
/* done already? */
if (ps->availability_checked && !do_check)
return ps->unavailable;
/* and it's not available, we can't use it */
if (!pkg_available(ss->db, pkg))
ps->unavailable = 1;
ps->availability_checked = 1;
return ps->unavailable;
}
static void foreach_common_dependency(
struct apk_solver_state *ss, struct apk_name *name,
void (*cb)(struct apk_solver_state *ss, struct apk_name *common_dependency))
@ -719,7 +713,7 @@ static int update_name_state(struct apk_solver_state *ss, struct apk_name *name)
if (ps0 == NULL || ps0->locked ||
ss->topology_position < pkg0->topology_hard ||
check_if_package_unavailable(ss, pkg0, 0))
!pkg_available(ss, pkg0))
continue;
/* preferred - currently most optimal for end solution */
@ -828,7 +822,7 @@ static void trigger_install_if(struct apk_solver_state *ss,
{
if (install_if_missing(ss, pkg) == 0) {
struct apk_name *name0 = decision_to_name(&ss->decisions[ss->num_decisions]);
struct apk_name_state *ns = ns = name_to_ns(pkg->name);
struct apk_name_state *ns = name_to_ns(pkg->name);
dbg_printf("trigger_install_if: " PKG_VER_FMT " triggered\n",
PKG_VER_PRINTF(pkg));
@ -913,8 +907,7 @@ static solver_result_t apply_decision(struct apk_solver_state *ss,
get_topology_score(ss, ns, pkg, &score);
addscore(&ss->score, &score);
if (cmpscore2(&ss->score, &ss->minimum_penalty, &ss->best_score) >= 0 ||
check_if_package_unavailable(ss, pkg, 1)) {
if (cmpscore2(&ss->score, &ss->minimum_penalty, &ss->best_score) >= 0) {
dbg_printf("install causing "SCORE_FMT", penalty too big: "SCORE_FMT"+"SCORE_FMT">="SCORE_FMT"\n",
SCORE_PRINTF(&score),
SCORE_PRINTF(&ss->score),
@ -1286,7 +1279,8 @@ static int expand_branch(struct apk_solver_state *ss)
continue;
get_topology_score(ss, ns, pkg0, &pkgscore);
if (cmpscore2(&score, &pkgscore, &ss->best_score) >= 0)
if (cmpscore2(&score, &pkgscore, &ss->best_score) >= 0 ||
!pkg_available(ss, pkg0))
return push_decision(ss, name, pkg0, DECISION_EXCLUDE, BRANCH_NO, FALSE);
}
@ -1318,7 +1312,7 @@ static int expand_branch(struct apk_solver_state *ss)
SCORE_PRINTF(&ss->best_score));
preferred_pinning = ns->preferred_pinning ?: APK_DEFAULT_PINNING_MASK;
allowed_pinning = ns->allowed_pinning | preferred_pinning;
allowed_pinning = ns->allowed_pinning | preferred_pinning | APK_DEFAULT_PINNING_MASK;
allowed_repos = get_pinning_mask_repos(ss->db, allowed_pinning);
if ((pkg0->repos != 0) && !(pkg0->repos & allowed_repos)) {