enable automatic update of indexes controlled by --cache-max-age
This modifies apk cache for indexes to be automatically refreshed periodically without explicit 'update' or '--update-cache' usage. The default is to do if-modified-since request if the local copy is older than 4 hours. This age can be changed with --cache-max-age. Using --update-cache will change this age to 60 seconds to make sure the cached copy is relatively new. The small age is in order to try to avoid downloading indexes second time when apk-tools is upgraded and apk re-execs after self-upgrade. Accordingly using explicitly 'apk update' will now enforce --force-refresh and request the very latest index by requesting any potential http proxy to do refresh too.cute-signatures
Timo Teräs
parent
2da67940d5
commit
8a28c6d0d4
16
src/apk.c
16
src/apk.c
|
|
@ -120,9 +120,6 @@ static int option_parse_global(void *ctx, struct apk_db_options *dbopts, int opt
|
|||
case 'i':
|
||||
apk_flags |= APK_INTERACTIVE;
|
||||
break;
|
||||
case 'U':
|
||||
apk_flags |= APK_UPDATE_CACHE;
|
||||
break;
|
||||
case 0x101:
|
||||
apk_flags |= APK_PROGRESS;
|
||||
break;
|
||||
|
|
@ -153,6 +150,14 @@ static int option_parse_global(void *ctx, struct apk_db_options *dbopts, int opt
|
|||
case 0x116:
|
||||
dbopts->cache_dir = optarg;
|
||||
break;
|
||||
case 'U':
|
||||
/* Make it one minute, to avoid updating indexes twice
|
||||
* when doing self-upgrade's re-exec */
|
||||
dbopts->cache_max_age = 60;
|
||||
break;
|
||||
case 0x119:
|
||||
dbopts->cache_max_age = atoi(optarg) * 60;
|
||||
break;
|
||||
case 0x112:
|
||||
dbopts->arch = optarg;
|
||||
break;
|
||||
|
|
@ -193,7 +198,7 @@ static const struct apk_option options_global[] = {
|
|||
{ 0x121, "force-old-apk", "Continue even if packages use unsupported features" },
|
||||
{ 0x120, "force-overwrite", "Overwrite files in other packages" },
|
||||
{ 0x123, "force-refresh", "Do not use cached files (local or from proxy)" },
|
||||
{ 'U', "update-cache", "Update the repository cache" },
|
||||
{ 'U', "update-cache", "Alias for --cache-max-age 60" },
|
||||
{ 0x101, "progress", "Show a progress bar" },
|
||||
{ 0x10f, "progress-fd", "Write progress to fd", required_argument, "FD" },
|
||||
{ 0x110, "no-progress", "Disable progress bar even for TTYs" },
|
||||
|
|
@ -211,6 +216,8 @@ static const struct apk_option options_global[] = {
|
|||
{ 0x115, "no-cache", "Do not use any local cache path" },
|
||||
{ 0x116, "cache-dir", "Override cache directory",
|
||||
required_argument, "CACHEDIR" },
|
||||
{ 0x119, "cache-max-age", "Maximum AGE (in minutes) for index in cache before refresh",
|
||||
required_argument, "AGE" },
|
||||
{ 0x112, "arch", "Use architecture with --root",
|
||||
required_argument, "ARCH" },
|
||||
{ 0x114, "print-arch", "Print default arch and exit" },
|
||||
|
|
@ -528,6 +535,7 @@ int main(int argc, char **argv)
|
|||
ctx = calloc(1, applet->context_size);
|
||||
dbopts.open_flags = applet->open_flags;
|
||||
apk_flags |= applet->forced_flags;
|
||||
apk_force |= applet->forced_force;
|
||||
}
|
||||
for (opt = all_options, sopt = short_options; opt->name != NULL; opt++) {
|
||||
if (opt->flag == NULL &&
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ struct apk_applet {
|
|||
const char *help;
|
||||
const struct apk_option_group *optgroups[4];
|
||||
|
||||
unsigned int open_flags, forced_flags;
|
||||
unsigned int open_flags, forced_flags, forced_force;
|
||||
int context_size;
|
||||
|
||||
int (*main)(void *ctx, struct apk_database *db, struct apk_string_array *args);
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ struct apk_repository_list {
|
|||
|
||||
struct apk_db_options {
|
||||
int lock_wait;
|
||||
unsigned int cache_max_age;
|
||||
unsigned long open_flags;
|
||||
const char *root;
|
||||
const char *arch;
|
||||
|
|
@ -150,8 +151,8 @@ struct apk_database {
|
|||
char *cache_remount_dir, *root_proc_dir;
|
||||
unsigned long cache_remount_flags;
|
||||
apk_blob_t *arch;
|
||||
unsigned int local_repos, available_repos;
|
||||
int repo_update_errors;
|
||||
unsigned int local_repos, available_repos, cache_max_age;
|
||||
unsigned int repo_update_errors, repo_update_counter;
|
||||
unsigned int pending_triggers;
|
||||
int performing_self_upgrade : 1;
|
||||
int permanent : 1;
|
||||
|
|
@ -246,7 +247,7 @@ unsigned int apk_db_get_pinning_mask_repos(struct apk_database *db, unsigned sho
|
|||
|
||||
int apk_db_cache_active(struct apk_database *db);
|
||||
int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
|
||||
struct apk_package *pkg, int verify,
|
||||
struct apk_package *pkg, int verify, int autoupdate,
|
||||
apk_progress_cb cb, void *cb_ctx);
|
||||
|
||||
typedef void (*apk_cache_item_cb)(struct apk_database *db,
|
||||
|
|
|
|||
|
|
@ -68,7 +68,6 @@ extern char **apk_argv;
|
|||
#define APK_CLEAN_PROTECTED 0x0004
|
||||
#define APK_PROGRESS 0x0008
|
||||
#define APK_RECURSIVE 0x0020
|
||||
#define APK_UPDATE_CACHE 0x0080
|
||||
#define APK_ALLOW_UNTRUSTED 0x0100
|
||||
#define APK_PURGE 0x0200
|
||||
#define APK_INTERACTIVE 0x0400
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream);
|
|||
struct apk_bstream *apk_bstream_from_fd_pid(int fd, pid_t pid, int (*translate_status)(int));
|
||||
struct apk_bstream *apk_bstream_from_file(int atfd, const char *file);
|
||||
struct apk_bstream *apk_bstream_from_fd_url_if_modified(int atfd, const char *url, time_t since);
|
||||
struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to,
|
||||
struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to, int copy_meta,
|
||||
apk_progress_cb cb, void *cb_ctx);
|
||||
|
||||
static inline struct apk_bstream *apk_bstream_from_fd(int fd)
|
||||
|
|
|
|||
|
|
@ -66,9 +66,9 @@ static int cache_download(struct apk_database *db)
|
|||
if (repo == NULL)
|
||||
continue;
|
||||
|
||||
r = apk_cache_download(db, repo, pkg, APK_SIGN_VERIFY_IDENTITY,
|
||||
r = apk_cache_download(db, repo, pkg, APK_SIGN_VERIFY_IDENTITY, 0,
|
||||
progress_cb, &prog);
|
||||
if (r) {
|
||||
if (r && r != -EALREADY) {
|
||||
apk_error(PKG_VER_FMT ": %s", PKG_VER_PRINTF(pkg), apk_error_str(r));
|
||||
ret++;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -611,10 +611,10 @@ int apk_repo_format_item(struct apk_database *db, struct apk_repository *repo, s
|
|||
}
|
||||
|
||||
int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
|
||||
struct apk_package *pkg, int verify,
|
||||
struct apk_package *pkg, int verify, int autoupdate,
|
||||
apk_progress_cb cb, void *cb_ctx)
|
||||
{
|
||||
struct stat st;
|
||||
struct stat st = {0};
|
||||
struct apk_istream *is;
|
||||
struct apk_bstream *bs;
|
||||
struct apk_sign_ctx sctx;
|
||||
|
|
@ -622,6 +622,7 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
|
|||
char tmpcacheitem[128], *cacheitem = &tmpcacheitem[tmpprefix.len];
|
||||
apk_blob_t b = APK_BLOB_BUF(tmpcacheitem);
|
||||
int r, fd;
|
||||
time_t now = time(NULL);
|
||||
|
||||
apk_blob_push_blob(&b, tmpprefix);
|
||||
if (pkg != NULL)
|
||||
|
|
@ -633,9 +634,9 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
|
|||
r = apk_repo_format_real_url(db, repo, pkg, url, sizeof(url));
|
||||
if (r < 0) return r;
|
||||
|
||||
if ((apk_force & APK_FORCE_REFRESH) ||
|
||||
fstatat(db->cache_fd, cacheitem, &st, 0) != 0)
|
||||
st.st_mtime = 0;
|
||||
if (!(apk_force & APK_FORCE_REFRESH))
|
||||
(void) fstatat(db->cache_fd, cacheitem, &st, 0);
|
||||
if (autoupdate && now - st.st_mtime <= db->cache_max_age) return -EALREADY;
|
||||
|
||||
apk_message("fetch %s", url);
|
||||
|
||||
|
|
@ -645,7 +646,7 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
|
|||
if (verify != APK_SIGN_NONE) {
|
||||
apk_sign_ctx_init(&sctx, APK_SIGN_VERIFY, NULL, db->keys_fd);
|
||||
bs = apk_bstream_from_url_if_modified(url, st.st_mtime);
|
||||
bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, cb, cb_ctx);
|
||||
bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, !autoupdate, cb, cb_ctx);
|
||||
is = apk_bstream_gunzip_mpart(bs, apk_sign_ctx_mpart_cb, &sctx);
|
||||
if (!IS_ERR_OR_NULL(is))
|
||||
r = apk_tar_parse(is, apk_sign_ctx_verify_tar, &sctx, FALSE, &db->id_cache);
|
||||
|
|
@ -662,13 +663,18 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
|
|||
if (fd >= 0) {
|
||||
struct apk_file_meta meta;
|
||||
r = apk_istream_splice(is, fd, APK_SPLICE_ALL, cb, cb_ctx);
|
||||
apk_istream_get_meta(is, &meta);
|
||||
apk_file_meta_to_fd(fd, &meta);
|
||||
if (!autoupdate) {
|
||||
apk_istream_get_meta(is, &meta);
|
||||
apk_file_meta_to_fd(fd, &meta);
|
||||
}
|
||||
close(fd);
|
||||
}
|
||||
}
|
||||
if (!IS_ERR_OR_NULL(is)) apk_istream_close(is);
|
||||
if (r == -EALREADY) return 0;
|
||||
if (r == -EALREADY) {
|
||||
if (autoupdate) utimensat(db->cache_fd, cacheitem, NULL, 0);
|
||||
return r;
|
||||
}
|
||||
if (r < 0) {
|
||||
unlinkat(db->cache_fd, tmpcacheitem, 0);
|
||||
return r;
|
||||
|
|
@ -1515,6 +1521,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
|
|||
|
||||
apk_db_setup_repositories(db, dbopts->cache_dir);
|
||||
|
||||
db->cache_max_age = dbopts->cache_max_age ?: 4*60*60; /* 4 hours default */
|
||||
db->root = strdup(dbopts->root ?: "/");
|
||||
db->root_fd = openat(AT_FDCWD, db->root, O_RDONLY | O_CLOEXEC);
|
||||
if (db->root_fd < 0 && (dbopts->open_flags & APK_OPENF_CREATE)) {
|
||||
|
|
@ -1677,7 +1684,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
|
|||
add_repos_from_file(db, db->root_fd, dbopts->repositories_file);
|
||||
}
|
||||
|
||||
if (apk_flags & APK_UPDATE_CACHE)
|
||||
if (db->repo_update_counter)
|
||||
apk_db_index_write_nr_cache(db);
|
||||
|
||||
apk_hash_foreach(&db->available.names, apk_db_name_rdepends, db);
|
||||
|
|
@ -2118,10 +2125,13 @@ static int apk_repository_update(struct apk_database *db, struct apk_repository
|
|||
{
|
||||
int r, verify = (apk_flags & APK_ALLOW_UNTRUSTED) ? APK_SIGN_NONE : APK_SIGN_VERIFY;
|
||||
|
||||
r = apk_cache_download(db, repo, NULL, verify, NULL, NULL);
|
||||
r = apk_cache_download(db, repo, NULL, verify, 1, NULL, NULL);
|
||||
if (r == -EALREADY) return 0;
|
||||
if (r != 0) {
|
||||
apk_error("%s: %s", repo->url, apk_error_str(r));
|
||||
db->repo_update_errors++;
|
||||
} else {
|
||||
db->repo_update_counter++;
|
||||
}
|
||||
|
||||
return r;
|
||||
|
|
@ -2247,16 +2257,13 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t _repository)
|
|||
apk_blob_checksum(brepo, apk_checksum_default(), &repo->csum);
|
||||
|
||||
if (apk_url_local_file(repo->url) == NULL) {
|
||||
if (!(apk_flags & APK_NO_NETWORK)) {
|
||||
if (!(apk_flags & APK_NO_NETWORK))
|
||||
db->available_repos |= BIT(repo_num);
|
||||
if (apk_flags & APK_UPDATE_CACHE)
|
||||
apk_repository_update(db, repo);
|
||||
}
|
||||
if (apk_flags & APK_NO_CACHE) {
|
||||
r = apk_repo_format_real_url(db, repo, NULL, buf, sizeof(buf));
|
||||
if (r == 0)
|
||||
apk_message("fetch %s", buf);
|
||||
if (r == 0) apk_message("fetch %s", buf);
|
||||
} else {
|
||||
apk_repository_update(db, repo);
|
||||
r = apk_repo_format_cache_index(APK_BLOB_BUF(buf), repo);
|
||||
}
|
||||
} else {
|
||||
|
|
@ -2723,7 +2730,7 @@ static int apk_db_unpack_pkg(struct apk_database *db,
|
|||
apk_blob_t b = APK_BLOB_BUF(tmpcacheitem);
|
||||
apk_blob_push_blob(&b, tmpprefix);
|
||||
apk_pkg_format_cache_pkg(b, pkg);
|
||||
cache_bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, NULL, NULL);
|
||||
cache_bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, 1, NULL, NULL);
|
||||
if (!IS_ERR_OR_NULL(cache_bs))
|
||||
bs = cache_bs;
|
||||
else
|
||||
|
|
|
|||
12
src/io.c
12
src/io.c
|
|
@ -441,7 +441,7 @@ struct apk_bstream *apk_bstream_from_file(int atfd, const char *file)
|
|||
struct apk_tee_bstream {
|
||||
struct apk_bstream bs;
|
||||
struct apk_bstream *inner_bs;
|
||||
int fd;
|
||||
int fd, copy_meta;
|
||||
size_t size;
|
||||
apk_progress_cb cb;
|
||||
void *cb_ctx;
|
||||
|
|
@ -475,9 +475,10 @@ static void tee_close(void *stream, size_t *size)
|
|||
struct apk_tee_bstream *tbs =
|
||||
container_of(stream, struct apk_tee_bstream, bs);
|
||||
|
||||
/* copy info */
|
||||
apk_bstream_get_meta(tbs->inner_bs, &meta);
|
||||
apk_file_meta_to_fd(tbs->fd, &meta);
|
||||
if (tbs->copy_meta) {
|
||||
apk_bstream_get_meta(tbs->inner_bs, &meta);
|
||||
apk_file_meta_to_fd(tbs->fd, &meta);
|
||||
}
|
||||
|
||||
apk_bstream_close(tbs->inner_bs, NULL);
|
||||
if (size != NULL) *size = tbs->size;
|
||||
|
|
@ -491,7 +492,7 @@ static const struct apk_bstream_ops tee_bstream_ops = {
|
|||
.close = tee_close,
|
||||
};
|
||||
|
||||
struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to, apk_progress_cb cb, void *cb_ctx)
|
||||
struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to, int copy_meta, apk_progress_cb cb, void *cb_ctx)
|
||||
{
|
||||
struct apk_tee_bstream *tbs;
|
||||
int fd, r;
|
||||
|
|
@ -519,6 +520,7 @@ struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const ch
|
|||
};
|
||||
tbs->inner_bs = from;
|
||||
tbs->fd = fd;
|
||||
tbs->copy_meta = copy_meta;
|
||||
tbs->size = 0;
|
||||
tbs->cb = cb;
|
||||
tbs->cb_ctx = cb_ctx;
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ static struct apk_applet apk_update = {
|
|||
.name = "update",
|
||||
.help = "Update repository indexes from all remote repositories",
|
||||
.open_flags = APK_OPENF_WRITE,
|
||||
.forced_flags = APK_UPDATE_CACHE,
|
||||
.forced_force = APK_FORCE_REFRESH,
|
||||
.main = update_main,
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue