enable automatic update of indexes controlled by --cache-max-age

This modifies apk cache for indexes to be automatically refreshed
periodically without explicit 'update' or '--update-cache' usage.

The default is to do if-modified-since request if the local copy
is older than 4 hours. This age can be changed with --cache-max-age.
Using --update-cache will change this age to 60 seconds to make
sure the cached copy is relatively new. The small age is in order
to try to avoid downloading indexes second time when apk-tools is
upgraded and apk re-execs after self-upgrade.

Accordingly using explicitly 'apk update' will now enforce
--force-refresh and request the very latest index by requesting
any potential http proxy to do refresh too.

src/apk.c

@@ -120,9 +120,6 @@ static int option_parse_global(void *ctx, struct apk_db_options *dbopts, int opt
 	case 'i':
 		apk_flags |= APK_INTERACTIVE;
 		break;
-	case 'U':
-		apk_flags |= APK_UPDATE_CACHE;
-		break;
 	case 0x101:
 		apk_flags |= APK_PROGRESS;
 		break;
@@ -153,6 +150,14 @@ static int option_parse_global(void *ctx, struct apk_db_options *dbopts, int opt
 	case 0x116:
 		dbopts->cache_dir = optarg;
 		break;
+	case 'U':
+		/* Make it one minute, to avoid updating indexes twice
+		 * when doing self-upgrade's re-exec */
+		dbopts->cache_max_age = 60;
+		break;
+	case 0x119:
+		dbopts->cache_max_age = atoi(optarg) * 60;
+		break;
 	case 0x112:
 		dbopts->arch = optarg;
 		break;
@@ -193,7 +198,7 @@ static const struct apk_option options_global[] = {
 	{ 0x121, "force-old-apk", "Continue even if packages use unsupported features" },
 	{ 0x120, "force-overwrite", "Overwrite files in other packages" },
 	{ 0x123, "force-refresh", "Do not use cached files (local or from proxy)" },
-	{ 'U', "update-cache",	"Update the repository cache" },
+	{ 'U', "update-cache",	"Alias for --cache-max-age 60" },
 	{ 0x101, "progress",	"Show a progress bar" },
 	{ 0x10f, "progress-fd",	"Write progress to fd", required_argument, "FD" },
 	{ 0x110, "no-progress",	"Disable progress bar even for TTYs" },
@@ -211,6 +216,8 @@ static const struct apk_option options_global[] = {
 	{ 0x115, "no-cache",	"Do not use any local cache path" },
 	{ 0x116, "cache-dir",	"Override cache directory",
 				required_argument, "CACHEDIR" },
+	{ 0x119, "cache-max-age", "Maximum AGE (in minutes) for index in cache before refresh",
+				required_argument, "AGE" },
 	{ 0x112, "arch",	"Use architecture with --root",
 				required_argument, "ARCH" },
 	{ 0x114, "print-arch",	"Print default arch and exit" },
@@ -528,6 +535,7 @@ int main(int argc, char **argv)
 			ctx = calloc(1, applet->context_size);
 		dbopts.open_flags = applet->open_flags;
 		apk_flags |= applet->forced_flags;
+		apk_force |= applet->forced_force;
 	}
 	for (opt = all_options, sopt = short_options; opt->name != NULL; opt++) {
 		if (opt->flag == NULL &&

src/apk_applet.h

@@ -42,7 +42,7 @@ struct apk_applet {
 	const char *help;
 	const struct apk_option_group *optgroups[4];
 
-	unsigned int open_flags, forced_flags;
+	unsigned int open_flags, forced_flags, forced_force;
 	int context_size;
 
 	int (*main)(void *ctx, struct apk_database *db, struct apk_string_array *args);

src/apk_database.h

@@ -122,6 +122,7 @@ struct apk_repository_list {
 
 struct apk_db_options {
 	int lock_wait;
+	unsigned int cache_max_age;
 	unsigned long open_flags;
 	const char *root;
 	const char *arch;
@@ -150,8 +151,8 @@ struct apk_database {
 	char *cache_remount_dir, *root_proc_dir;
 	unsigned long cache_remount_flags;
 	apk_blob_t *arch;
-	unsigned int local_repos, available_repos;
+	unsigned int local_repos, available_repos, cache_max_age;
-	int repo_update_errors;
+	unsigned int repo_update_errors, repo_update_counter;
 	unsigned int pending_triggers;
 	int performing_self_upgrade : 1;
 	int permanent : 1;
@@ -246,7 +247,7 @@ unsigned int apk_db_get_pinning_mask_repos(struct apk_database *db, unsigned sho
 
 int apk_db_cache_active(struct apk_database *db);
 int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
-		       struct apk_package *pkg, int verify,
+		       struct apk_package *pkg, int verify, int autoupdate,
 		       apk_progress_cb cb, void *cb_ctx);
 
 typedef void (*apk_cache_item_cb)(struct apk_database *db,

src/apk_defines.h

@@ -68,7 +68,6 @@ extern char **apk_argv;
 #define APK_CLEAN_PROTECTED	0x0004
 #define APK_PROGRESS		0x0008
 #define APK_RECURSIVE		0x0020
-#define APK_UPDATE_CACHE	0x0080
 #define APK_ALLOW_UNTRUSTED	0x0100
 #define APK_PURGE		0x0200
 #define APK_INTERACTIVE		0x0400

src/apk_io.h

@@ -147,7 +147,7 @@ struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream);
 struct apk_bstream *apk_bstream_from_fd_pid(int fd, pid_t pid, int (*translate_status)(int));
 struct apk_bstream *apk_bstream_from_file(int atfd, const char *file);
 struct apk_bstream *apk_bstream_from_fd_url_if_modified(int atfd, const char *url, time_t since);
-struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to,
+struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to, int copy_meta,
 				    apk_progress_cb cb, void *cb_ctx);
 
 static inline struct apk_bstream *apk_bstream_from_fd(int fd)

src/cache.c

@@ -66,9 +66,9 @@ static int cache_download(struct apk_database *db)
 		if (repo == NULL)
 			continue;
 
-		r = apk_cache_download(db, repo, pkg, APK_SIGN_VERIFY_IDENTITY,
+		r = apk_cache_download(db, repo, pkg, APK_SIGN_VERIFY_IDENTITY, 0,
 				       progress_cb, &prog);
-		if (r) {
+		if (r && r != -EALREADY) {
 			apk_error(PKG_VER_FMT ": %s", PKG_VER_PRINTF(pkg), apk_error_str(r));
 			ret++;
 		}

src/database.c

@@ -611,10 +611,10 @@ int apk_repo_format_item(struct apk_database *db, struct apk_repository *repo, s
 }
 
 int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
-		       struct apk_package *pkg, int verify,
+		       struct apk_package *pkg, int verify, int autoupdate,
 		       apk_progress_cb cb, void *cb_ctx)
 {
-	struct stat st;
+	struct stat st = {0};
 	struct apk_istream *is;
 	struct apk_bstream *bs;
 	struct apk_sign_ctx sctx;
@@ -622,6 +622,7 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
 	char tmpcacheitem[128], *cacheitem = &tmpcacheitem[tmpprefix.len];
 	apk_blob_t b = APK_BLOB_BUF(tmpcacheitem);
 	int r, fd;
+	time_t now = time(NULL);
 
 	apk_blob_push_blob(&b, tmpprefix);
 	if (pkg != NULL)
@@ -633,9 +634,9 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
 	r = apk_repo_format_real_url(db, repo, pkg, url, sizeof(url));
 	if (r < 0) return r;
 
-	if ((apk_force & APK_FORCE_REFRESH) ||
+	if (!(apk_force & APK_FORCE_REFRESH))
-	    fstatat(db->cache_fd, cacheitem, &st, 0) != 0)
+		(void) fstatat(db->cache_fd, cacheitem, &st, 0);
-		st.st_mtime = 0;
+	if (autoupdate && now - st.st_mtime <= db->cache_max_age) return -EALREADY;
 
 	apk_message("fetch %s", url);
 
@@ -645,7 +646,7 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
 	if (verify != APK_SIGN_NONE) {
 		apk_sign_ctx_init(&sctx, APK_SIGN_VERIFY, NULL, db->keys_fd);
 		bs = apk_bstream_from_url_if_modified(url, st.st_mtime);
-		bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, cb, cb_ctx);
+		bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, !autoupdate, cb, cb_ctx);
 		is = apk_bstream_gunzip_mpart(bs, apk_sign_ctx_mpart_cb, &sctx);
 		if (!IS_ERR_OR_NULL(is))
 			r = apk_tar_parse(is, apk_sign_ctx_verify_tar, &sctx, FALSE, &db->id_cache);
@@ -662,13 +663,18 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
 		if (fd >= 0) {
 			struct apk_file_meta meta;
 			r = apk_istream_splice(is, fd, APK_SPLICE_ALL, cb, cb_ctx);
+			if (!autoupdate) {
 				apk_istream_get_meta(is, &meta);
 				apk_file_meta_to_fd(fd, &meta);
+			}
 			close(fd);
 		}
 	}
 	if (!IS_ERR_OR_NULL(is)) apk_istream_close(is);
-	if (r == -EALREADY) return 0;
+	if (r == -EALREADY) {
+		if (autoupdate) utimensat(db->cache_fd, cacheitem, NULL, 0);
+		return r;
+	}
 	if (r < 0) {
 		unlinkat(db->cache_fd, tmpcacheitem, 0);
 		return r;
@@ -1515,6 +1521,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
 
 	apk_db_setup_repositories(db, dbopts->cache_dir);
 
+	db->cache_max_age = dbopts->cache_max_age ?: 4*60*60; /* 4 hours default */
 	db->root = strdup(dbopts->root ?: "/");
 	db->root_fd = openat(AT_FDCWD, db->root, O_RDONLY | O_CLOEXEC);
 	if (db->root_fd < 0 && (dbopts->open_flags & APK_OPENF_CREATE)) {
@@ -1677,7 +1684,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
 			add_repos_from_file(db, db->root_fd, dbopts->repositories_file);
 		}
 
-		if (apk_flags & APK_UPDATE_CACHE)
+		if (db->repo_update_counter)
 			apk_db_index_write_nr_cache(db);
 
 		apk_hash_foreach(&db->available.names, apk_db_name_rdepends, db);
@@ -2118,10 +2125,13 @@ static int apk_repository_update(struct apk_database *db, struct apk_repository
 {
 	int r, verify = (apk_flags & APK_ALLOW_UNTRUSTED) ? APK_SIGN_NONE : APK_SIGN_VERIFY;
 
-	r = apk_cache_download(db, repo, NULL, verify, NULL, NULL);
+	r = apk_cache_download(db, repo, NULL, verify, 1, NULL, NULL);
+	if (r == -EALREADY) return 0;
 	if (r != 0) {
 		apk_error("%s: %s", repo->url, apk_error_str(r));
 		db->repo_update_errors++;
+	} else {
+		db->repo_update_counter++;
 	}
 
 	return r;
@@ -2247,16 +2257,13 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t _repository)
 	apk_blob_checksum(brepo, apk_checksum_default(), &repo->csum);
 
 	if (apk_url_local_file(repo->url) == NULL) {
-		if (!(apk_flags & APK_NO_NETWORK)) {
+		if (!(apk_flags & APK_NO_NETWORK))
 			db->available_repos |= BIT(repo_num);
-			if (apk_flags & APK_UPDATE_CACHE)
-				apk_repository_update(db, repo);
-		}
 		if (apk_flags & APK_NO_CACHE) {
 			r = apk_repo_format_real_url(db, repo, NULL, buf, sizeof(buf));
-			if (r == 0)
+			if (r == 0) apk_message("fetch %s", buf);
-				apk_message("fetch %s", buf);
 		} else {
+			apk_repository_update(db, repo);
 			r = apk_repo_format_cache_index(APK_BLOB_BUF(buf), repo);
 		}
 	} else {
@@ -2723,7 +2730,7 @@ static int apk_db_unpack_pkg(struct apk_database *db,
 		apk_blob_t b = APK_BLOB_BUF(tmpcacheitem);
 		apk_blob_push_blob(&b, tmpprefix);
 		apk_pkg_format_cache_pkg(b, pkg);
-		cache_bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, NULL, NULL);
+		cache_bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, 1, NULL, NULL);
 		if (!IS_ERR_OR_NULL(cache_bs))
 			bs = cache_bs;
 		else

src/io.c

@@ -441,7 +441,7 @@ struct apk_bstream *apk_bstream_from_file(int atfd, const char *file)
 struct apk_tee_bstream {
 	struct apk_bstream bs;
 	struct apk_bstream *inner_bs;
-	int fd;
+	int fd, copy_meta;
 	size_t size;
 	apk_progress_cb cb;
 	void *cb_ctx;
@@ -475,9 +475,10 @@ static void tee_close(void *stream, size_t *size)
 	struct apk_tee_bstream *tbs =
 		container_of(stream, struct apk_tee_bstream, bs);
 
-	/* copy info */
+	if (tbs->copy_meta) {
 		apk_bstream_get_meta(tbs->inner_bs, &meta);
 		apk_file_meta_to_fd(tbs->fd, &meta);
+	}
 
 	apk_bstream_close(tbs->inner_bs, NULL);
 	if (size != NULL) *size = tbs->size;
@@ -491,7 +492,7 @@ static const struct apk_bstream_ops tee_bstream_ops = {
 	.close = tee_close,
 };
 
-struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to, apk_progress_cb cb, void *cb_ctx)
+struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to, int copy_meta, apk_progress_cb cb, void *cb_ctx)
 {
 	struct apk_tee_bstream *tbs;
 	int fd, r;
@@ -519,6 +520,7 @@ struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const ch
 	};
 	tbs->inner_bs = from;
 	tbs->fd = fd;
+	tbs->copy_meta = copy_meta;
 	tbs->size = 0;
 	tbs->cb = cb;
 	tbs->cb_ctx = cb_ctx;

src/update.c

@@ -49,7 +49,7 @@ static struct apk_applet apk_update = {
 	.name = "update",
 	.help = "Update repository indexes from all remote repositories",
 	.open_flags = APK_OPENF_WRITE,
-	.forced_flags = APK_UPDATE_CACHE,
+	.forced_force = APK_FORCE_REFRESH,
 	.main = update_main,
 };