various: make fancy progress bar and update todo

TODO

@@ -1,3 +1,12 @@
+- list of files on stdin, minimum dependencies on stdout
+- way to test if packages is installed
+
+- confirm whether to act (show changeset-size, installed, removed) if
+  other packages affected then the ones explicitly specified
+
+- complete lbu replacement
+- links for lbu* and apk_*
+
 - Index/pkginfo reader: same field multiple times -> memleak
 
 - Compress 'installed' and 'scripts'
@@ -23,7 +32,6 @@
 - Remember counts for hash table creation
 
 - Possibly create a token hash for package names, versions and licenses, etc.
-- Calculate changeset installed-size change
 - Option to not read fs entry cache
 
 - Special packages?:

src/apk.c

@@ -22,7 +22,7 @@
 
 const char *apk_root = "/";
 const char *apk_repository = NULL;
-int apk_quiet = 0;
+int apk_quiet = 0, apk_progress = 0;
 int apk_cwd_fd;
 
 void apk_log(const char *prefix, const char *format, ...)
@@ -73,6 +73,7 @@ int main(int argc, char **argv)
 		{"root", required_argument, NULL, 'Q' },
 		{"repository", required_argument, NULL, 'X' },
 		{"quiet", no_argument, NULL, 'q' },
+		{"progress", no_argument, NULL, 0x100 },
 		{0, 0, 0, 0},
 	};
 	struct apk_applet *applet = NULL;
@@ -103,6 +104,9 @@ int main(int argc, char **argv)
 		case 'q':
 			apk_quiet = 1;
 			break;
+		case 0x100:
+			apk_progress = 1;
+			break;
 		default:
 			return usage();
 		}

src/apk_archive.h

@@ -27,6 +27,7 @@ int apk_parse_tar_gz(struct apk_bstream *, apk_archive_entry_parser parser, void
 
 int apk_archive_entry_extract(const struct apk_file_info *ae,
 			      struct apk_istream *is,
-			      const char *to);
+			      const char *to,
+			      apk_progress_cb cb, void *cb_ctx);
 
 #endif

src/apk_database.h

@@ -109,6 +109,7 @@ int apk_db_recalculate_and_commit(struct apk_database *db);
 
 int apk_db_install_pkg(struct apk_database *db,
 		       struct apk_package *oldpkg,
-		       struct apk_package *newpkg);
+		       struct apk_package *newpkg,
+		       apk_progress_cb cb, void *cb_ctx);
 
 #endif

src/apk_defines.h

@@ -50,7 +50,7 @@ extern csum_t bad_checksum;
 #define csum_valid(buf)			memcmp(buf, bad_checksum, sizeof(csum_t))
 #endif
 
-extern int apk_cwd_fd, apk_quiet;
+extern int apk_cwd_fd, apk_quiet, apk_progress;
 
 #define apk_error(args...)	apk_log("ERROR: ", args);
 #define apk_warning(args...)	if (!apk_quiet) { apk_log("WARNING: ", args); }
@@ -58,6 +58,25 @@ extern int apk_cwd_fd, apk_quiet;
 
 void apk_log(const char *prefix, const char *format, ...);
 
+static inline size_t apk_calc_installed_size(size_t size)
+{
+	const size_t bsize = 4 * 1024;
+
+	return (size + bsize - 1) & ~(bsize - 1);
+}
+static inline size_t muldiv(size_t a, size_t b, size_t c)
+{
+	unsigned long long tmp;
+	tmp = a;
+	tmp *= b;
+	tmp /= c;
+	return (size_t) tmp;
+}
+
+typedef void (*apk_progress_cb)(void *cb_ctx, size_t);
+
+#define APK_PROGRESS_SCALE 0x100
+
 #define APK_ARRAY(array_type_name, elem_type_name)			\
 	struct array_type_name {					\
 		int num;						\

src/apk_io.h

@@ -51,7 +51,8 @@ struct apk_istream *apk_istream_from_file(const char *file);
 struct apk_istream *apk_istream_from_file_gz(const char *file);
 struct apk_istream *apk_istream_from_url(const char *url);
 size_t apk_istream_skip(struct apk_istream *istream, size_t size);
-size_t apk_istream_splice(void *stream, int fd, size_t size);
+size_t apk_istream_splice(void *stream, int fd, size_t size,
+			  apk_progress_cb cb, void *cb_ctx);
 
 struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream);
 struct apk_bstream *apk_bstream_from_fd(int fd);

src/apk_state.h

@@ -19,8 +19,15 @@
 #define APK_STATE_INSTALL			2
 #define APK_STATE_NO_INSTALL			3
 
+struct apk_change {
+	struct list_head change_list;
+	struct apk_package *oldpkg;
+	struct apk_package *newpkg;
+};
+
 struct apk_state {
 	int refs;
+	struct list_head change_list_head;
 	unsigned char bitarray[];
 };
 
@@ -39,9 +46,9 @@ int apk_state_commit(struct apk_state *state, struct apk_database *db);
 
 int apk_state_satisfy_deps(struct apk_state *state,
 			   struct apk_dependency_array *deps);
+int apk_state_purge_unneeded(struct apk_state *state,
+			     struct apk_database *db);
 
-void apk_state_pkg_set(struct apk_state *state,
-		       struct apk_package *pkg);
 int apk_state_pkg_install(struct apk_state *state,
 			  struct apk_package *pkg);
 int apk_state_pkg_is_installed(struct apk_state *state,

src/archive.c

@@ -191,7 +191,8 @@ int apk_parse_tar_gz(struct apk_bstream *bs, apk_archive_entry_parser parser,
 
 int apk_archive_entry_extract(const struct apk_file_info *ae,
 			      struct apk_istream *is,
-			      const char *fn)
+			      const char *fn, apk_progress_cb cb,
+			      void *cb_ctx)
 {
 	int r = -1, fd;
 
@@ -214,7 +215,8 @@ int apk_archive_entry_extract(const struct apk_file_info *ae,
 				r = -1;
 				break;
 			}
-			if (apk_istream_splice(is, fd, ae->size) == ae->size)
+			if (apk_istream_splice(is, fd, ae->size, cb, cb_ctx)
+			    == ae->size)
 				r = 0;
 			close(fd);
 		} else {

src/database.c

@@ -30,6 +30,11 @@ struct install_ctx {
 	int script;
 	struct apk_db_dir_instance *diri;
 
+	apk_progress_cb cb;
+	void *cb_ctx;
+	size_t installed_size;
+	size_t current_file_size;
+
 	struct hlist_node **diri_node;
 	struct hlist_node **file_dir_node;
 	struct hlist_node **file_diri_node;
@@ -174,9 +179,11 @@ static struct apk_db_dir_instance *apk_db_diri_new(struct apk_database *db,
 	struct apk_db_dir_instance *diri;
 
 	diri = calloc(1, sizeof(struct apk_db_dir_instance));
-	hlist_add_after(&diri->pkg_dirs_list, after);
-	diri->dir = apk_db_dir_get_db(db, name);
-	diri->pkg = pkg;
+	if (diri != NULL) {
+		hlist_add_after(&diri->pkg_dirs_list, after);
+		diri->dir = apk_db_dir_get_db(db, name);
+		diri->pkg = pkg;
+	}
 
 	return diri;
 }
@@ -438,6 +445,9 @@ static int apk_db_write_fdb(struct apk_database *db, struct apk_ostream *os)
 					return -1;
 				n = 0;
 			}
+			if (n != 0 && os->write(os, buf, n) != n)
+				return -1;
+			n = 0;
 		}
 		os->write(os, "\n", 1);
 	}
@@ -745,6 +755,12 @@ int apk_db_recalculate_and_commit(struct apk_database *db)
 	state = apk_state_new(db);
 	r = apk_state_satisfy_deps(state, db->world);
 	if (r == 0) {
+		r = apk_state_purge_unneeded(state, db);
+		if (r != 0) {
+			apk_error("Failed to clean up state");
+			return r;
+		}
+
 		r = apk_state_commit(state, db);
 		if (r != 0) {
 			apk_error("Failed to commit changes");
@@ -764,6 +780,21 @@ int apk_db_recalculate_and_commit(struct apk_database *db)
 	return r;
 }
 
+static void extract_cb(void *_ctx, size_t progress)
+{
+	struct install_ctx *ctx = (struct install_ctx *) _ctx;
+
+	if (ctx->cb) {
+		size_t size = ctx->installed_size;
+
+		size += muldiv(progress, ctx->current_file_size, APK_PROGRESS_SCALE);
+		if (size > ctx->pkg->installed_size)
+			size = ctx->pkg->installed_size;
+
+		ctx->cb(ctx->cb_ctx, muldiv(APK_PROGRESS_SCALE, size, ctx->pkg->installed_size));
+	}
+}
+
 static int apk_db_install_archive_entry(void *_ctx,
 					const struct apk_file_info *ae,
 					struct apk_istream *is)
@@ -815,7 +846,16 @@ static int apk_db_install_archive_entry(void *_ctx,
 		return r;
 	}
 
+	/* Show progress */
+	if (ctx->cb) {
+		size_t size = ctx->installed_size;
+		if (size > pkg->installed_size)
+			size = pkg->installed_size;
+		ctx->cb(ctx->cb_ctx, muldiv(APK_PROGRESS_SCALE, size, pkg->installed_size));
+	}
+
 	/* Installable entry */
+	ctx->current_file_size = apk_calc_installed_size(ae->size);
 	if (!S_ISDIR(ae->mode)) {
 		if (!apk_blob_rsplit(name, '/', &bdir, &bfile))
 			return 0;
@@ -877,9 +917,11 @@ static int apk_db_install_archive_entry(void *_ctx,
 			snprintf(alt_name, sizeof(alt_name),
 				 "%s/%s.apk-new",
 				 diri->dir->dirname, file->filename);
-			r = apk_archive_entry_extract(ae, is, alt_name);
+			r = apk_archive_entry_extract(ae, is, alt_name,
+						      extract_cb, ctx);
 		} else {
-			r = apk_archive_entry_extract(ae, is, NULL);
+			r = apk_archive_entry_extract(ae, is, NULL,
+						      extract_cb, ctx);
 		}
 		memcpy(file->csum, ae->csum, sizeof(csum_t));
 	} else {
@@ -897,6 +939,7 @@ static int apk_db_install_archive_entry(void *_ctx,
 		apk_db_diri_set(diri, ae->mode & 0777, ae->uid, ae->gid);
 		apk_db_diri_mkdir(diri);
 	}
+	ctx->installed_size += ctx->current_file_size;
 
 	return r;
 }
@@ -929,7 +972,8 @@ static void apk_db_purge_pkg(struct apk_database *db,
 
 int apk_db_install_pkg(struct apk_database *db,
 		       struct apk_package *oldpkg,
-		       struct apk_package *newpkg)
+		       struct apk_package *newpkg,
+		       apk_progress_cb cb, void *cb_ctx)
 {
 	struct apk_bstream *bs;
 	struct install_ctx ctx;
@@ -975,6 +1019,8 @@ int apk_db_install_pkg(struct apk_database *db,
 		.pkg = newpkg,
 		.script = (oldpkg == NULL) ?
 			APK_SCRIPT_PRE_INSTALL : APK_SCRIPT_PRE_UPGRADE,
+		.cb = cb,
+		.cb_ctx = cb_ctx,
 	};
 	if (apk_parse_tar_gz(bs, apk_db_install_archive_entry, &ctx) != 0)
 		goto err_close;
@@ -993,8 +1039,6 @@ int apk_db_install_pkg(struct apk_database *db,
 	if (r != 0) {
 		apk_error("%s-%s: Failed to execute post-install/upgrade script",
 			  newpkg->name->name, newpkg->version);
-	} else if (apk_quiet) {
-		write(STDOUT_FILENO, ".", 1);
 	}
 	return r;
 err_close:

src/io.c

@@ -108,7 +108,8 @@ size_t apk_istream_skip(struct apk_istream *is, size_t size)
 	return done;
 }
 
-size_t apk_istream_splice(void *stream, int fd, size_t size)
+size_t apk_istream_splice(void *stream, int fd, size_t size,
+			  apk_progress_cb cb, void *cb_ctx)
 {
 	struct apk_istream *is = (struct apk_istream *) stream;
 	unsigned char *buf;
@@ -123,6 +124,9 @@ size_t apk_istream_splice(void *stream, int fd, size_t size)
 		return -1;
 
 	while (done < size) {
+		if (done != 0 && cb != NULL)
+			cb(cb_ctx, muldiv(APK_PROGRESS_SCALE, done, size));
+
 		togo = size - done;
 		if (togo > bufsz)
 			togo = bufsz;

src/package.c

@@ -275,7 +275,6 @@ static int read_info_entry(void *ctx, const struct apk_file_info *ae,
 	struct read_info_ctx *ri = (struct read_info_ctx *) ctx;
 	struct apk_database *db = ri->db;
 	struct apk_package *pkg = ri->pkg;
-	const int bsize = 4 * 1024;
 	apk_blob_t name, version;
 	char *slash;
 	int i;
@@ -321,7 +320,7 @@ static int read_info_entry(void *ctx, const struct apk_file_info *ae,
 		    apk_script_type(slash+1) == APK_SCRIPT_PRE_INSTALL)
 			ri->has_install = 1;
 	} else {
-		pkg->installed_size += (ae->size + bsize - 1) & ~(bsize - 1);
+		pkg->installed_size += apk_calc_installed_size(ae->size);
 	}
 
 	return 0;

src/state.c

@@ -15,10 +15,6 @@
 #include "apk_state.h"
 #include "apk_database.h"
 
-static int apk_state_commit_deps(struct apk_state *state,
-				 struct apk_database *db,
-				 struct apk_dependency_array *deps);
-
 struct apk_state *apk_state_new(struct apk_database *db)
 {
 	struct apk_state *state;
@@ -27,6 +23,7 @@ struct apk_state *apk_state_new(struct apk_database *db)
 	num_bytes = sizeof(struct apk_state) + (db->pkg_id * 2 + 7) / 8;
 	state = (struct apk_state*) calloc(1, num_bytes);
 	state->refs = 1;
+	list_init(&state->change_list_head);
 
 	return state;
 }
@@ -45,6 +42,24 @@ void apk_state_unref(struct apk_state *state)
 	free(state);
 }
 
+static int apk_state_add_change(struct apk_state *state,
+				struct apk_package *oldpkg,
+				struct apk_package *newpkg)
+{
+	struct apk_change *change;
+
+	change = (struct apk_change *) malloc(sizeof(struct apk_change));
+	if (change == NULL)
+		return -1;
+
+	list_init(&change->change_list);
+	list_add_tail(&change->change_list, &state->change_list_head);
+	change->oldpkg = oldpkg;
+	change->newpkg = newpkg;
+
+	return 0;
+}
+
 static void apk_state_set(struct apk_state *state, int pos, int val)
 {
 	int byte = pos / 4, offs = pos % 4;
@@ -63,20 +78,18 @@ static int apk_state_get(struct apk_state *state, int pos)
 	return (state->bitarray[byte] >> (offs * 2)) & 0x3;
 }
 
-static int apk_state_commit_pkg(struct apk_state *state,
-				struct apk_database *db,
-				struct apk_name *name,
-				struct apk_package *oldpkg,
-				struct apk_package *newpkg)
+static void apk_print_change(struct apk_database *db,
+			     struct apk_package *oldpkg,
+			     struct apk_package *newpkg)
 {
 	const char *msg = NULL;
-	int r, upgrade = 0;
+	int r;
+	struct apk_name *name;
 
-	if (newpkg != NULL) {
-		r = apk_state_commit_deps(state, db, newpkg->depends);
-		if (r != 0)
-			return r;
-	}
+	if (oldpkg != NULL)
+		name = oldpkg->name;
+	else
+		name = newpkg->name;
 
 	if (oldpkg == NULL) {
 		apk_message("Installing %s (%s)",
@@ -90,79 +103,100 @@ static int apk_state_commit_pkg(struct apk_state *state,
 		switch (r) {
 		case APK_VERSION_LESS:
 			msg = "Downgrading";
-			upgrade = 1;
 			break;
 		case APK_VERSION_EQUAL:
 			msg = "Re-installing";
 			break;
 		case APK_VERSION_GREATER:
 			msg = "Upgrading";
-			upgrade = 1;
 			break;
 		}
 		apk_message("%s %s (%s -> %s)",
-			    msg, name->name, oldpkg->version, newpkg->version);
+			    msg, name->name, oldpkg->version,
+			    newpkg->version);
 	}
-
-	return apk_db_install_pkg(db, oldpkg, newpkg);
 }
 
-static int apk_state_commit_name(struct apk_state *state,
-				 struct apk_database *db,
-				 struct apk_name *name)
+struct apk_stats {
+	unsigned int bytes;
+	unsigned int packages;
+};
+
+static void apk_count_change(struct apk_change *change, struct apk_stats *stats)
 {
-	struct apk_package *oldpkg = NULL, *newpkg = NULL;
+	if (change->newpkg != NULL) {
+		stats->bytes += change->newpkg->installed_size;
+		stats->packages ++;
+	}
+	if (change->oldpkg != NULL)
+		stats->packages ++;
+}
+
+static inline void apk_draw_progress(int x, int last)
+{
+	char tmp[] =
+		"-[                    ]- "
+		"\b\b\b\b\b\b\b\b\b\b\b\b\b"
+		"\b\b\b\b\b\b\b\b\b\b\b\b";
 	int i;
 
-	for (i = 0; i < name->pkgs->num; i++) {
-		if (apk_pkg_get_state(name->pkgs->item[i]) == APK_STATE_INSTALL)
-			oldpkg = name->pkgs->item[i];
-		if (apk_state_get(state, name->pkgs->item[i]->id) == APK_STATE_INSTALL)
-			newpkg = name->pkgs->item[i];
-	}
+	for (i = 0; i < x; i++)
+		tmp[2+i] = '#';
 
-	if (oldpkg == NULL && newpkg == NULL)
-		return 0;
-
-	/* No reinstallations for now */
-	if (newpkg == oldpkg)
-		return 0;
-
-	return apk_state_commit_pkg(state, db, name, oldpkg, newpkg);
+	fwrite(tmp, last ? 25 : sizeof(tmp)-1, 1, stderr);
+	fflush(stderr);
 }
 
-static int apk_state_commit_deps(struct apk_state *state,
-				 struct apk_database *db,
-				 struct apk_dependency_array *deps)
+struct progress {
+	struct apk_stats done;
+	struct apk_stats total;
+	struct apk_package *pkg;
+	size_t count;
+};
+
+static void progress_cb(void *ctx, size_t progress)
 {
-	int r, i;
+	struct progress *prog = (struct progress *) ctx;
+	size_t partial = 0, count;
 
-	if (deps == NULL)
-		return 0;
+	if (prog->pkg != NULL)
+		partial = muldiv(progress, prog->pkg->installed_size, APK_PROGRESS_SCALE);
 
-	for (i = 0; i < deps->num; i++) {
-		r = apk_state_commit_name(state, db, deps->item[i].name);
-		if (r != 0)
-			return r;
-	}
+        count = muldiv(20, prog->done.bytes + prog->done.packages + partial,
+		       prog->total.bytes + prog->total.packages);
 
-	return 0;
+	if (prog->count != count)
+		apk_draw_progress(count, 0);
+	prog->count = count;
 }
 
 int apk_state_commit(struct apk_state *state,
 		     struct apk_database *db)
 {
-	struct apk_package *pkg, *n;
+	struct progress prog;
+	struct apk_change *change;
 	int r;
 
-	/* Check all dependencies */
-	r = apk_state_commit_deps(state, db, db->world);
-	if (r != 0)
-		return r;
+	/* Count what needs to be done */
+	memset(&prog, 0, sizeof(prog));
+	list_for_each_entry(change, &state->change_list_head, change_list)
+		apk_count_change(change, &prog.total);
 
-	/* And purge all installed packages that were not considered */
-	list_for_each_entry_safe(pkg, n, &db->installed.packages, installed_pkgs_list)
-		apk_state_commit_name(state, db, pkg->name);
+	/* Go through changes */
+	list_for_each_entry(change, &state->change_list_head, change_list) {
+		apk_print_change(db, change->oldpkg, change->newpkg);
+		prog.pkg = change->newpkg;
+
+		r = apk_db_install_pkg(db, change->oldpkg, change->newpkg,
+				       apk_progress ? progress_cb : NULL,
+				       &prog);
+		if (r != 0)
+			return r;
+
+		apk_count_change(change, &prog.done);
+	}
+	if (apk_progress)
+		apk_draw_progress(20, 1);
 
 	return 0;
 }
@@ -170,9 +204,8 @@ int apk_state_commit(struct apk_state *state,
 int apk_state_satisfy_name(struct apk_state *state,
 			   struct apk_name *name)
 {
-	struct apk_package *preferred = NULL;
-	int i;
-	int upgrading = 1;
+	struct apk_package *preferred = NULL, *installed = NULL;
+	int i, r, upgrading = 1;
 
 	/* Is something already installed? Or figure out the preferred
 	 * package. */
@@ -181,35 +214,52 @@ int apk_state_satisfy_name(struct apk_state *state,
 		    APK_STATE_INSTALL)
 			return 0;
 
+		if (apk_pkg_get_state(name->pkgs->item[i]) == APK_STATE_INSTALL) {
+			installed = name->pkgs->item[i];
+			if (!upgrading) {
+				preferred = installed;
+				continue;
+			}
+		}
+
 		if (preferred == NULL) {
 			preferred = name->pkgs->item[i];
 			continue;
 		}
 
-		if (upgrading) {
-			if (apk_version_compare(APK_BLOB_STR(name->pkgs->item[i]->version),
-						APK_BLOB_STR(preferred->version)) ==
-			    APK_VERSION_GREATER) {
-				preferred = name->pkgs->item[i];
-				continue;
-			}
-		} else {
-			if (apk_pkg_get_state(name->pkgs->item[i]) ==
-			    APK_STATE_INSTALL) {
-				preferred = name->pkgs->item[i];
-				continue;
-			}
+		if (apk_version_compare(APK_BLOB_STR(name->pkgs->item[i]->version),
+					APK_BLOB_STR(preferred->version)) ==
+		    APK_VERSION_GREATER) {
+			preferred = name->pkgs->item[i];
+			continue;
 		}
 	}
 
-	/* Mark conflicting names as no install */
+	/* FIXME: current code considers only the prefferred package. */
+
+	/* Can we install? */
+	switch (apk_state_get(state, preferred->id)) {
+	case APK_STATE_INSTALL:
+		return 0;
+	case APK_STATE_NO_INSTALL:
+		return -1;
+	}
+
+	/* Update state bits and track changes */
 	for (i = 0; i < name->pkgs->num; i++) {
 		if (name->pkgs->item[i] != preferred)
 			apk_state_set(state, name->pkgs->item[i]->id,
 				      APK_STATE_NO_INSTALL);
 	}
+	apk_state_set(state, preferred->id, APK_STATE_INSTALL);
 
-	return apk_state_pkg_install(state, preferred);
+	if (preferred != installed) {
+		r = apk_state_add_change(state, installed, preferred);
+		if (r != 0)
+			return r;
+	}
+
+	return apk_state_satisfy_deps(state, preferred->depends);
 }
 
 int apk_state_satisfy_deps(struct apk_state *state,
@@ -234,33 +284,26 @@ int apk_state_satisfy_deps(struct apk_state *state,
 	return 0;
 }
 
-void apk_state_pkg_set(struct apk_state *state,
-		       struct apk_package *pkg)
+int apk_state_purge_unneeded(struct apk_state *state,
+			     struct apk_database *db)
 {
-	apk_state_set(state, pkg->id, APK_STATE_INSTALL);
-}
+	struct apk_package *pkg;
+	int r;
 
-int apk_state_pkg_install(struct apk_state *state,
-			  struct apk_package *pkg)
-{
-	switch (apk_state_get(state, pkg->id)) {
-	case APK_STATE_INSTALL:
-		return 0;
-	case APK_STATE_NO_INSTALL:
-		return -1;
+	/* Purge unconsidered packages */
+	list_for_each_entry(pkg, &db->installed.packages, installed_pkgs_list) {
+		switch (apk_state_get(state, pkg->id)) {
+		case APK_STATE_INSTALL:
+		case APK_STATE_NO_INSTALL:
+			break;
+		default:
+			r = apk_state_add_change(state, pkg, NULL);
+			if (r != 0)
+				return r;
+			break;
+		}
 	}
 
-	apk_state_set(state, pkg->id, APK_STATE_INSTALL);
-
-	if (pkg->depends == NULL)
-		return 0;
-
-	return apk_state_satisfy_deps(state, pkg->depends);
-}
-
-int apk_state_pkg_is_installed(struct apk_state *state,
-			       struct apk_package *pkg)
-{
-	return apk_state_get(state, pkg->id);
+	return 0;
 }