From 3b00c0dc808f4b6c3809c91a67709ca2d7bf8865 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timo=20Ter=C3=A4s?= <timo.teras@iki.fi>
Date: Fri, 16 Jul 2021 19:01:41 +0300
Subject: [PATCH] adb: unify various interfaces to adb_m_process

Removes code duplication, and puts important checks in one place.
Support seamless decompression in adbdump.
---
 src/Makefile                                |   2 +-
 src/adb.c                                   | 173 ++++++++++----------
 src/adb.h                                   |  32 ++--
 src/adb_walk_adb.c                          | 111 +++++++------
 src/{adb_walk_istream.c => adb_walk_text.c} |   2 +-
 src/apk_io.h                                |   7 +-
 src/app_adbdump.c                           |  30 ++--
 src/app_adbsign.c                           |  51 +++---
 src/app_extract.c                           |   7 +-
 src/app_mkndx.c                             |   4 +-
 src/io.c                                    |  53 +++++-
 src/meson.build                             |   2 +-
 12 files changed, 253 insertions(+), 221 deletions(-)
 rename src/{adb_walk_istream.c => adb_walk_text.c} (98%)

diff --git a/src/Makefile b/src/Makefile
index 79ff5d6..970354b 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -19,7 +19,7 @@ ZLIB_LIBS		:= $(shell $(PKG_CONFIG) --libs zlib)
 libapk_soname		:= 2.99.0
 libapk_so		:= $(obj)/libapk.so.$(libapk_soname)
 libapk.so.$(libapk_soname)-objs := \
-	adb.o adb_comp.o adb_walk_adb.o adb_walk_genadb.o adb_walk_gentext.o adb_walk_istream.o apk_adb.o \
+	adb.o adb_comp.o adb_walk_adb.o adb_walk_genadb.o adb_walk_gentext.o adb_walk_text.o apk_adb.o \
 	atom.o blob.o commit.o common.o context.o crypto_openssl.o database.o hash.o \
 	io.o io_archive.o io_gunzip.o io_url.o \
 	package.o pathbuilder.o print.o solver.o trust.o version.o
diff --git a/src/adb.c b/src/adb.c
index 77ceb63..3a7c684 100644
--- a/src/adb.c
+++ b/src/adb.c
@@ -26,22 +26,27 @@ static inline struct adb_block *adb_block_validate(struct adb_block *blk, apk_bl
 	return blk;
 }
 
-struct adb_block *adb_block_first(apk_blob_t b)
+static struct adb_block *adb_block_first(apk_blob_t b)
 {
 	return adb_block_validate((struct adb_block*)b.ptr, b);
 }
 
-struct adb_block *adb_block_next(struct adb_block *cur, apk_blob_t b)
+static struct adb_block *adb_block_next(struct adb_block *cur, apk_blob_t b)
 {
 	return adb_block_validate((struct adb_block*)((char*)cur + adb_block_size(cur)), b);
 }
 
+#define adb_foreach_block(__blk, __adb) \
+	for (__blk = adb_block_first(__adb); !IS_ERR_OR_NULL(__blk); __blk = adb_block_next(__blk, __adb))
+
 /* Init stuff */
 int adb_free(struct adb *db)
 {
-	if (db->mmap.ptr) {
-		munmap(db->mmap.ptr, db->mmap.len);
+	if (db->is) {
+		// read-only adb
+		apk_istream_close(db->is);
 	} else {
+		// writable adb
 		struct adb_w_bucket *bucket, *nxt;
 		int i;
 		for (i = 0; i < db->num_buckets; i++)
@@ -65,14 +70,16 @@ void adb_reset(struct adb *db)
 	db->adb.len = 0;
 }
 
-static int __adb_m_parse(struct adb *db, struct apk_trust *t)
+static int __adb_m_parse(struct adb *db, apk_blob_t data, struct apk_trust *t,
+	int (*cb)(struct adb *, struct adb_block *, struct apk_istream *))
 {
 	struct adb_verify_ctx vfy = {};
 	struct adb_block *blk;
+	struct apk_istream is;
 	int r = -APKE_ADB_BLOCK;
 	int trusted = t ? 0 : 1;
 
-	adb_foreach_block(blk, db->data) {
+	adb_foreach_block(blk, data) {
 		apk_blob_t b = adb_block_blob(blk);
 		switch (adb_block_type(blk)) {
 		case ADB_BLOCK_ADB:
@@ -89,12 +96,15 @@ static int __adb_m_parse(struct adb *db, struct apk_trust *t)
 			if (APK_BLOB_IS_NULL(db->adb)) break;
 			break;
 		}
+		r = cb(db, blk, apk_istream_from_blob(&is, b));
+		if (r < 0) goto err;
 	}
 	if (IS_ERR(blk)) r = PTR_ERR(blk);
 	else if (!trusted) r = -APKE_SIGNATURE_UNTRUSTED;
 	else if (db->adb.ptr) r = 0;
 
 	if (r != 0) {
+	err:
 		db->adb = APK_BLOB_NULL;
 	}
 	return r;
@@ -102,31 +112,27 @@ static int __adb_m_parse(struct adb *db, struct apk_trust *t)
 
 int adb_m_blob(struct adb *db, apk_blob_t blob, struct apk_trust *t)
 {
-	*db = (struct adb) { .data = blob };
-	return __adb_m_parse(db, t);
+	adb_init(db);
+	return __adb_m_parse(db, blob, t, 0);
 }
 
-int adb_m_map(struct adb *db, int fd, uint32_t expected_schema, struct apk_trust *t)
+static int __adb_m_mmap(struct adb *db, apk_blob_t mmap, uint32_t expected_schema, struct apk_trust *t,
+	int (*cb)(struct adb *, struct adb_block *, struct apk_istream *))
 {
-	struct stat st;
 	struct adb_header *hdr;
 	int r = -APKE_ADB_HEADER;
+	apk_blob_t data = mmap;
 
-	if (fstat(fd, &st) != 0) return -errno;
-	if (st.st_size < sizeof *hdr) return -EIO;
+	if (!(expected_schema & ADB_SCHEMA_IMPLIED)) {
+		if (mmap.len < sizeof *hdr) return -APKE_ADB_HEADER;
+		hdr = (struct adb_header *) mmap.ptr;
+		if (hdr->magic != htole32(ADB_FORMAT_MAGIC)) return -APKE_ADB_HEADER;
+		if (expected_schema && expected_schema != le32toh(hdr->schema)) return -APKE_ADB_SCHEMA;
+		db->hdr = *hdr;
+		data = APK_BLOB_PTR_LEN(mmap.ptr + sizeof *hdr, mmap.len - sizeof *hdr);
+	}
 
-	memset(db, 0, sizeof *db);
-	db->mmap.ptr = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
-	db->mmap.len = st.st_size;
-	if (db->mmap.ptr == MAP_FAILED) return -errno;
-
-	hdr = (struct adb_header *) db->mmap.ptr;
-	if (hdr->magic != htole32(ADB_FORMAT_MAGIC)) goto err;
-	if (expected_schema && expected_schema != le32toh(hdr->schema)) goto err;
-
-	db->hdr = *hdr;
-	db->data = APK_BLOB_PTR_LEN(db->mmap.ptr + sizeof *hdr, db->mmap.len - sizeof *hdr);
-	r = __adb_m_parse(db, t);
+	r = __adb_m_parse(db, data, t, cb);
 	if (r) goto err;
 	return 0;
 err:
@@ -134,8 +140,8 @@ err:
 	return r;
 }
 
-int adb_m_stream(struct adb *db, struct apk_istream *is, uint32_t expected_schema,
-	struct apk_trust *t, int (*datacb)(struct adb *, size_t, struct apk_istream *))
+static int __adb_m_stream(struct adb *db, struct apk_istream *is, uint32_t expected_schema,
+	struct apk_trust *t, int (*cb)(struct adb *, struct adb_block *, struct apk_istream *))
 {
 	struct adb_verify_ctx vfy = {};
 	struct adb_block blk;
@@ -146,8 +152,18 @@ int adb_m_stream(struct adb *db, struct apk_istream *is, uint32_t expected_schem
 	size_t sz;
 
 	if (IS_ERR(is)) return PTR_ERR(is);
-	if ((r = apk_istream_read(is, &db->hdr, sizeof db->hdr)) != sizeof db->hdr) goto err;
-	if (db->hdr.magic != htole32(ADB_FORMAT_MAGIC)) goto bad_msg;
+
+	if (!(expected_schema & ADB_SCHEMA_IMPLIED)) {
+		if ((r = apk_istream_read(is, &db->hdr, sizeof db->hdr)) != sizeof db->hdr) goto err;
+		if (db->hdr.magic != htole32(ADB_FORMAT_MAGIC)) {
+			r = -APKE_ADB_HEADER;
+			goto err;
+		}
+		if (expected_schema && expected_schema != le32toh(db->hdr.schema)) {
+			r = -APKE_ADB_SCHEMA;
+			goto err;
+		}
+	}
 
 	do {
 		r = apk_istream_read(is, &blk, sizeof blk);
@@ -168,10 +184,12 @@ int adb_m_stream(struct adb *db, struct apk_istream *is, uint32_t expected_schem
 			db->adb.ptr = malloc(sz);
 			db->adb.len = adb_block_length(&blk);
 			if ((r = apk_istream_read(is, db->adb.ptr, sz)) != sz) goto err;
-			break;
+			r = cb(db, &blk, apk_istream_from_blob(&seg.is, db->adb));
+			if (r < 0) goto err;
+			continue;
 		case ADB_BLOCK_SIG:
 			if (APK_BLOB_IS_NULL(db->adb)) goto bad_msg;
-			sig = apk_istream_get(is, sz);
+			sig = apk_istream_peek(is, sz);
 			if (IS_ERR(sig)) {
 				r = PTR_ERR(sig);
 				goto err;
@@ -186,17 +204,16 @@ int adb_m_stream(struct adb *db, struct apk_istream *is, uint32_t expected_schem
 				r = -APKE_SIGNATURE_UNTRUSTED;
 				goto err;
 			}
-			r = datacb(db, adb_block_length(&blk),
-				apk_istream_segment(&seg, is, adb_block_size(&blk) - sizeof blk, 0));
-			if (r < 0) goto err;
-			if (seg.bytes_left) {
-				r = apk_istream_read(is, NULL, seg.bytes_left);
-				if (r < 0) goto err;
-			}
 			break;
 		default:
 			goto bad_msg;
 		}
+
+		apk_istream_segment(&seg, is, adb_block_size(&blk) - sizeof blk, 0);
+		r = cb(db, &blk, &seg.is);
+		if (r < 0) goto err;
+		r = apk_istream_close(&seg.is);
+		if (r < 0) goto err;
 	} while (1);
 bad_msg:
 	r = -APKE_ADB_BLOCK;
@@ -207,6 +224,30 @@ done:
 	return r;
 }
 
+static int __adb_dummy_cb(struct adb *db, struct adb_block *b, struct apk_istream *is)
+{
+	return 0;
+}
+
+int adb_m_process(struct adb *db, struct apk_istream *is, uint32_t expected_schema,
+	struct apk_trust *t, int (*cb)(struct adb *, struct adb_block *, struct apk_istream *))
+{
+	apk_blob_t mmap = apk_istream_mmap(is);
+	memset(db, 0, sizeof *db);
+	if (expected_schema & ADB_SCHEMA_IMPLIED) {
+		db->hdr = (struct adb_header) {
+			.magic = ADB_FORMAT_MAGIC,
+			.schema = expected_schema & ~ADB_SCHEMA_IMPLIED,
+		};
+	}
+	if (!cb) cb = __adb_dummy_cb;
+	if (!APK_BLOB_IS_NULL(mmap)) {
+		db->is = is;
+		return __adb_m_mmap(db, mmap, expected_schema, t, cb);
+	}
+	return __adb_m_stream(db, is, expected_schema, t, cb);
+}
+
 int adb_w_init_dynamic(struct adb *db, uint32_t schema, void *buckets, size_t num_buckets)
 {
 	size_t i;
@@ -231,7 +272,7 @@ int adb_w_init_static(struct adb *db, void *buf, size_t bufsz)
 	*db = (struct adb) {
 		.hdr.magic = htole32(ADB_FORMAT_MAGIC),
 		.adb.ptr = buf,
-		.mmap.len = bufsz,
+		.alloc_len = bufsz,
 	};
 	return 0;
 }
@@ -488,12 +529,12 @@ static size_t adb_w_raw(struct adb *db, struct iovec *vec, size_t n, size_t len,
 		db->adb.len += i;
 	}
 
-	if (db->adb.len + len > db->mmap.len) {
+	if (db->adb.len + len > db->alloc_len) {
 		assert(db->num_buckets);
-		if (!db->mmap.len) db->mmap.len = 8192;
-		while (db->adb.len + len > db->mmap.len)
-			db->mmap.len *= 2;
-		ptr = realloc(db->adb.ptr, db->mmap.len);
+		if (!db->alloc_len) db->alloc_len = 8192;
+		while (db->adb.len + len > db->alloc_len)
+			db->alloc_len *= 2;
+		ptr = realloc(db->adb.ptr, db->alloc_len);
 		assert(ptr);
 		db->adb.ptr = ptr;
 	}
@@ -1093,49 +1134,3 @@ int adb_trust_verify_signature(struct apk_trust *trust, struct adb *db, struct a
 
 	return -APKE_SIGNATURE_UNTRUSTED;
 }
-
-/* Container transformation interface */
-int adb_c_xfrm(struct adb_xfrm *x, int (*cb)(struct adb_xfrm *, struct adb_block *, struct apk_istream *))
-{
-	struct adb_block blk;
-	struct apk_segment_istream seg;
-	int r, block_no = 0;
-	size_t sz;
-
-	r = apk_istream_read(x->is, &x->db.hdr, sizeof x->db.hdr);
-	if (r != sizeof x->db.hdr) goto err;
-
-	if (x->db.hdr.magic != htole32(ADB_FORMAT_MAGIC)) goto bad_msg;
-	r = apk_ostream_write(x->os, &x->db.hdr, sizeof x->db.hdr);
-	if (r < 0) goto err;
-
-	do {
-		r = apk_istream_read(x->is, &blk, sizeof blk);
-		if (r != sizeof blk) {
-			if (r != 0) goto err;
-			return cb(x, NULL, NULL);
-		}
-
-		if ((block_no++ == 0) != (adb_block_type(&blk) == ADB_BLOCK_ADB)) goto bad_msg;
-
-		sz = adb_block_size(&blk) - sizeof blk;
-		r = cb(x, &blk, apk_istream_segment(&seg, x->is, sz, 0));
-		if (r < 0) goto err;
-
-		if (r == 0 && seg.bytes_left == sz) {
-			r = apk_ostream_write(x->os, &blk, sizeof blk);
-			if (r < 0) goto err;
-			r = apk_stream_copy(x->is, x->os, sz, 0, 0, 0);
-			if (r < 0) goto err;
-		} else if (seg.bytes_left > 0) {
-			r = apk_istream_read(x->is, NULL, seg.bytes_left);
-			if (r != seg.bytes_left) goto err;
-		}
-	} while (1);
-bad_msg:
-	r = -APKE_ADB_BLOCK;
-err:
-	if (r >= 0) r = -APKE_ADB_BLOCK;
-	apk_ostream_cancel(x->os, r);
-	return r;
-}
diff --git a/src/adb.h b/src/adb.h
index 9cb73b4..fd5462b 100644
--- a/src/adb.h
+++ b/src/adb.h
@@ -44,6 +44,7 @@ typedef uint32_t adb_val_t;
 
 /* File Header */
 #define ADB_FORMAT_MAGIC	0x2e424441	// ADB.
+#define ADB_SCHEMA_IMPLIED	0x80000000
 
 struct adb_header {
 	uint32_t magic;
@@ -52,7 +53,6 @@ struct adb_header {
 
 /* Blocks */
 #define ADB_BLOCK_ALIGNMENT	8
-#define ADB_BLOCK_END		-1
 #define ADB_BLOCK_ADB		0
 #define ADB_BLOCK_SIG		2
 #define ADB_BLOCK_DATA		3
@@ -86,12 +86,6 @@ struct adb_sign_v0 {
 	uint8_t sig[0];
 };
 
-/* Block enumeration */
-struct adb_block *adb_block_first(apk_blob_t b);
-struct adb_block *adb_block_next(struct adb_block *cur, apk_blob_t b);
-#define adb_foreach_block(__blk, __adb) \
-	for (__blk = adb_block_first(__adb); !IS_ERR_OR_NULL(__blk); __blk = adb_block_next(__blk, __adb))
-
 /* Schema */
 #define ADB_KIND_ADB	1
 #define ADB_KIND_OBJECT	2
@@ -144,9 +138,11 @@ struct adb_w_bucket {
 };
 
 struct adb {
-	apk_blob_t mmap, data, adb;
+	struct apk_istream *is;
+	apk_blob_t adb;
 	struct adb_header hdr;
 	size_t num_buckets;
+	size_t alloc_len;
 	struct list_head *bucket;
 };
 
@@ -158,12 +154,15 @@ struct adb_obj {
 };
 
 /* Container read interface */
+static inline void adb_init(struct adb *db) { memset(db, 0, sizeof *db); }
 int adb_free(struct adb *);
 void adb_reset(struct adb *);
 
 int adb_m_blob(struct adb *, apk_blob_t, struct apk_trust *);
-int adb_m_map(struct adb *, int fd, uint32_t expected_schema, struct apk_trust *);
-int adb_m_stream(struct adb *db, struct apk_istream *is, uint32_t expected_schema, struct apk_trust *trust, int (*datacb)(struct adb *, size_t, struct apk_istream *));
+int adb_m_process(struct adb *db, struct apk_istream *is, uint32_t expected_schema, struct apk_trust *trust, int (*cb)(struct adb *, struct adb_block *, struct apk_istream *));
+static inline int adb_m_open(struct adb *db, struct apk_istream *is, uint32_t expected_schema, struct apk_trust *trust) {
+	return adb_m_process(db, is, expected_schema, trust, 0);
+}
 #define adb_w_init_alloca(db, schema, num_buckets) adb_w_init_dynamic(db, schema, alloca(sizeof(struct list_head[num_buckets])), num_buckets)
 #define adb_w_init_tmp(db, size) adb_w_init_static(db, alloca(size), size)
 int adb_w_init_dynamic(struct adb *db, uint32_t schema, void *buckets, size_t num_buckets);
@@ -240,15 +239,6 @@ struct adb_verify_ctx {
 int adb_trust_write_signatures(struct apk_trust *trust, struct adb *db, struct adb_verify_ctx *vfy, struct apk_ostream *os);
 int adb_trust_verify_signature(struct apk_trust *trust, struct adb *db, struct adb_verify_ctx *vfy, apk_blob_t sigb);
 
-/* Transform existing file */
-struct adb_xfrm {
-	struct apk_istream *is;
-	struct apk_ostream *os;
-	struct adb db;
-	struct adb_verify_ctx vfy;
-};
-int adb_c_xfrm(struct adb_xfrm *, int (*cb)(struct adb_xfrm *, struct adb_block *, struct apk_istream *));
-
 /* SAX style event based handling of ADB */
 
 struct adb_db_schema {
@@ -297,8 +287,8 @@ struct adb_walk_genadb {
 	adb_val_t vals[ADB_WALK_GENADB_MAX_VALUES];
 };
 
-int adb_walk_adb(struct adb_walk *d, struct adb *db, struct apk_trust *trust);
-int adb_walk_istream(struct adb_walk *d, struct apk_istream *is);
+int adb_walk_adb(struct adb_walk *d, struct apk_istream *is, struct apk_trust *trust);
+int adb_walk_text(struct adb_walk *d, struct apk_istream *is);
 
 // Seamless compression support
 
diff --git a/src/adb_walk_adb.c b/src/adb_walk_adb.c
index 2b3b34b..306c496 100644
--- a/src/adb_walk_adb.c
+++ b/src/adb_walk_adb.c
@@ -8,19 +8,21 @@
 
 struct adb_walk_ctx {
 	struct adb_walk *d;
-	struct adb *db;
 	struct apk_trust *trust;
+	struct adb db;
+	struct adb_verify_ctx vfy;
 };
 
+static int adb_walk_block(struct adb *db, struct adb_block *b, struct apk_istream *is);
 static int dump_object(struct adb_walk_ctx *ctx, const struct adb_object_schema *schema, adb_val_t v);
-static int dump_adb(struct adb_walk_ctx *ctx);
 
 static int dump_item(struct adb_walk_ctx *ctx, const char *name, const uint8_t *kind, adb_val_t v)
 {
 	struct adb_walk *d = ctx->d;
-	struct adb db, *origdb;
+	struct adb origdb;
 	struct adb_obj o;
 	struct adb_object_schema *obj_schema;
+	struct apk_istream is;
 	char tmp[256];
 	apk_blob_t b;
 
@@ -31,7 +33,7 @@ static int dump_item(struct adb_walk_ctx *ctx, const char *name, const uint8_t *
 	switch (*kind) {
 	case ADB_KIND_ARRAY:
 		obj_schema = container_of(kind, struct adb_object_schema, kind);
-		adb_r_obj(ctx->db, v, &o, obj_schema);
+		adb_r_obj(&ctx->db, v, &o, obj_schema);
 		//if (!adb_ra_num(&o)) return 0;
 
 		d->ops->start_array(d, adb_ra_num(&o));
@@ -41,12 +43,12 @@ static int dump_item(struct adb_walk_ctx *ctx, const char *name, const uint8_t *
 		d->ops->end(d);
 		break;
 	case ADB_KIND_ADB:
-		db.hdr.schema = container_of(kind, struct adb_adb_schema, kind)->schema_id;
-		db.data = adb_r_blob(ctx->db, v);
+		apk_istream_from_blob(&is, adb_r_blob(&ctx->db, v));
 		origdb = ctx->db;
-		ctx->db = &db;
 		d->ops->start_object(d);
-		dump_adb(ctx);
+		adb_m_process(&ctx->db, &is,
+			container_of(kind, struct adb_adb_schema, kind)->schema_id | ADB_SCHEMA_IMPLIED,
+			0, adb_walk_block);
 		d->ops->end(d);
 		ctx->db = origdb;
 		break;
@@ -64,7 +66,7 @@ static int dump_item(struct adb_walk_ctx *ctx, const char *name, const uint8_t *
 	case ADB_KIND_INT:;
 		struct adb_scalar_schema *scalar = container_of(kind, struct adb_scalar_schema, kind);
 		if (scalar->tostring) {
-			b = scalar->tostring(ctx->db, v, tmp, sizeof tmp);
+			b = scalar->tostring(&ctx->db, v, tmp, sizeof tmp);
 		} else {
 			b = APK_BLOB_STR("(unknown)");
 		}
@@ -83,7 +85,7 @@ static int dump_object(struct adb_walk_ctx *ctx, const struct adb_object_schema
 	apk_blob_t b;
 	struct adb_walk *d = ctx->d;
 
-	adb_r_obj(ctx->db, v, &o, schema);
+	adb_r_obj(&ctx->db, v, &o, schema);
 	if (schema) {
 		if (schema->tostring) {
 			b = schema->tostring(&o, tmp, sizeof tmp);
@@ -104,63 +106,60 @@ static int dump_object(struct adb_walk_ctx *ctx, const struct adb_object_schema
 	return 0;
 }
 
-static int dump_adb(struct adb_walk_ctx *ctx)
+static int adb_walk_block(struct adb *db, struct adb_block *b, struct apk_istream *is)
 {
-	char tmp[16+ADB_MAX_SIGNATURE_LEN*2];
-	struct adb_block *blk;
-	struct adb_sign_hdr *s;
-	struct adb_verify_ctx vfy = {};
-	uint32_t schema_magic = ctx->db->hdr.schema;
-	const struct adb_db_schema *ds;
+	struct adb_walk_ctx *ctx = container_of(db, struct adb_walk_ctx, db);
 	struct adb_walk *d = ctx->d;
+	char tmp[16+ADB_MAX_SIGNATURE_LEN*2];
+	struct adb_sign_hdr *s;
+	uint32_t schema_magic = ctx->db.hdr.schema;
+	const struct adb_db_schema *ds;
 	int r, len;
+	size_t sz = adb_block_length(b);
+	apk_blob_t data;
 
-	for (ds = d->schemas; ds->magic; ds++)
-		if (ds->magic == schema_magic) break;
-
-	adb_foreach_block(blk, ctx->db->data) {
-		apk_blob_t b = adb_block_blob(blk);
-		switch (adb_block_type(blk)) {
-		case ADB_BLOCK_ADB:
-			len = snprintf(tmp, sizeof tmp, "ADB block, size: %u", adb_block_length(blk));
-			d->ops->comment(d, APK_BLOB_PTR_LEN(tmp, len));
-			if (ds->root) {
-				ctx->db->adb = b;
-				dump_object(ctx, ds->root, adb_r_root(ctx->db));
-			}
-			break;
-		case ADB_BLOCK_SIG:
-			s = (struct adb_sign_hdr*) b.ptr;
-			r = adb_trust_verify_signature(ctx->trust, ctx->db, &vfy, b);
-			len = snprintf(tmp, sizeof tmp, "sig v%02x h%02x ", s->sign_ver, s->hash_alg);
-			for (size_t j = sizeof *s; j < b.len; j++)
-				len += snprintf(&tmp[len], sizeof tmp - len, "%02x", (uint8_t)b.ptr[j]);
-			len += snprintf(&tmp[len], sizeof tmp - len, ": %s", r ? apk_error_str(r) : "OK");
-			d->ops->comment(d, APK_BLOB_PTR_LEN(tmp, len));
-			break;
-		case ADB_BLOCK_DATA:
-			len = snprintf(tmp, sizeof tmp, "data block, size: %d", adb_block_length(blk));
-			d->ops->comment(d, APK_BLOB_PTR_LEN(tmp, len));
-			break;
-		default:
-			len = snprintf(tmp, sizeof tmp, "unknown block %d, size: %d",
-				adb_block_type(blk), adb_block_length(blk));
-			d->ops->comment(d, APK_BLOB_PTR_LEN(tmp, len));
-		}
-	}
-	if (IS_ERR(blk)) {
-		d->ops->comment(d, APK_BLOB_STRLIT("block enumeration error: corrupt data area"));
+	switch (adb_block_type(b)) {
+	case ADB_BLOCK_ADB:
+		d->ops->schema(d, db->hdr.schema);
+		for (ds = d->schemas; ds->magic; ds++)
+			if (ds->magic == schema_magic) break;
+		len = snprintf(tmp, sizeof tmp, "ADB block, size: %zu", sz);
+		d->ops->comment(d, APK_BLOB_PTR_LEN(tmp, len));
+		if (ds->root) dump_object(ctx, ds->root, adb_r_root(db));
+		break;
+	case ADB_BLOCK_SIG:
+		s = (struct adb_sign_hdr*) apk_istream_get(is, sz);
+		data = APK_BLOB_PTR_LEN((char*)s, sz);
+		r = adb_trust_verify_signature(ctx->trust, db, &ctx->vfy, data);
+		len = snprintf(tmp, sizeof tmp, "sig v%02x h%02x ", s->sign_ver, s->hash_alg);
+		for (size_t j = sizeof *s; j < data.len; j++)
+			len += snprintf(&tmp[len], sizeof tmp - len, "%02x", (uint8_t)data.ptr[j]);
+		len += snprintf(&tmp[len], sizeof tmp - len, ": %s", r ? apk_error_str(r) : "OK");
+		d->ops->comment(d, APK_BLOB_PTR_LEN(tmp, len));
+		break;
+	case ADB_BLOCK_DATA:
+		len = snprintf(tmp, sizeof tmp, "data block, size: %zu", sz);
+		d->ops->comment(d, APK_BLOB_PTR_LEN(tmp, len));
+		break;
+	default:
+		len = snprintf(tmp, sizeof tmp, "unknown block %d, size: %zu",
+			adb_block_type(b), sz);
+		d->ops->comment(d, APK_BLOB_PTR_LEN(tmp, len));
 	}
 	return 0;
 }
 
-int adb_walk_adb(struct adb_walk *d, struct adb *db, struct apk_trust *trust)
+int adb_walk_adb(struct adb_walk *d, struct apk_istream *is, struct apk_trust *trust)
 {
 	struct adb_walk_ctx ctx = {
 		.d = d,
-		.db = db,
 		.trust = trust,
 	};
-	d->ops->schema(d, db->hdr.schema);
-	return dump_adb(&ctx);
+	int r;
+
+	if (IS_ERR(is)) return PTR_ERR(is);
+
+	r = adb_m_process(&ctx.db, is, 0, 0, adb_walk_block);
+	adb_free(&ctx.db);
+	return r;
 }
diff --git a/src/adb_walk_istream.c b/src/adb_walk_text.c
similarity index 98%
rename from src/adb_walk_istream.c
rename to src/adb_walk_text.c
index b85565f..b713f10 100644
--- a/src/adb_walk_istream.c
+++ b/src/adb_walk_text.c
@@ -9,7 +9,7 @@
 #define dbg_printf(args...)
 #endif
 
-int adb_walk_istream(struct adb_walk *d, struct apk_istream *is)
+int adb_walk_text(struct adb_walk *d, struct apk_istream *is)
 {
 	const apk_blob_t token = APK_BLOB_STR("\n");
 	const apk_blob_t comment = APK_BLOB_STR(" #");
diff --git a/src/apk_io.h b/src/apk_io.h
index 61aee4f..b0971cd 100644
--- a/src/apk_io.h
+++ b/src/apk_io.h
@@ -87,11 +87,14 @@ typedef int (*apk_archive_entry_parser)(void *ctx,
 
 #define APK_ISTREAM_FORCE_REFRESH		((time_t) -1)
 
-struct apk_istream *apk_istream_from_file(int atfd, const char *file);
-struct apk_istream *apk_istream_from_file_gz(int atfd, const char *file);
+struct apk_istream *apk_istream_from_blob(struct apk_istream *, apk_blob_t);
+struct apk_istream *__apk_istream_from_file(int atfd, const char *file, int try_mmap);
+static inline struct apk_istream *apk_istream_from_file(int atfd, const char *file) { return __apk_istream_from_file(atfd, file, 0); }
+static inline struct apk_istream *apk_istream_from_file_mmap(int atfd, const char *file) { return __apk_istream_from_file(atfd, file, 1); }
 struct apk_istream *apk_istream_from_fd(int fd);
 struct apk_istream *apk_istream_from_fd_url_if_modified(int atfd, const char *url, time_t since);
 static inline int apk_istream_error(struct apk_istream *is, int err) { if (!is->err) is->err = err; return err; }
+apk_blob_t apk_istream_mmap(struct apk_istream *is);
 ssize_t apk_istream_read(struct apk_istream *is, void *ptr, size_t size);
 void *apk_istream_peek(struct apk_istream *is, size_t len);
 void *apk_istream_get(struct apk_istream *is, size_t len);
diff --git a/src/app_adbdump.c b/src/app_adbdump.c
index 39cc074..ce7b5d6 100644
--- a/src/app_adbdump.c
+++ b/src/app_adbdump.c
@@ -11,24 +11,6 @@ static const struct adb_db_schema dbschemas[] = {
 	{},
 };
 
-static int mmap_and_dump_adb(struct apk_trust *trust, int fd, struct apk_out *out)
-{
-	struct adb db;
-	struct adb_walk_gentext td = {
-		.d.ops = &adb_walk_gentext_ops,
-		.d.schemas = dbschemas,
-		.out = out->out,
-	};
-	int r;
-
-	r = adb_m_map(&db, fd, 0, NULL);
-	if (r) return r;
-
-	adb_walk_adb(&td.d, &db, trust);
-	adb_free(&db);
-	return 0;
-}
-
 static int adbdump_main(void *pctx, struct apk_ctx *ac, struct apk_string_array *args)
 {
 	struct apk_out *out = &ac->out;
@@ -36,7 +18,15 @@ static int adbdump_main(void *pctx, struct apk_ctx *ac, struct apk_string_array
 	int r;
 
 	foreach_array_item(arg, args) {
-		r = mmap_and_dump_adb(apk_ctx_get_trust(ac), open(*arg, O_RDONLY), out);
+		struct adb_walk_gentext td = {
+			.d.ops = &adb_walk_gentext_ops,
+			.d.schemas = dbschemas,
+			.out = out->out,
+		};
+
+		r = adb_walk_adb(&td.d,
+			adb_decompress(apk_istream_from_file_mmap(AT_FDCWD, *arg), 0),
+			apk_ctx_get_trust(ac));
 		if (r) {
 			apk_err(out, "%s: %s", *arg, apk_error_str(r));
 			return r;
@@ -68,7 +58,7 @@ static int adbgen_main(void *pctx, struct apk_ctx *ac, struct apk_string_array *
 	foreach_array_item(arg, args) {
 		adb_reset(&genadb.db);
 		adb_reset(&genadb.idb[0]);
-		r = adb_walk_istream(&genadb.d, apk_istream_from_file(AT_FDCWD, *arg));
+		r = adb_walk_text(&genadb.d, apk_istream_from_file(AT_FDCWD, *arg));
 		if (!r) {
 			adb_w_root(&genadb.db, genadb.stored_object);
 			r = adb_c_create(apk_ostream_to_fd(STDOUT_FILENO), &genadb.db,
diff --git a/src/app_adbsign.c b/src/app_adbsign.c
index 23bcc8b..82604cd 100644
--- a/src/app_adbsign.c
+++ b/src/app_adbsign.c
@@ -8,7 +8,12 @@
 
 struct sign_ctx {
 	struct apk_ctx *ac;
-	struct adb_xfrm xfrm;
+
+	struct adb db;
+	struct apk_istream *is;
+	struct apk_ostream *os;
+	struct adb_verify_ctx vfy;
+
 	int reset_signatures : 1;
 	int signatures_written : 1;
 };
@@ -37,27 +42,34 @@ static const struct apk_option_group optgroup_applet = {
 	.parse = option_parse_applet,
 };
 
-static int update_signatures(struct adb_xfrm *xfrm, struct adb_block *blk, struct apk_istream *is)
+static int process_signatures(struct sign_ctx *ctx)
 {
-	struct sign_ctx *ctx = container_of(xfrm, struct sign_ctx, xfrm);
-	struct apk_trust *trust = apk_ctx_get_trust(ctx->ac);
 	int r;
 
-	switch (blk ? adb_block_type(blk) : -1) {
+	if (ctx->signatures_written) return 0;
+	ctx->signatures_written = 1;
+	r = adb_trust_write_signatures(apk_ctx_get_trust(ctx->ac), &ctx->db, &ctx->vfy, ctx->os);
+	if (r < 0) apk_ostream_cancel(ctx->os, r);
+	return r;
+}
+
+static int process_block(struct adb *db, struct adb_block *blk, struct apk_istream *is)
+{
+	struct sign_ctx *ctx = container_of(db, struct sign_ctx, db);
+	int r;
+
+	switch (adb_block_type(blk)) {
 	case ADB_BLOCK_ADB:
-		return adb_c_block_copy(xfrm->os, blk, is, &xfrm->vfy);
+		adb_c_header(ctx->os, db);
+		return adb_c_block_copy(ctx->os, blk, is, &ctx->vfy);
 	case ADB_BLOCK_SIG:
 		if (ctx->reset_signatures)
 			break;
-		return adb_c_block_copy(xfrm->os, blk, is, NULL);
+		return adb_c_block_copy(ctx->os, blk, is, NULL);
 	default:
-		if (!ctx->signatures_written) {
-			ctx->signatures_written = 1;
-			r = adb_trust_write_signatures(trust, &xfrm->db, &xfrm->vfy, xfrm->os);
-			if (r) return r;
-		}
-		if (!blk) break;
-		return adb_c_block_copy(xfrm->os, blk, is, NULL);
+		r = process_signatures(ctx);
+		if (r < 0) return r;
+		return adb_c_block_copy(ctx->os, blk, is, NULL);
 	}
 	return 0;
 }
@@ -72,11 +84,12 @@ static int adbsign_main(void *pctx, struct apk_ctx *ac, struct apk_string_array
 
 	ctx->ac = ac;
 	foreach_array_item(arg, args) {
-		ctx->xfrm.is = adb_decompress(apk_istream_from_file(AT_FDCWD, *arg), &comp);
-		ctx->xfrm.os = adb_compress(apk_ostream_to_file(AT_FDCWD, *arg, 0644), comp);
-		adb_c_xfrm(&ctx->xfrm, update_signatures);
-		apk_istream_close(ctx->xfrm.is);
-		r = apk_ostream_close(ctx->xfrm.os);
+		struct apk_istream *is = adb_decompress(apk_istream_from_file_mmap(AT_FDCWD, *arg), &comp);
+		ctx->os = adb_compress(apk_ostream_to_file(AT_FDCWD, *arg, 0644), comp);
+		adb_m_process(&ctx->db, is, 0, 0, process_block);
+		process_signatures(ctx);
+		adb_free(&ctx->db);
+		r = apk_ostream_close(ctx->os);
 		if (r) apk_err(out, "%s: %s", *arg, apk_error_str(r));
 	}
 
diff --git a/src/app_extract.c b/src/app_extract.c
index a68cfbf..9de2bc2 100644
--- a/src/app_extract.c
+++ b/src/app_extract.c
@@ -287,12 +287,15 @@ static int apk_extract_next_file(struct extract_ctx *ctx)
 	} while (1);
 }
 
-static int apk_extract_data_block(struct adb *db, size_t sz, struct apk_istream *is)
+static int apk_extract_data_block(struct adb *db, struct adb_block *b, struct apk_istream *is)
 {
 	struct extract_ctx *ctx = container_of(db, struct extract_ctx, db);
 	struct adb_data_package *hdr;
+	size_t sz = adb_block_length(b);
 	int r;
 
+	if (adb_block_type(b) != ADB_BLOCK_DATA) return 0;
+
 	r = apk_extract_next_file(ctx);
 	if (r != 0) {
 		if (r > 0) r = -APKE_ADB_BLOCK;
@@ -319,7 +322,7 @@ static int apk_extract_pkg(struct extract_ctx *ctx, const char *fn)
 	struct apk_trust *trust = apk_ctx_get_trust(ac);
 	int r;
 
-	r = adb_m_stream(&ctx->db,
+	r = adb_m_process(&ctx->db,
 		adb_decompress(apk_istream_from_fd_url(AT_FDCWD, fn, apk_ctx_since(ac, 0)), 0),
 		ADB_SCHEMA_PACKAGE, trust, apk_extract_data_block);
 	if (r == 0) {
diff --git a/src/app_mkndx.c b/src/app_mkndx.c
index 4599fd3..bab6983 100644
--- a/src/app_mkndx.c
+++ b/src/app_mkndx.c
@@ -204,6 +204,7 @@ static int mkndx_main(void *pctx, struct apk_ctx *ac, struct apk_string_array *a
 		return -1;
 	}
 
+	adb_init(&odb);
 	adb_w_init_tmp(&tmpdb, 200);
 	adb_wo_alloca(&tmpl, &schema_pkginfo, &tmpdb);
 
@@ -215,7 +216,8 @@ static int mkndx_main(void *pctx, struct apk_ctx *ac, struct apk_string_array *a
 		apk_fileinfo_get(AT_FDCWD, ctx->index, 0, &fi, 0);
 		index_mtime = fi.mtime;
 
-		r = adb_m_map(&odb, open(ctx->index, O_RDONLY), ADB_SCHEMA_INDEX, trust);
+		r = adb_m_open(&odb, apk_istream_from_file_mmap(AT_FDCWD, ctx->index),
+			ADB_SCHEMA_INDEX, trust);
 		if (r) {
 			apk_err(out, "%s: %s", ctx->index, apk_error_str(r));
 			return r;
diff --git a/src/io.c b/src/io.c
index 0ec278b..ccd2ec0 100644
--- a/src/io.c
+++ b/src/io.c
@@ -70,6 +70,13 @@ void apk_file_meta_to_fd(int fd, struct apk_file_meta *meta)
 	futimens(fd, times);
 }
 
+apk_blob_t apk_istream_mmap(struct apk_istream *is)
+{
+	if (is->flags & APK_ISTREAM_SINGLE_READ)
+		return APK_BLOB_PTR_LEN((char*)is->buf, is->buf_size);
+	return APK_BLOB_NULL;
+}
+
 ssize_t apk_istream_read(struct apk_istream *is, void *ptr, size_t size)
 {
 	ssize_t left = size, r = 0;
@@ -160,7 +167,7 @@ apk_blob_t apk_istream_get_max(struct apk_istream *is, size_t max)
 
 	if (is->ptr != is->end) {
 		apk_blob_t ret = APK_BLOB_PTR_LEN((char*)is->ptr, min((size_t)(is->end - is->ptr), max));
-		is->ptr = is->end = 0;
+		is->ptr += ret.len;
 		return ret;
 	}
 
@@ -193,6 +200,41 @@ apk_blob_t apk_istream_get_delim(struct apk_istream *is, apk_blob_t token)
 	return (struct apk_blob) { .len = is->err < 0 ? is->err : 0 };
 }
 
+static void blob_get_meta(struct apk_istream *is, struct apk_file_meta *meta)
+{
+	*meta = (struct apk_file_meta) { };
+}
+
+static ssize_t blob_read(struct apk_istream *is, void *ptr, size_t size)
+{
+	return 0;
+}
+
+static int blob_close(struct apk_istream *is)
+{
+	return is->err < 0 ? is->err : 0;
+}
+
+static const struct apk_istream_ops blob_istream_ops = {
+	.get_meta = blob_get_meta,
+	.read = blob_read,
+	.close = blob_close,
+};
+
+struct apk_istream *apk_istream_from_blob(struct apk_istream *is, apk_blob_t blob)
+{
+	*is = (struct apk_istream) {
+		.ops = &blob_istream_ops,
+		.buf = (uint8_t*) blob.ptr,
+		.buf_size = blob.len,
+		.ptr = (uint8_t*) blob.ptr,
+		.end = (uint8_t*) blob.ptr + blob.len,
+		.flags = APK_ISTREAM_SINGLE_READ,
+		.err = 1,
+	};
+	return is;
+}
+
 static void segment_get_meta(struct apk_istream *is, struct apk_file_meta *meta)
 {
 	struct apk_segment_istream *sis = container_of(is, struct apk_segment_istream, is);
@@ -493,14 +535,14 @@ struct apk_istream *apk_istream_from_fd(int fd)
 	return &fis->is;
 }
 
-struct apk_istream *apk_istream_from_file(int atfd, const char *file)
+struct apk_istream *__apk_istream_from_file(int atfd, const char *file, int try_mmap)
 {
 	int fd;
 
 	fd = openat(atfd, file, O_RDONLY | O_CLOEXEC);
 	if (fd < 0) return ERR_PTR(-errno);
 
-	if (0) {
+	if (try_mmap) {
 		struct apk_istream *is = apk_mmap_istream_from_fd(fd);
 		if (!IS_ERR_OR_NULL(is)) return is;
 	}
@@ -834,11 +876,6 @@ int apk_dir_foreach_file(int dirfd, apk_dir_file_cb cb, void *ctx)
 	return ret;
 }
 
-struct apk_istream *apk_istream_from_file_gz(int atfd, const char *file)
-{
-	return apk_istream_gunzip(apk_istream_from_file(atfd, file));
-}
-
 struct apk_fd_ostream {
 	struct apk_ostream os;
 	int fd;
diff --git a/src/meson.build b/src/meson.build
index ff0b3cc..98b2461 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -5,7 +5,7 @@ libapk_src = [
 	'adb_walk_adb.c',
 	'adb_walk_genadb.c',
 	'adb_walk_gentext.c',
-	'adb_walk_istream.c',
+	'adb_walk_text.c',
 	'apk_adb.c',
 	'atom.c',
 	'blob.c',