adb: support seamless de/compression of adb files

Add compression header of adb files. Support uncompressed and
deflate compression at this time.

src/Makefile

@@ -19,9 +19,9 @@ ZLIB_LIBS		:= $(shell $(PKG_CONFIG) --libs zlib)
 libapk_soname		:= 2.99.0
 libapk_so		:= $(obj)/libapk.so.$(libapk_soname)
 libapk.so.$(libapk_soname)-objs := \
-	adb.o adb_walk_adb.o adb_walk_genadb.o adb_walk_gentext.o adb_walk_istream.o apk_adb.o \
+	adb.o adb_comp.o adb_walk_adb.o adb_walk_genadb.o adb_walk_gentext.o adb_walk_istream.o apk_adb.o \
 	atom.o blob.o commit.o common.o context.o crypto_openssl.o database.o hash.o \
-	io.o io_url.o io_gunzip.o io_archive.o \
+	io.o io_archive.o io_gunzip.o io_url.o \
 	package.o pathbuilder.o print.o solver.o trust.o version.o
 
 libapk.so.$(libapk_soname)-libs := libfetch/libfetch.a

src/adb.h

@@ -300,4 +300,14 @@ struct adb_walk_genadb {
 int adb_walk_adb(struct adb_walk *d, struct adb *db, struct apk_trust *trust);
 int adb_walk_istream(struct adb_walk *d, struct apk_istream *is);
 
+// Seamless compression support
+
+typedef unsigned int adb_comp_t;
+
+#define ADB_COMP_NONE		0
+#define ADB_COMP_DEFLATE	1
+
+struct apk_istream *adb_decompress(struct apk_istream *is, adb_comp_t *compression);
+struct apk_ostream *adb_compress(struct apk_ostream *os, adb_comp_t compression);
+
 #endif

src/adb_comp.c

@@ -0,0 +1,48 @@
+/* adb_comp.c - Alpine Package Keeper (APK)
+ *
+ * Copyright (C) 2021 Timo Teräs <timo.teras@iki.fi>
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ */
+
+#include "apk_defines.h"
+#include "adb.h"
+
+struct apk_istream *adb_decompress(struct apk_istream *is, adb_comp_t *compression)
+{
+	 adb_comp_t c = ADB_COMP_NONE;
+
+	if (IS_ERR_OR_NULL(is)) return is;
+
+	uint8_t *buf = apk_istream_peek(is, 4);
+	if (memcmp(buf, "ADB", 3) == 0) switch (buf[3]) {
+	case '.':
+		c = ADB_COMP_NONE;
+		break;
+	case 'd':
+		c = ADB_COMP_DEFLATE;
+		is = apk_istream_deflate(is);
+		break;
+	default:
+		apk_istream_close(is);
+		return ERR_PTR(-APKE_ADB_COMPRESSION);
+	}
+	if (compression) *compression = c;
+	return is;
+}
+
+struct apk_ostream *adb_compress(struct apk_ostream *os, adb_comp_t compression)
+{
+	if (IS_ERR_OR_NULL(os)) return os;
+	switch (compression) {
+	case ADB_COMP_NONE:
+		return os;
+	case ADB_COMP_DEFLATE:
+		if (apk_ostream_write(os, "ADBd", 4) < 0) goto err;
+		return apk_ostream_deflate(os);
+	}
+err:
+	apk_ostream_cancel(os, -APKE_ADB_COMPRESSION);
+	return ERR_PTR(apk_ostream_close(os));
+}

src/apk_defines.h

@@ -43,6 +43,7 @@ enum {
 	APKE_SIGNATURE_FAIL,
 	APKE_SIGNATURE_UNTRUSTED,
 	APKE_SIGNATURE_INVALID,
+	APKE_ADB_COMPRESSION,
 	APKE_ADB_HEADER,
 	APKE_ADB_SCHEMA,
 	APKE_ADB_BLOCK,

src/apk_io.h

@@ -93,6 +93,7 @@ struct apk_istream *apk_istream_from_fd(int fd);
 struct apk_istream *apk_istream_from_fd_url_if_modified(int atfd, const char *url, time_t since);
 static inline int apk_istream_error(struct apk_istream *is, int err) { if (!is->err) is->err = err; return err; }
 ssize_t apk_istream_read(struct apk_istream *is, void *ptr, size_t size);
+void *apk_istream_peek(struct apk_istream *is, size_t len);
 void *apk_istream_get(struct apk_istream *is, size_t len);
 apk_blob_t apk_istream_get_max(struct apk_istream *is, size_t size);
 apk_blob_t apk_istream_get_delim(struct apk_istream *is, apk_blob_t token);
@@ -119,19 +120,6 @@ static inline int apk_istream_close(struct apk_istream *is)
 	return is->ops->close(is);
 }
 
-#define APK_MPART_DATA		1 /* data processed so far */
-#define APK_MPART_BOUNDARY	2 /* final part of data, before boundary */
-#define APK_MPART_END		3 /* signals end of stream */
-
-typedef int (*apk_multipart_cb)(void *ctx, int part, apk_blob_t data);
-
-struct apk_istream *apk_istream_gunzip_mpart(struct apk_istream *,
-					     apk_multipart_cb cb, void *ctx);
-static inline struct apk_istream *apk_istream_gunzip(struct apk_istream *is)
-{
-	return apk_istream_gunzip_mpart(is, NULL, NULL);
-}
-
 struct apk_segment_istream {
 	struct apk_istream is;
 	struct apk_istream *pis;
@@ -152,7 +140,6 @@ struct apk_ostream {
 	int rc;
 };
 
-struct apk_ostream *apk_ostream_gzip(struct apk_ostream *);
 struct apk_ostream *apk_ostream_counter(off_t *);
 struct apk_ostream *apk_ostream_to_fd(int fd);
 struct apk_ostream *apk_ostream_to_file(int atfd, const char *file, mode_t mode);
@@ -199,4 +186,33 @@ gid_t apk_id_cache_resolve_gid(struct apk_id_cache *idc, apk_blob_t groupname, g
 apk_blob_t apk_id_cache_resolve_user(struct apk_id_cache *idc, uid_t uid);
 apk_blob_t apk_id_cache_resolve_group(struct apk_id_cache *idc, gid_t gid);
 
+// Gzip support
+
+#define APK_MPART_DATA		1 /* data processed so far */
+#define APK_MPART_BOUNDARY	2 /* final part of data, before boundary */
+#define APK_MPART_END		3 /* signals end of stream */
+
+typedef int (*apk_multipart_cb)(void *ctx, int part, apk_blob_t data);
+
+struct apk_istream *apk_istream_zlib(struct apk_istream *, int,
+				     apk_multipart_cb cb, void *ctx);
+static inline struct apk_istream *apk_istream_gunzip_mpart(struct apk_istream *is,
+					     apk_multipart_cb cb, void *ctx) {
+	return apk_istream_zlib(is, 0, cb, ctx);
+}
+static inline struct apk_istream *apk_istream_gunzip(struct apk_istream *is) {
+	return apk_istream_zlib(is, 0, NULL, NULL);
+}
+static inline struct apk_istream *apk_istream_deflate(struct apk_istream *is) {
+	return apk_istream_zlib(is, 1, NULL, NULL);
+}
+
+struct apk_ostream *apk_ostream_zlib(struct apk_ostream *, int);
+static inline struct apk_ostream *apk_ostream_gzip(struct apk_ostream *os) {
+	return apk_ostream_zlib(os, 0);
+}
+static inline struct apk_ostream *apk_ostream_deflate(struct apk_ostream *os) {
+	return apk_ostream_zlib(os, 1);
+}
+
 #endif

src/io.c

@@ -130,12 +130,11 @@ static int __apk_istream_fill(struct apk_istream *is)
 	return 0;
 }
 
-void *apk_istream_get(struct apk_istream *is, size_t len)
+void *apk_istream_peek(struct apk_istream *is, size_t len)
 {
 	do {
 		if (is->end - is->ptr >= len) {
 			void *ptr = is->ptr;
-			is->ptr += len;
 			return ptr;
 		}
 	} while (!__apk_istream_fill(is));
@@ -147,6 +146,13 @@ void *apk_istream_get(struct apk_istream *is, size_t len)
 	return ERR_PTR(-EIO);
 }
 
+void *apk_istream_get(struct apk_istream *is, size_t len)
+{
+	void *p = apk_istream_peek(is, len);
+	if (!IS_ERR_OR_NULL(p)) is->ptr += len;
+	return p;
+}
+
 apk_blob_t apk_istream_get_max(struct apk_istream *is, size_t max)
 {
 	if (is->ptr == is->end)

src/io_gunzip.c

@@ -135,7 +135,13 @@ static const struct apk_istream_ops gunzip_istream_ops = {
 	.close = gzi_close,
 };
 
-struct apk_istream *apk_istream_gunzip_mpart(struct apk_istream *is, apk_multipart_cb cb, void *ctx)
+static int window_bits(int window_bits, int raw)
+{
+	if (raw) return -window_bits;	// raw mode
+	return window_bits | 16;	// gzip mode
+}
+
+struct apk_istream *apk_istream_zlib(struct apk_istream *is, int raw, apk_multipart_cb cb, void *ctx)
 {
 	struct apk_gzip_istream *gis;
 
@@ -153,7 +159,7 @@ struct apk_istream *apk_istream_gunzip_mpart(struct apk_istream *is, apk_multipa
 		.cbctx = ctx,
 	};
 
-	if (inflateInit2(&gis->zs, 15+32) != Z_OK) {
+	if (inflateInit2(&gis->zs, window_bits(15, raw)) != Z_OK) {
 		free(gis);
 		goto err;
 	}
@@ -225,7 +231,7 @@ static const struct apk_ostream_ops gzip_ostream_ops = {
 	.close = gzo_close,
 };
 
-struct apk_ostream *apk_ostream_gzip(struct apk_ostream *output)
+struct apk_ostream *apk_ostream_zlib(struct apk_ostream *output, int raw)
 {
 	struct apk_gzip_ostream *gos;
 
@@ -239,7 +245,7 @@ struct apk_ostream *apk_ostream_gzip(struct apk_ostream *output)
 		.output = output,
 	};
 
-	if (deflateInit2(&gos->zs, 9, Z_DEFLATED, 15 | 16, 8,
+	if (deflateInit2(&gos->zs, 9, Z_DEFLATED, window_bits(15, raw), 8,
 			 Z_DEFAULT_STRATEGY) != Z_OK) {
 		free(gos);
 		goto err;

src/meson.build

@@ -1,6 +1,7 @@
 libapk_so_version = '2.99.0'
 libapk_src = [
 	'adb.c',
+	'adb_comp.c',
 	'adb_walk_adb.c',
 	'adb_walk_genadb.c',
 	'adb_walk_gentext.c',

src/print.c

@@ -39,6 +39,7 @@ const char *apk_error_str(int error)
 	case APKE_SIGNATURE_FAIL:		return "signing failure";
 	case APKE_SIGNATURE_UNTRUSTED:		return "UNTRUSTED signature";
 	case APKE_SIGNATURE_INVALID:		return "BAD signature";
+	case APKE_ADB_COMPRESSION:		return "ADB compression not supported";
 	case APKE_ADB_HEADER:			return "ADB header error";
 	case APKE_ADB_SCHEMA:			return "ADB schema error";
 	case APKE_ADB_BLOCK:			return "ADB block error";