From e69b81f5259f532d5f5ae9c0a0f9bbd81240fbaf Mon Sep 17 00:00:00 2001 From: Timo Teras Date: Mon, 13 Jul 2009 20:37:03 +0300 Subject: [PATCH] io: move csumming away from bstream to gunzip in future we want to checksum on gzip boundary basis, not the full file. --- src/apk_io.h | 17 +++++++++++++++-- src/archive.c | 4 ++-- src/audit.c | 18 +++++------------- src/database.c | 44 ++++++++++++++++++++++++++++++-------------- src/gunzip.c | 49 +++++++++++++++++++++++++++++++++++++++++-------- src/io.c | 48 ++++++++++++++++-------------------------------- src/package.c | 28 +++++++++++++++++++++++----- 7 files changed, 132 insertions(+), 76 deletions(-) diff --git a/src/apk_io.h b/src/apk_io.h index b5e984a..629729a 100644 --- a/src/apk_io.h +++ b/src/apk_io.h @@ -36,7 +36,7 @@ struct apk_istream { struct apk_bstream { size_t (*read)(void *stream, void **ptr); - void (*close)(void *stream, csum_p csum, size_t *size); + void (*close)(void *stream, size_t *size); }; struct apk_ostream { @@ -44,7 +44,20 @@ struct apk_ostream { void (*close)(void *stream); }; -struct apk_istream *apk_bstream_gunzip(struct apk_bstream *, int); +#define APK_MPART_BEGIN 0 +#define APK_MPART_BOUNDARY 1 +#define APK_MPART_END 2 + +typedef int (*apk_multipart_cb)(void *ctx, EVP_MD_CTX *mdctx, int part); + +struct apk_istream *apk_bstream_gunzip_mpart(struct apk_bstream *, int, + apk_multipart_cb cb, void *ctx); +static inline struct apk_istream *apk_bstream_gunzip(struct apk_bstream *bs, + int autoclose) +{ + return apk_bstream_gunzip_mpart(bs, autoclose, NULL, NULL); +} + struct apk_ostream *apk_ostream_gzip(struct apk_ostream *); struct apk_istream *apk_istream_from_fd(int fd); diff --git a/src/archive.c b/src/archive.c index a05aee2..6681b61 100644 --- a/src/archive.c +++ b/src/archive.c @@ -4,7 +4,7 @@ * Copyright (C) 2008 Timo Teräs * All rights reserved. * - * This program is free software; you can redistribute it and/or modify it + * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published * by the Free Software Foundation. See http://www.gnu.org/ for details. */ @@ -91,7 +91,7 @@ int apk_parse_tar(struct apk_istream *is, apk_archive_entry_parser parser, if (buf.name[0] == '\0') { if (end) { r = 0; - break; + //break; } end++; continue; diff --git a/src/audit.c b/src/audit.c index 9ea0f25..3732aac 100644 --- a/src/audit.c +++ b/src/audit.c @@ -27,12 +27,10 @@ static int audit_directory(apk_hash_item item, void *ctx) struct apk_db_file *dbf; struct apk_database *db = (struct apk_database *) ctx; struct dirent *de; - struct stat st; - struct apk_bstream *bs; + struct apk_file_info fi; + apk_blob_t bdir = APK_BLOB_STR(dbd->dirname); char tmp[512], reason; DIR *dir; - apk_blob_t bdir = APK_BLOB_STR(dbd->dirname); - csum_t csum; if (!(dbd->flags & APK_DBDIRF_PROTECTED)) return 0; @@ -49,10 +47,10 @@ static int audit_directory(apk_hash_item item, void *ctx) snprintf(tmp, sizeof(tmp), "%s/%s", dbd->dirname, de->d_name); - if (stat(tmp, &st) < 0) + if (apk_file_get_info(tmp, &fi) < 0) continue; - if (S_ISDIR(st.st_mode)) { + if (S_ISDIR(fi.mode)) { if (apk_db_dir_query(db, APK_BLOB_STR(tmp)) != NULL) continue; @@ -60,13 +58,7 @@ static int audit_directory(apk_hash_item item, void *ctx) } else { dbf = apk_db_file_query(db, bdir, APK_BLOB_STR(de->d_name)); if (dbf != NULL) { - bs = apk_bstream_from_file(tmp); - if (bs == NULL) - continue; - - bs->close(bs, csum, NULL); - - if (apk_blob_compare(APK_BLOB_BUF(csum), + if (apk_blob_compare(APK_BLOB_BUF(fi.csum), APK_BLOB_BUF(dbf->csum)) == 0) continue; diff --git a/src/database.c b/src/database.c index 293af4c..10c0b5e 100644 --- a/src/database.c +++ b/src/database.c @@ -36,6 +36,7 @@ struct install_ctx { int script; struct apk_db_dir_instance *diri; + csum_t data_csum; apk_progress_cb cb; void *cb_ctx; @@ -1271,13 +1272,28 @@ static void apk_db_purge_pkg(struct apk_database *db, apk_pkg_set_state(db, pkg, APK_PKG_NOT_INSTALLED); } +static int apk_db_gzip_part(void *pctx, EVP_MD_CTX *mdctx, int part) +{ + struct install_ctx *ctx = (struct install_ctx *) pctx; + + switch (part) { + case APK_MPART_BEGIN: + EVP_DigestInit_ex(mdctx, EVP_md5(), NULL); + break; + case APK_MPART_END: + EVP_DigestFinal_ex(mdctx, ctx->data_csum, NULL); + break; + } + return 0; +} + static int apk_db_unpack_pkg(struct apk_database *db, struct apk_package *newpkg, - int upgrade, csum_t csum, - apk_progress_cb cb, void *cb_ctx) + int upgrade, apk_progress_cb cb, void *cb_ctx) { struct install_ctx ctx; struct apk_bstream *bs = NULL; + struct apk_istream *tar; char pkgname[256], file[256]; int i, need_copy = FALSE; size_t length; @@ -1334,10 +1350,17 @@ static int apk_db_unpack_pkg(struct apk_database *db, .cb = cb, .cb_ctx = cb_ctx, }; - if (apk_parse_tar_gz(bs, apk_db_install_archive_entry, &ctx) != 0) - goto err_close; - bs->close(bs, csum, &length); + tar = apk_bstream_gunzip_mpart(bs, FALSE, apk_db_gzip_part, &ctx); + if (apk_parse_tar(tar, apk_db_install_archive_entry, &ctx) != 0) + goto err_close; + bs->close(bs, &length); + + /* Check the package checksum */ + if (memcmp(ctx.data_csum, newpkg->csum, sizeof(csum_t)) != 0) + apk_warning("%s-%s: checksum does not match", + newpkg->name->name, newpkg->version); + if (need_copy) { if (length == newpkg->size) { char file2[256]; @@ -1351,7 +1374,7 @@ static int apk_db_unpack_pkg(struct apk_database *db, return 0; err_close: - bs->close(bs, NULL, NULL); + bs->close(bs, NULL); return -1; } @@ -1360,7 +1383,6 @@ int apk_db_install_pkg(struct apk_database *db, struct apk_package *newpkg, apk_progress_cb cb, void *cb_ctx) { - csum_t csum; int r; if (fchdir(db->root_fd) < 0) @@ -1382,19 +1404,13 @@ int apk_db_install_pkg(struct apk_database *db, /* Install the new stuff */ if (!(newpkg->name->flags & APK_NAME_VIRTUAL)) { - r = apk_db_unpack_pkg(db, newpkg, (oldpkg != NULL), csum, - cb, cb_ctx); + r = apk_db_unpack_pkg(db, newpkg, (oldpkg != NULL), cb, cb_ctx); if (r != 0) return r; } apk_pkg_set_state(db, newpkg, APK_PKG_INSTALLED); - if (!(newpkg->name->flags & APK_NAME_VIRTUAL) && - memcmp(csum, newpkg->csum, sizeof(csum)) != 0) - apk_warning("%s-%s: checksum does not match", - newpkg->name->name, newpkg->version); - if (oldpkg != NULL) apk_db_purge_pkg(db, oldpkg); diff --git a/src/gunzip.c b/src/gunzip.c index 2c7ffc2..1d2881c 100644 --- a/src/gunzip.c +++ b/src/gunzip.c @@ -23,13 +23,18 @@ struct apk_gzip_istream { z_stream zs; int z_err; int autoclose; + + EVP_MD_CTX mdctx; + void *mdblock; + + apk_multipart_cb cb; + void *cbctx; }; static size_t gz_read(void *stream, void *ptr, size_t size) { struct apk_gzip_istream *gis = container_of(stream, struct apk_gzip_istream, is); - int restart_count = 0; if (gis->z_err == Z_DATA_ERROR || gis->z_err == Z_ERRNO) return -1; @@ -44,22 +49,40 @@ static size_t gz_read(void *stream, void *ptr, size_t size) while (gis->zs.avail_out != 0 && gis->z_err == Z_OK) { if (gis->zs.avail_in == 0) { - gis->zs.avail_in = gis->bs->read(gis->bs, (void **) &gis->zs.next_in); + if (gis->cb != NULL && gis->mdblock != NULL) { + /* Digest the inflated bytes */ + EVP_DigestUpdate(&gis->mdctx, gis->mdblock, + (void *)gis->zs.next_in - gis->mdblock); + } + gis->zs.avail_in = gis->bs->read(gis->bs, &gis->mdblock); + gis->zs.next_in = (void *) gis->mdblock; if (gis->zs.avail_in < 0) { gis->z_err = Z_DATA_ERROR; return size - gis->zs.avail_out; + } else if (gis->zs.avail_in == 0) { + if (gis->cb != NULL) + gis->cb(gis->cbctx, &gis->mdctx, + APK_MPART_END); + gis->z_err = Z_STREAM_END; + return size - gis->zs.avail_out; } } gis->z_err = inflate(&gis->zs, Z_NO_FLUSH); - if (restart_count == 0 && gis->z_err == Z_STREAM_END) { + if (gis->z_err == Z_STREAM_END) { + /* Digest the inflated bytes */ + if (gis->cb != NULL) { + EVP_DigestUpdate(&gis->mdctx, gis->mdblock, + (void *)gis->zs.next_in - gis->mdblock); + gis->mdblock = gis->zs.next_in; + gis->cb(gis->cbctx, &gis->mdctx, + APK_MPART_BOUNDARY); + } inflateEnd(&gis->zs); + if (inflateInit2(&gis->zs, 15+32) != Z_OK) return -1; gis->z_err = Z_OK; - restart_count++; - } else { - restart_count = 0; } } @@ -74,13 +97,16 @@ static void gz_close(void *stream) struct apk_gzip_istream *gis = container_of(stream, struct apk_gzip_istream, is); + if (gis->cb != NULL) + EVP_MD_CTX_cleanup(&gis->mdctx); inflateEnd(&gis->zs); if (gis->autoclose) - gis->bs->close(gis->bs, NULL, NULL); + gis->bs->close(gis->bs, NULL); free(gis); } -struct apk_istream *apk_bstream_gunzip(struct apk_bstream *bs, int autoclose) +struct apk_istream *apk_bstream_gunzip_mpart(struct apk_bstream *bs, int autoclose, + apk_multipart_cb cb, void *ctx) { struct apk_gzip_istream *gis; @@ -97,6 +123,8 @@ struct apk_istream *apk_bstream_gunzip(struct apk_bstream *bs, int autoclose) .bs = bs, .z_err = 0, .autoclose = autoclose, + .cb = cb, + .cbctx = ctx, }; if (inflateInit2(&gis->zs, 15+32) != Z_OK) { @@ -104,6 +132,11 @@ struct apk_istream *apk_bstream_gunzip(struct apk_bstream *bs, int autoclose) return NULL; } + if (gis->cb != NULL) { + EVP_MD_CTX_init(&gis->mdctx); + cb(ctx, &gis->mdctx, APK_MPART_BEGIN); + } + return &gis->is; } diff --git a/src/io.c b/src/io.c index 722bccb..a250b27 100644 --- a/src/io.c +++ b/src/io.c @@ -150,7 +150,6 @@ err: struct apk_istream_bstream { struct apk_bstream bs; struct apk_istream *is; - csum_ctx_t csum_ctx; unsigned char buffer[8*1024]; size_t size; }; @@ -165,31 +164,17 @@ static size_t is_bs_read(void *stream, void **ptr) if (size <= 0) return size; - csum_process(&isbs->csum_ctx, isbs->buffer, size); isbs->size += size; *ptr = isbs->buffer; return size; } -static void is_bs_close(void *stream, csum_t csum, size_t *size) +static void is_bs_close(void *stream, size_t *size) { struct apk_istream_bstream *isbs = container_of(stream, struct apk_istream_bstream, bs); - if (csum != NULL) { - size_t size; - - do { - size = isbs->is->read(isbs->is, isbs->buffer, - sizeof(isbs->buffer)); - csum_process(&isbs->csum_ctx, isbs->buffer, size); - isbs->size += size; - } while (size == sizeof(isbs->buffer)); - - csum_finish(&isbs->csum_ctx, csum); - } - if (size != NULL) *size = isbs->size; @@ -210,14 +195,12 @@ struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream) .close = is_bs_close, }; isbs->is = istream; - csum_init(&isbs->csum_ctx); return &isbs->bs; } struct apk_mmap_bstream { struct apk_bstream bs; - csum_ctx_t csum_ctx; int fd; size_t size; unsigned char *ptr; @@ -235,24 +218,16 @@ static size_t mmap_read(void *stream, void **ptr) size = 1024*1024; *ptr = (void *) &mbs->ptr[mbs->pos]; - csum_process(&mbs->csum_ctx, &mbs->ptr[mbs->pos], size); mbs->pos += size; return size; } -static void mmap_close(void *stream, csum_t csum, size_t *size) +static void mmap_close(void *stream, size_t *size) { struct apk_mmap_bstream *mbs = container_of(stream, struct apk_mmap_bstream, bs); - if (csum != NULL) { - if (mbs->pos != mbs->size) - csum_process(&mbs->csum_ctx, &mbs->ptr[mbs->pos], - mbs->size - mbs->pos); - csum_finish(&mbs->csum_ctx, csum); - } - if (size != NULL) *size = mbs->size; @@ -288,7 +263,6 @@ static struct apk_bstream *apk_mmap_bstream_from_fd(int fd) mbs->size = st.st_size; mbs->ptr = ptr; mbs->pos = 0; - csum_init(&mbs->csum_ctx); return &mbs->bs; } @@ -340,12 +314,12 @@ static size_t tee_read(void *stream, void **ptr) return size; } -static void tee_close(void *stream, csum_t csum, size_t *size) +static void tee_close(void *stream, size_t *size) { struct apk_tee_bstream *tbs = container_of(stream, struct apk_tee_bstream, bs); - tbs->inner_bs->close(tbs->inner_bs, csum, NULL); + tbs->inner_bs->close(tbs->inner_bs, NULL); if (size != NULL) *size = tbs->size; close(tbs->fd); @@ -431,6 +405,7 @@ int apk_file_get_info(const char *filename, struct apk_file_info *fi) { struct stat st; struct apk_bstream *bs; + csum_ctx_t ctx; if (stat(filename, &st) != 0) return -1; @@ -445,8 +420,17 @@ int apk_file_get_info(const char *filename, struct apk_file_info *fi) }; bs = apk_bstream_from_file(filename); - if (bs != NULL) - bs->close(bs, fi->csum, NULL); + if (bs != NULL) { + ssize_t size; + void *ptr; + + csum_init(&ctx); + while ((size = bs->read(bs, &ptr)) > 0) + csum_process(&ctx, ptr, size); + csum_finish(&ctx, fi->csum); + + bs->close(bs, NULL); + } return 0; } diff --git a/src/package.c b/src/package.c index 6c51924..30b09b1 100644 --- a/src/package.c +++ b/src/package.c @@ -4,7 +4,7 @@ * Copyright (C) 2008 Timo Teräs * All rights reserved. * - * This program is free software; you can redistribute it and/or modify it + * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published * by the Free Software Foundation. See http://www.gnu.org/ for details. */ @@ -415,10 +415,26 @@ static int read_info_entry(void *ctx, const struct apk_file_info *ae, return 0; } +static int apk_pkg_gzip_part(void *ctx, EVP_MD_CTX *mdctx, int part) +{ + struct read_info_ctx *ri = (struct read_info_ctx *) ctx; + + switch (part) { + case APK_MPART_BEGIN: + EVP_DigestInit_ex(mdctx, EVP_md5(), NULL); + break; + case APK_MPART_END: + EVP_DigestFinal_ex(mdctx, ri->pkg->csum, NULL); + break; + } + return 0; +} + struct apk_package *apk_pkg_read(struct apk_database *db, const char *file) { struct read_info_ctx ctx; struct apk_bstream *bs; + struct apk_istream *tar; char realfile[PATH_MAX]; if (realpath(file, realfile) < 0) @@ -434,12 +450,14 @@ struct apk_package *apk_pkg_read(struct apk_database *db, const char *file) ctx.db = db; ctx.has_install = 0; - if (apk_parse_tar_gz(bs, read_info_entry, &ctx) < 0) { + + tar = apk_bstream_gunzip_mpart(bs, FALSE, apk_pkg_gzip_part, &ctx); + if (apk_parse_tar(tar, read_info_entry, &ctx) < 0) { apk_error("File %s is not an APK archive", file); - bs->close(bs, NULL, NULL); + bs->close(bs, NULL); goto err; } - bs->close(bs, ctx.pkg->csum, &ctx.pkg->size); + bs->close(bs, &ctx.pkg->size); if (ctx.pkg->name == NULL) { apk_error("File %s is corrupted", file); @@ -682,7 +700,7 @@ struct apk_dependency apk_dep_from_str(struct apk_database *db, mask = apk_version_result_mask(v++); if (*v == '=') v++; - } + } return (struct apk_dependency) { .name = apk_db_get_name(db, name), .version = v,