jaiden
/
apk-tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bstream: make tokenizable and load index using bstream 

some fixes on index reading code too.
cute-signatures
Timo Teras 2009-07-14 09:33:32 +03:00
parent e69b81f525
commit 4562f44f9b
10 changed files with 222 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/apk_blob.h
View File

@ -22,7 +22,7 @@ struct apk_blob {
typedef struct apk_blob apk_blob_t;
typedef int (*apk_blob_cb)(void *ctx, apk_blob_t blob);
#define APK_BLOB_IS_NULL(blob) (blob.ptr == NULL)
#define APK_BLOB_IS_NULL(blob) ((blob).ptr == NULL)
#define APK_BLOB_NULL ((apk_blob_t){0, NULL})
#define APK_BLOB_BUF(buf) ((apk_blob_t){sizeof(buf), (char *)(buf)})
@ -39,11 +39,12 @@ static inline apk_blob_t APK_BLOB_STR(const char *str)
char *apk_blob_cstr(apk_blob_t str);
int apk_blob_spn(apk_blob_t blob, const char *accept, apk_blob_t *l, apk_blob_t *r);
int apk_blob_cspn(apk_blob_t blob, const char *reject, apk_blob_t *l, apk_blob_t *r);
int apk_blob_splitstr(apk_blob_t blob, const char *split, apk_blob_t *l, apk_blob_t *r);
int apk_blob_split(apk_blob_t blob, apk_blob_t split, apk_blob_t *l, apk_blob_t *r);
int apk_blob_rsplit(apk_blob_t blob, char split, apk_blob_t *l, apk_blob_t *r);
unsigned apk_blob_uint(apk_blob_t blob, int base);
unsigned long apk_blob_hash(apk_blob_t str);
int apk_blob_compare(apk_blob_t a, apk_blob_t b);
unsigned int apk_blob_parse_uint(apk_blob_t *b, int radix);
int apk_blob_parse_char(apk_blob_t *b);
int apk_blob_for_each_segment(apk_blob_t blob, const char *split,
apk_blob_cb cb, void *ctx);

2
src/apk_database.h
View File

@ -135,7 +135,7 @@ struct apk_package *apk_db_pkg_add(struct apk_database *db, struct apk_package *
struct apk_package *apk_db_get_pkg(struct apk_database *db, csum_t sum);
struct apk_package *apk_db_get_file_owner(struct apk_database *db, apk_blob_t filename);
int apk_db_index_read(struct apk_database *db, struct apk_istream *is, int repo);
int apk_db_index_read(struct apk_database *db, struct apk_bstream *bs, int repo);
int apk_db_index_write(struct apk_database *db, struct apk_ostream *os);
int apk_db_add_repository(apk_database_t db, apk_blob_t repository);

2
src/apk_io.h
View File

@ -35,7 +35,7 @@ struct apk_istream {
};
struct apk_bstream {
size_t (*read)(void *stream, void **ptr);
apk_blob_t (*read)(void *stream, apk_blob_t token);
void (*close)(void *stream, size_t *size);
};

7
src/archive.c
View File

@ -44,7 +44,12 @@ struct tar_header {
char padding[12]; /* 500-512 */
};
#define GET_OCTAL(s) apk_blob_uint(APK_BLOB_PTR_LEN(s, sizeof(s)), 8)
#define GET_OCTAL(s) get_octal(s, sizeof(s))
static int get_octal(char *s, size_t l)
{
apk_blob_t b = APK_BLOB_PTR_LEN(s, l);
return apk_blob_parse_uint(&b, 8);
}
struct apk_tar_entry_istream {
struct apk_istream is;

48
src/blob.c
View File

@ -75,26 +75,25 @@ int apk_blob_rsplit(apk_blob_t blob, char split, apk_blob_t *l, apk_blob_t *r)
return 1;
}
int apk_blob_splitstr(apk_blob_t blob, const char *split, apk_blob_t *l, apk_blob_t *r)
int apk_blob_split(apk_blob_t blob, apk_blob_t split, apk_blob_t *l, apk_blob_t *r)
{
int splitlen = strlen(split);
char *pos = blob.ptr, *end = blob.ptr + blob.len - splitlen + 1;
char *pos = blob.ptr, *end = blob.ptr + blob.len - split.len + 1;
if (end < pos)
return 0;
while (1) {
pos = memchr(pos, split[0], end - pos);
pos = memchr(pos, split.ptr[0], end - pos);
if (pos == NULL)
return 0;
if (memcmp(pos, split, splitlen) != 0) {
if (split.len > 1 && memcmp(pos, split.ptr, split.len) != 0) {
pos++;
continue;
}
*l = APK_BLOB_PTR_PTR(blob.ptr, pos-1);
*r = APK_BLOB_PTR_PTR(pos+splitlen, blob.ptr+blob.len-1);
*r = APK_BLOB_PTR_PTR(pos+split.len, blob.ptr+blob.len-1);
return 1;
}
}
@ -122,11 +121,11 @@ int apk_blob_compare(apk_blob_t a, apk_blob_t b)
int apk_blob_for_each_segment(apk_blob_t blob, const char *split,
int (*cb)(void *ctx, apk_blob_t blob), void *ctx)
{
apk_blob_t l, r;
apk_blob_t l, r, s = APK_BLOB_STR(split);
int rc;
r = blob;
while (apk_blob_splitstr(r, split, &l, &r)) {
while (apk_blob_split(r, s, &l, &r)) {
rc = cb(ctx, l);
if (rc != 0)
return rc;
@ -136,7 +135,7 @@ int apk_blob_for_each_segment(apk_blob_t blob, const char *split,
return 0;
}
static int dx(int c)
static inline int dx(int c)
{
if (c >= '0' && c <= '9')
return c - '0';
@ -147,24 +146,39 @@ static int dx(int c)
return -1;
}
unsigned apk_blob_uint(apk_blob_t blob, int base)
unsigned int apk_blob_parse_uint(apk_blob_t *blob, int base)
{
unsigned val;
int i, ch;
unsigned int val;
int ch;
val = 0;
for (i = 0; i < blob.len; i++) {
if (blob.ptr[i] == 0)
break;
ch = dx(blob.ptr[i]);
while (blob->len && blob->ptr[0] != 0) {
ch = dx(blob->ptr[0]);
if (ch < 0 || ch >= base)
return 0;
break;
val *= base;
val += ch;
blob->ptr++;
blob->len--;
}
return val;
}
int apk_blob_parse_char(apk_blob_t *blob)
{
int r;
if (blob->len == 0 || blob->ptr == NULL)
return -1;
r = blob->ptr[0];
blob->ptr++;
blob->len--;
return r;
}
int apk_hexdump_parse(apk_blob_t to, apk_blob_t from)
{
int i;

202
src/database.c
View File

@ -205,7 +205,8 @@ static struct apk_db_dir *apk_db_dir_get(struct apk_database *db,
return apk_db_dir_ref(dir);
db->installed.stats.dirs++;
dir = calloc(1, sizeof(*dir) + name.len + 1);
dir = malloc(sizeof(*dir) + name.len + 1);
memset(dir, 0, sizeof(*dir));
dir->refs = 1;
memcpy(dir->dirname, name.ptr, name.len);
dir->dirname[name.len] = 0;
@ -311,7 +312,8 @@ static struct apk_db_file *apk_db_file_get(struct apk_database *db,
if (file != NULL)
return file;
file = calloc(1, sizeof(*file) + name.len + 1);
file = malloc(sizeof(*file) + name.len + 1);
memset(file, 0, sizeof(*file));
memcpy(file->filename, name.ptr, name.len);
file->filename[name.len] = 0;
@ -376,116 +378,107 @@ struct apk_package *apk_db_pkg_add(struct apk_database *db, struct apk_package *
return idb;
}
int apk_db_index_read(struct apk_database *db, struct apk_istream *is, int repo)
int apk_db_index_read(struct apk_database *db, struct apk_bstream *bs, int repo)
{
struct apk_package *pkg = NULL;
struct apk_db_dir_instance *diri = NULL;
struct apk_db_file *file = NULL;
struct hlist_node **diri_node = NULL;
struct hlist_node **file_diri_node = NULL;
apk_blob_t token = APK_BLOB_STR("\n"), l;
int field;
char buf[1024];
apk_blob_t l, r;
int n, field;
r = APK_BLOB_PTR_LEN(buf, 0);
while (1) {
n = is->read(is, &r.ptr[r.len], sizeof(buf) - r.len);
if (n <= 0)
break;
r.len += n;
while (apk_blob_splitstr(r, "\n", &l, &r)) {
if (l.len < 2 || l.ptr[1] != ':') {
if (pkg == NULL)
continue;
if (repo != -1)
pkg->repos |= BIT(repo);
else
apk_pkg_set_state(db, pkg, APK_PKG_INSTALLED);
if (apk_db_pkg_add(db, pkg) != pkg && repo == -1) {
apk_error("Installed database load failed");
return -1;
}
pkg = NULL;
continue;
}
/* Get field */
field = l.ptr[0];
l.ptr += 2;
l.len -= 2;
/* If no package, create new */
if (pkg == NULL) {
pkg = apk_pkg_new();
diri = NULL;
diri_node = hlist_tail_ptr(&pkg->owned_dirs);
file_diri_node = NULL;
}
/* Standard index line? */
if (apk_pkg_add_info(db, pkg, field, l) == 0)
while (!APK_BLOB_IS_NULL(l = bs->read(bs, token))) {
if (l.len < 2 || l.ptr[1] != ':') {
if (pkg == NULL)
continue;
if (repo != -1) {
apk_error("Invalid index entry '%c'", field);
return -1;
}
if (repo != -1)
pkg->repos |= BIT(repo);
else
apk_pkg_set_state(db, pkg, APK_PKG_INSTALLED);
/* Check FDB special entries */
switch (field) {
case 'F':
if (pkg->name == NULL) {
apk_error("FDB directory entry before package entry");
return -1;
}
diri = apk_db_diri_new(db, pkg, l, &diri_node);
file_diri_node = &diri->owned_files.first;
break;
case 'M':
if (diri == NULL) {
apk_error("FDB directory metadata entry before directory entry");
return -1;
}
/* FIXME: sscanf may touch unallocated area */
if (sscanf(l.ptr, "%d:%d:%o",
&diri->uid, &diri->gid, &diri->mode) != 3) {
apk_error("FDB bad directory mode entry");
return -1;
}
break;
case 'R':
if (diri == NULL) {
apk_error("FDB file entry before directory entry");
return -1;
}
file = apk_db_file_get(db, diri, l,
&file_diri_node);
break;
case 'Z':
if (file == NULL) {
apk_error("FDB checksum entry before file entry");
return -1;
}
if (apk_hexdump_parse(APK_BLOB_BUF(file->csum), l)) {
apk_error("Not a valid checksum");
return -1;
}
break;
default:
apk_error("FDB entry '%c' unsupported", n);
if (apk_db_pkg_add(db, pkg) != pkg && repo == -1) {
apk_error("Installed database load failed");
return -1;
}
pkg = NULL;
continue;
}
memcpy(&buf[0], r.ptr, r.len);
r = APK_BLOB_PTR_LEN(buf, r.len);
/* Get field */
field = l.ptr[0];
l.ptr += 2;
l.len -= 2;
/* If no package, create new */
if (pkg == NULL) {
pkg = apk_pkg_new();
diri = NULL;
diri_node = hlist_tail_ptr(&pkg->owned_dirs);
file_diri_node = NULL;
}
/* Standard index line? */
if (apk_pkg_add_info(db, pkg, field, l) == 0)
continue;
if (repo != -1) {
apk_error("Invalid index entry '%c'", field);
return -1;
}
/* Check FDB special entries */
switch (field) {
case 'F':
if (pkg->name == NULL) {
apk_error("FDB directory entry before package entry");
return -1;
}
diri = apk_db_diri_new(db, pkg, l, &diri_node);
file_diri_node = &diri->owned_files.first;
break;
case 'M':
if (diri == NULL) {
apk_error("FDB directory metadata entry before directory entry");
return -1;
}
diri->uid = apk_blob_parse_uint(&l, 10);
if (apk_blob_parse_char(&l) != ':')
goto bad_mode;
diri->gid = apk_blob_parse_uint(&l, 10);
if (apk_blob_parse_char(&l) != ':')
goto bad_mode;
diri->mode = apk_blob_parse_uint(&l, 8);
break;
case 'R':
if (diri == NULL) {
apk_error("FDB file entry before directory entry");
return -1;
}
file = apk_db_file_get(db, diri, l,
&file_diri_node);
break;
case 'Z':
if (file == NULL) {
apk_error("FDB checksum entry before file entry");
return -1;
}
if (apk_hexdump_parse(APK_BLOB_BUF(file->csum), l)) {
apk_error("Not a valid checksum");
return -1;
}
break;
default:
apk_error("FDB entry '%c' unsupported", field);
return -1;
}
}
return 0;
bad_mode:
apk_error("FDB bad directory mode entry");
return -1;
}
static int apk_db_write_fdb(struct apk_database *db, struct apk_ostream *os)
@ -581,6 +574,7 @@ static int apk_db_scriptdb_read(struct apk_database *db, struct apk_istream *is)
static int apk_db_read_state(struct apk_database *db, int flags)
{
struct apk_istream *is;
struct apk_bstream *bs;
apk_blob_t blob;
int i;
@ -606,10 +600,10 @@ static int apk_db_read_state(struct apk_database *db, int flags)
}
if (!(flags & APK_OPENF_NO_INSTALLED)) {
is = apk_istream_from_file("var/lib/apk/installed");
if (is != NULL) {
apk_db_index_read(db, is, -1);
is->close(is);
bs = apk_bstream_from_file("var/lib/apk/installed");
if (bs != NULL) {
apk_db_index_read(db, bs, -1);
bs->close(bs, NULL);
}
}
@ -1013,7 +1007,6 @@ int apk_repository_update(struct apk_database *db, struct apk_repository *repo)
int apk_db_add_repository(apk_database_t _db, apk_blob_t repository)
{
struct apk_database *db = _db.db;
struct apk_istream *is = NULL;
struct apk_bstream *bs = NULL;
struct apk_repository *repo;
int r, n = 1;
@ -1050,13 +1043,13 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t repository)
} else {
bs = apk_repository_file_open(repo, apk_index_gz);
}
is = apk_bstream_gunzip(bs, 1);
if (is == NULL) {
bs = apk_bstream_from_istream(apk_bstream_gunzip(bs, TRUE));
if (bs == NULL) {
apk_warning("Failed to open index for %s", repo->url);
return -1;
}
apk_db_index_read(db, is, r);
is->close(is);
apk_db_index_read(db, bs, r);
bs->close(bs, NULL);
return 0;
}
@ -1276,6 +1269,7 @@ static int apk_db_gzip_part(void *pctx, EVP_MD_CTX *mdctx, int part)
{
struct install_ctx *ctx = (struct install_ctx *) pctx;
fprintf(stderr, "part %d\n", part);
switch (part) {
case APK_MPART_BEGIN:
EVP_DigestInit_ex(mdctx, EVP_md5(), NULL);

6
src/gunzip.c
View File

@ -49,12 +49,16 @@ static size_t gz_read(void *stream, void *ptr, size_t size)
while (gis->zs.avail_out != 0 && gis->z_err == Z_OK) {
if (gis->zs.avail_in == 0) {
apk_blob_t blob;
if (gis->cb != NULL && gis->mdblock != NULL) {
/* Digest the inflated bytes */
EVP_DigestUpdate(&gis->mdctx, gis->mdblock,
(void *)gis->zs.next_in - gis->mdblock);
}
gis->zs.avail_in = gis->bs->read(gis->bs, &gis->mdblock);
blob = gis->bs->read(gis->bs, APK_BLOB_NULL);
gis->mdblock = blob.ptr;
gis->zs.avail_in = blob.len;
gis->zs.next_in = (void *) gis->mdblock;
if (gis->zs.avail_in < 0) {
gis->z_err = Z_DATA_ERROR;

10
src/index.c
View File

@ -41,15 +41,15 @@ static int index_parse(void *ctx, int optch, int optindex, const char *optarg)
static int index_read_file(struct apk_database *db, struct index_ctx *ictx)
{
struct apk_istream *is;
struct apk_bstream *bs;
int r;
if (ictx->index_file == NULL)
return 0;
is = apk_bstream_gunzip(apk_bstream_from_url(ictx->index_file), 1);
if (is == NULL)
bs = apk_bstream_from_istream(apk_bstream_gunzip(apk_bstream_from_url(ictx->index_file), 1));
if (bs == NULL)
return -1;
r = apk_db_index_read(db, is, -1);
is->close(is);
r = apk_db_index_read(db, bs, -1);
bs->close(bs, NULL);
return r;
}

98
src/io.c
View File

@ -150,24 +150,60 @@ err:
struct apk_istream_bstream {
struct apk_bstream bs;
struct apk_istream *is;
unsigned char buffer[8*1024];
apk_blob_t left;
char buffer[8*1024];
size_t size;
};
static size_t is_bs_read(void *stream, void **ptr)
static apk_blob_t is_bs_read(void *stream, apk_blob_t token)
{
struct apk_istream_bstream *isbs =
container_of(stream, struct apk_istream_bstream, bs);
size_t size;
ssize_t size;
apk_blob_t ret;
size = isbs->is->read(isbs->is, isbs->buffer, sizeof(isbs->buffer));
if (size <= 0)
return size;
/* If we have cached stuff, first check if it full fills the request */
if (isbs->left.len != 0) {
if (!APK_BLOB_IS_NULL(token)) {
/* If we have tokenized thingy left, return it */
if (apk_blob_split(isbs->left, token, &ret, &isbs->left))
goto ret;
} else
goto ret_all;
}
isbs->size += size;
/* If we've exchausted earlier, it's end of stream */
if (APK_BLOB_IS_NULL(isbs->left))
return APK_BLOB_NULL;
*ptr = isbs->buffer;
return size;
/* We need more data */
if (isbs->left.len != 0)
memcpy(isbs->buffer, isbs->left.ptr, isbs->left.len);
isbs->left.ptr = isbs->buffer;
size = isbs->is->read(isbs->is, isbs->buffer + isbs->left.len,
sizeof(isbs->buffer) - isbs->left.len);
if (size > 0) {
isbs->size += size;
isbs->left.len += size;
} else if (size == 0) {
if (isbs->left.len == 0)
isbs->left = APK_BLOB_NULL;
goto ret_all;
}
if (!APK_BLOB_IS_NULL(token)) {
/* If we have tokenized thingy left, return it */
if (apk_blob_split(isbs->left, token, &ret, &isbs->left))
goto ret;
/* No token found; just return the full buffer */
}
ret_all:
/* Return all that is in cache */
ret = isbs->left;
isbs->left.len = 0;
ret:
return ret;
}
static void is_bs_close(void *stream, size_t *size)
@ -195,6 +231,8 @@ struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream)
.close = is_bs_close,
};
isbs->is = istream;
isbs->left = APK_BLOB_PTR_LEN(isbs->buffer, 0),
isbs->size = 0;
return &isbs->bs;
}
@ -204,23 +242,24 @@ struct apk_mmap_bstream {
int fd;
size_t size;
unsigned char *ptr;
size_t pos;
apk_blob_t left;
};
static size_t mmap_read(void *stream, void **ptr)
static apk_blob_t mmap_read(void *stream, apk_blob_t token)
{
struct apk_mmap_bstream *mbs =
container_of(stream, struct apk_mmap_bstream, bs);
size_t size;
apk_blob_t ret;
size = mbs->size - mbs->pos;
if (size > 1024*1024)
size = 1024*1024;
if (!APK_BLOB_IS_NULL(token) && !APK_BLOB_IS_NULL(mbs->left)) {
if (apk_blob_split(mbs->left, token, &ret, &mbs->left))
return ret;
}
*ptr = (void *) &mbs->ptr[mbs->pos];
mbs->pos += size;
ret = mbs->left;
mbs->left = APK_BLOB_NULL;
return size;
return ret;
}
static void mmap_close(void *stream, size_t *size)
@ -245,7 +284,7 @@ static struct apk_bstream *apk_mmap_bstream_from_fd(int fd)
if (fstat(fd, &st) < 0)
return NULL;
ptr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
ptr = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (ptr == MAP_FAILED)
return NULL;
@ -262,7 +301,7 @@ static struct apk_bstream *apk_mmap_bstream_from_fd(int fd)
mbs->fd = fd;
mbs->size = st.st_size;
mbs->ptr = ptr;
mbs->pos = 0;
mbs->left = APK_BLOB_PTR_LEN(ptr, mbs->size);
return &mbs->bs;
}
@ -301,17 +340,17 @@ struct apk_tee_bstream {
size_t size;
};
static size_t tee_read(void *stream, void **ptr)
static apk_blob_t tee_read(void *stream, apk_blob_t token)
{
struct apk_tee_bstream *tbs =
container_of(stream, struct apk_tee_bstream, bs);
ssize_t size;
apk_blob_t blob;
size = tbs->inner_bs->read(tbs->inner_bs, ptr);
if (size >= 0)
tbs->size += write(tbs->fd, *ptr, size);
blob = tbs->inner_bs->read(tbs->inner_bs, token);
if (!APK_BLOB_IS_NULL(blob))
tbs->size += write(tbs->fd, blob.ptr, blob.len);
return size;
return blob;
}
static void tee_close(void *stream, size_t *size)
@ -421,12 +460,11 @@ int apk_file_get_info(const char *filename, struct apk_file_info *fi)
bs = apk_bstream_from_file(filename);
if (bs != NULL) {
ssize_t size;
void *ptr;
apk_blob_t blob;
csum_init(&ctx);
while ((size = bs->read(bs, &ptr)) > 0)
csum_process(&ctx, ptr, size);
while (!APK_BLOB_IS_NULL(blob = bs->read(bs, APK_BLOB_NULL)))
csum_process(&ctx, (void*) blob.ptr, blob.len);
csum_finish(&ctx, fi->csum);
bs->close(bs, NULL);

6
src/package.c
View File

@ -293,10 +293,10 @@ int apk_pkg_add_info(struct apk_database *db, struct apk_package *pkg,
apk_hexdump_parse(APK_BLOB_BUF(pkg->csum), value);
break;
case 'S':
pkg->size = apk_blob_uint(value, 10);
pkg->size = apk_blob_parse_uint(&value, 10);
break;
case 'I':
pkg->installed_size = apk_blob_uint(value, 10);
pkg->installed_size = apk_blob_parse_uint(&value, 10);
break;
default:
return -1;
@ -325,7 +325,7 @@ static int read_info_line(void *ctx, apk_blob_t line)
if (line.ptr == NULL || line.len < 1 || line.ptr[0] == '#')
return 0;
if (!apk_blob_splitstr(line, " = ", &l, &r))
if (!apk_blob_split(line, APK_BLOB_STR(" = "), &l, &r))
return 0;
for (i = 0; i < ARRAY_SIZE(fields); i++) {