add adbgen applet to generate databases from it's text dump

Timo Teräs 2021-06-02 22:35:58 +03:00
parent 209201bc5d
commit 1456296b43
14 changed files with 430 additions and 8 deletions

View File

@ -252,6 +252,14 @@ struct adb_obj *adb_r_rootobj(struct adb *db, struct adb_obj *obj, const struct
return adb_r_obj(db, adb_r_root(db), obj, schema);
}
const uint8_t *adb_ro_kind(const struct adb_obj *o, unsigned i)
{
if (o->schema->kind == ADB_KIND_ADB ||
o->schema->kind == ADB_KIND_ARRAY)
i = 1;
return o->schema->fields[i-1].kind;
}
adb_val_t adb_ro_val(const struct adb_obj *o, unsigned i)
{
if (i >= o->num) return ADB_NULL;
@ -593,6 +601,7 @@ adb_val_t adb_w_fromstring(struct adb *db, const uint8_t *kind, apk_blob_t val)
struct adb_obj obj;
struct adb_object_schema *schema = container_of(kind, struct adb_object_schema, kind);
adb_wo_alloca(&obj, schema, db);
if (!schema->fromstring) return ADB_ERROR(EAPKDBFORMAT);
r = schema->fromstring(&obj, val);
if (r) return ADB_ERROR(r);
return adb_w_obj(&obj);
@ -618,6 +627,25 @@ struct adb_obj *adb_wo_init(struct adb_obj *o, adb_val_t *p, const struct adb_ob
return o;
}
struct adb_obj *adb_wo_init_val(struct adb_obj *o, adb_val_t *p, const struct adb_obj *parent, unsigned i)
{
const uint8_t *kind = adb_ro_kind(parent, i);
const struct adb_object_schema *schema = 0;
switch (*kind) {
case ADB_KIND_OBJECT:
case ADB_KIND_ARRAY:
schema = container_of(kind, struct adb_object_schema, kind);
break;
case ADB_KIND_ADB:
schema = container_of(kind, struct adb_adb_schema, kind)->schema;
break;
default:
assert(1);
}
return adb_wo_init(o, p, schema, parent->db);
}
void adb_wo_reset(struct adb_obj *o)
{
uint32_t max = o->obj[ADBI_NUM_ENTRIES];
@ -765,9 +793,17 @@ void adb_wa_sort_unique(struct adb_obj *arr)
}
/* Schema helpers */
int adb_s_field_by_name_blob(const struct adb_object_schema *schema, apk_blob_t blob)
{
for (int i = 0; i < schema->num_fields-1 && schema->fields[i].name; i++)
if (apk_blob_compare(APK_BLOB_STR(schema->fields[i].name), blob) == 0)
return i + 1;
return 0;
}
int adb_s_field_by_name(const struct adb_object_schema *schema, const char *name)
{
for (int i = 0; i < schema->num_fields; i++)
for (int i = 0; i < schema->num_fields-1 && schema->fields[i].name; i++)
if (strcmp(schema->fields[i].name, name) == 0)
return i + 1;
return 0;

View File

@ -172,6 +172,7 @@ struct adb_obj *adb_r_obj(struct adb *, adb_val_t, struct adb_obj *o, const stru
static inline uint32_t adb_ro_num(const struct adb_obj *o) { return o->num; }
static inline uint32_t adb_ra_num(const struct adb_obj *o) { return (o->num ?: 1) - 1; }
const uint8_t *adb_ro_kind(const struct adb_obj *o, unsigned i);
adb_val_t adb_ro_val(const struct adb_obj *o, unsigned i);
uint32_t adb_ro_int(const struct adb_obj *o, unsigned i);
apk_blob_t adb_ro_blob(const struct adb_obj *o, unsigned i);
@ -192,6 +193,7 @@ adb_val_t adb_w_fromstring(struct adb *, const uint8_t *kind, apk_blob_t);
#define adb_wo_alloca(o, schema, db) adb_wo_init(o, alloca(sizeof(adb_val_t[(schema)->num_fields])), schema, db)
struct adb_obj *adb_wo_init(struct adb_obj *, adb_val_t *, const struct adb_object_schema *, struct adb *);
struct adb_obj *adb_wo_init_val(struct adb_obj *, adb_val_t *, const struct adb_obj *, unsigned i);
void adb_wo_reset(struct adb_obj *);
void adb_wo_resetdb(struct adb_obj *);
adb_val_t adb_w_obj(struct adb_obj *);
@ -210,6 +212,7 @@ void adb_wa_sort(struct adb_obj *);
void adb_wa_sort_unique(struct adb_obj *);
/* Schema helpers */
int adb_s_field_by_name_blob(const struct adb_object_schema *schema, apk_blob_t blob);
int adb_s_field_by_name(const struct adb_object_schema *, const char *);
/* Creation */
@ -254,7 +257,7 @@ struct adb_walk_ops {
int (*scalar)(struct adb_walk *, apk_blob_t scalar, int multiline);
};
extern const struct adb_walk_ops adb_walk_gentext_ops;
extern const struct adb_walk_ops adb_walk_gentext_ops, adb_walk_genadb_ops;
struct adb_walk {
const struct adb_walk_ops *ops;
@ -269,6 +272,20 @@ struct adb_walk_gentext {
int key_printed : 1;
};
#define ADB_WALK_GENADB_MAX_NESTING 32
#define ADB_WALK_GENADB_MAX_VALUES 100000
struct adb_walk_genadb {
struct adb_walk d;
struct adb db;
struct adb idb[2];
int nest, nestdb, num_vals;
struct adb_obj objs[ADB_WALK_GENADB_MAX_NESTING];
unsigned int curkey[ADB_WALK_GENADB_MAX_NESTING];
adb_val_t vals[ADB_WALK_GENADB_MAX_VALUES];
};
int adb_walk_adb(struct adb_walk *d, struct adb *db, struct apk_trust *trust);
int adb_walk_istream(struct adb_walk *d, struct apk_istream *is);
#endif

View File

@ -51,10 +51,15 @@ static int dump_item(struct adb_walk_ctx *ctx, const char *name, const uint8_t *
d->ops->end(d);
ctx->db = origdb;
break;
case ADB_KIND_OBJECT:
d->ops->start_object(d);
dump_object(ctx, container_of(kind, struct adb_object_schema, kind), v);
d->ops->end(d);
case ADB_KIND_OBJECT:;
struct adb_object_schema *object = container_of(kind, struct adb_object_schema, kind);
if (!object->tostring) {
d->ops->start_object(d);
dump_object(ctx, object, v);
d->ops->end(d);
} else {
dump_object(ctx, object, v);
}
break;
case ADB_KIND_BLOB:
case ADB_KIND_INT:;
@ -162,5 +167,6 @@ int adb_walk_adb(struct adb_walk *d, struct adb *db, struct apk_trust *trust)
.db = db,
.trust = trust,
};
d->ops->schema(d, db->hdr.schema);
return dump_adb(&ctx);
}

138
src/adb_walk_genadb.c Normal file
View File

@ -0,0 +1,138 @@
#include <errno.h>
#include "adb.h"
#include "apk_print.h"
static int adb_walk_genadb_schema(struct adb_walk *d, uint32_t schema_id)
{
struct adb_walk_genadb *dt = container_of(d, struct adb_walk_genadb, d);
const struct adb_db_schema *s;
dt->db.hdr.schema = htole32(schema_id);
for (s = d->schemas; s->magic; s++)
if (s->magic == schema_id) break;
if (!s) return -EAPKDBFORMAT;
adb_wo_init(&dt->objs[0], &dt->vals[0], s->root, &dt->db);
dt->num_vals += s->root->num_fields;
if (dt->num_vals >= ARRAY_SIZE(dt->vals)) return -E2BIG;
dt->nest = 0;
return 0;
}
static int adb_walk_genadb_comment(struct adb_walk *d, apk_blob_t comment)
{
return 0;
}
static int adb_walk_genadb_start_object(struct adb_walk *d)
{
struct adb_walk_genadb *dt = container_of(d, struct adb_walk_genadb, d);
if (!dt->db.hdr.schema) return -EAPKDBFORMAT;
if (dt->nest >= ARRAY_SIZE(dt->objs)) return -EAPKDBFORMAT;
if (dt->curkey[dt->nest] == 0 &&
dt->objs[dt->nest].schema->kind == ADB_KIND_OBJECT)
return -EAPKDBFORMAT;
dt->nest++;
adb_wo_init_val(
&dt->objs[dt->nest], &dt->vals[dt->num_vals],
&dt->objs[dt->nest-1], dt->curkey[dt->nest-1]);
if (*adb_ro_kind(&dt->objs[dt->nest-1], dt->curkey[dt->nest-1]) == ADB_KIND_ADB) {
struct adb_adb_schema *schema = container_of(&dt->objs[dt->nest-1].schema->kind, struct adb_adb_schema, kind);
adb_reset(&dt->idb[dt->nestdb]);
dt->idb[dt->nestdb].hdr.schema = htole32(schema->schema_id);
dt->objs[dt->nest].db = &dt->idb[dt->nestdb];
dt->nestdb++;
}
dt->num_vals += dt->objs[dt->nest].schema->num_fields;
if (dt->num_vals >= ARRAY_SIZE(dt->vals)) return -E2BIG;
return 0;
}
static int adb_walk_genadb_start_array(struct adb_walk *d, unsigned int num)
{
return adb_walk_genadb_start_object(d);
}
static int adb_walk_genadb_end(struct adb_walk *d)
{
struct adb_walk_genadb *dt = container_of(d, struct adb_walk_genadb, d);
adb_val_t val;
val = adb_w_obj(&dt->objs[dt->nest]);
if (ADB_IS_ERROR(val))
return -ADB_VAL_VALUE(val);
dt->curkey[dt->nest] = 0;
dt->num_vals -= dt->objs[dt->nest].schema->num_fields;
if (dt->nest == 0) {
adb_w_root(&dt->db, val);
return 0;
}
dt->nest--;
if (*adb_ro_kind(&dt->objs[dt->nest], dt->curkey[dt->nest]) == ADB_KIND_ADB) {
dt->nestdb--;
adb_w_root(&dt->idb[dt->nestdb], val);
val = adb_w_adb(&dt->db, &dt->idb[dt->nestdb]);
}
if (dt->curkey[dt->nest] == 0) {
adb_wa_append(&dt->objs[dt->nest], val);
} else {
adb_wo_val(&dt->objs[dt->nest], dt->curkey[dt->nest], val);
dt->curkey[dt->nest] = 0;
}
return 0;
}
static int adb_walk_genadb_key(struct adb_walk *d, apk_blob_t key)
{
struct adb_walk_genadb *dt = container_of(d, struct adb_walk_genadb, d);
uint8_t kind = dt->objs[dt->nest].schema->kind;
if (kind != ADB_KIND_OBJECT && kind != ADB_KIND_ADB)
return -EAPKDBFORMAT;
dt->curkey[dt->nest] = adb_s_field_by_name_blob(dt->objs[dt->nest].schema, key);
if (dt->curkey[dt->nest] == 0)
return -EAPKDBFORMAT;
return 0;
}
static int adb_walk_genadb_scalar(struct adb_walk *d, apk_blob_t scalar, int multiline)
{
struct adb_walk_genadb *dt = container_of(d, struct adb_walk_genadb, d);
if (dt->objs[dt->nest].schema->kind == ADB_KIND_ARRAY) {
adb_wa_append_fromstring(&dt->objs[dt->nest], scalar);
} else {
if (dt->curkey[dt->nest] == 0)
adb_wo_fromstring(&dt->objs[dt->nest], scalar);
else
adb_wo_val_fromstring(&dt->objs[dt->nest], dt->curkey[dt->nest], scalar);
}
dt->curkey[dt->nest] = 0;
return 0;
}
const struct adb_walk_ops adb_walk_genadb_ops = {
.schema = adb_walk_genadb_schema,
.comment = adb_walk_genadb_comment,
.start_array = adb_walk_genadb_start_array,
.start_object = adb_walk_genadb_start_object,
.end = adb_walk_genadb_end,
.key = adb_walk_genadb_key,
.scalar = adb_walk_genadb_scalar,
};

View File

@ -111,6 +111,7 @@ static int adb_walk_gentext_scalar(struct adb_walk *d, apk_blob_t scalar, int mu
fprintf(out, "|\n");
adb_walk_gentext_newline(dt);
dt->nest++;
while (apk_blob_split(scalar, nl, &l, &scalar)) {
adb_walk_gentext_indent(dt);

131
src/adb_walk_istream.c Normal file
View File

@ -0,0 +1,131 @@
#include <errno.h>
#include "adb.h"
//#define DEBUG_PRINT
#ifdef DEBUG_PRINT
#include <stdio.h>
#define dbg_printf(args...) fprintf(stderr, args)
#else
#define dbg_printf(args...)
#endif
int adb_walk_istream(struct adb_walk *d, struct apk_istream *is)
{
const apk_blob_t token = APK_BLOB_STR("\n");
const apk_blob_t comment = APK_BLOB_STR(" #");
const apk_blob_t key_sep = APK_BLOB_STR(": ");
char mblockdata[1024*4];
apk_blob_t l, comm, mblock = APK_BLOB_BUF(mblockdata);
int r = 0, i, multi_line = 0, nesting = 0, new_item = 0;
uint8_t started[64] = {0};
if (IS_ERR_OR_NULL(is)) return PTR_ERR(is);
l = apk_istream_get_delim(is, token);
apk_blob_pull_blob_match(&l, APK_BLOB_STR("#%SCHEMA: "));
if ((r = d->ops->schema(d, apk_blob_pull_uint(&l, 16))) != 0) goto err;
started[0] = 1;
while (!APK_BLOB_IS_NULL(l = apk_istream_get_delim(is, token))) {
for (i = 0; l.len >= 2 && l.ptr[0] == ' ' && l.ptr[1] == ' '; i++, l.ptr += 2, l.len -= 2)
if (multi_line && i >= multi_line) break;
for (; nesting > i; nesting--) {
if (multi_line) {
apk_blob_t data = apk_blob_pushed(APK_BLOB_BUF(mblockdata), mblock);
if (APK_BLOB_IS_NULL(data)) {
r = -E2BIG;
goto err;
}
if (data.len && data.ptr[data.len-1] == '\n') data.len--;
dbg_printf("Multiline-Scalar >%d> "BLOB_FMT"\n", nesting, BLOB_PRINTF(data));
if ((r = d->ops->scalar(d, data, 1)) != 0) goto err;
mblock = APK_BLOB_BUF(mblockdata);
multi_line = 0;
}
if (started[nesting]) {
dbg_printf("End %d\n", nesting);
if ((r = d->ops->end(d)) != 0) goto err;
}
}
if (l.ptr[0] == '-' && l.ptr[1] == ' ') {
l.ptr += 2, l.len -= 2;
if (!started[nesting]) {
dbg_printf("Array %d\n", nesting);
if ((r = d->ops->start_array(d, 0)) != 0) goto err;
started[nesting] = 1;
}
new_item = 1;
}
dbg_printf(" >%d/%d< >"BLOB_FMT"<\n", nesting, i, BLOB_PRINTF(l));
if (multi_line) {
dbg_printf("Scalar-Block:>%d> "BLOB_FMT"\n", nesting, BLOB_PRINTF(l));
apk_blob_push_blob(&mblock, l);
apk_blob_push_blob(&mblock, APK_BLOB_STR("\n"));
new_item = 0;
continue;
}
if (l.ptr[0] == '#') {
if ((r = d->ops->comment(d, l)) != 0) goto err;
continue;
}
// contains ' #' -> comment
if (!apk_blob_split(l, comment, &l, &comm))
comm.len = 0;
if (l.len) {
apk_blob_t key = APK_BLOB_NULL, scalar = APK_BLOB_NULL;
int start = 0;
if (apk_blob_split(l, key_sep, &key, &scalar)) {
// contains ': ' -> key + scalar
} else if (l.ptr[l.len-1] == ':') {
// ends ':' -> key + indented object/array
key = APK_BLOB_PTR_LEN(l.ptr, l.len-1);
start = 1;
} else {
scalar = l;
}
if (key.len) {
if (new_item) {
started[++nesting] = 0;
dbg_printf("Array-Object %d\n", nesting);
}
if (!started[nesting]) {
dbg_printf("Object %d\n", nesting);
if ((r = d->ops->start_object(d)) != 0) goto err;
started[nesting] = 1;
}
dbg_printf("Key >%d> "BLOB_FMT"\n", nesting, BLOB_PRINTF(key));
if ((r = d->ops->key(d, key)) != 0) goto err;
if (start) started[++nesting] = 0;
}
if (scalar.len) {
if (scalar.ptr[0] == '|') {
dbg_printf("Scalar-block >%d> "BLOB_FMT"\n", nesting, BLOB_PRINTF(scalar));
// scalar '|' -> starts string literal block
started[++nesting] = 0;
multi_line = nesting;
} else {
dbg_printf("Scalar >%d> "BLOB_FMT"\n", nesting, BLOB_PRINTF(scalar));
if ((r = d->ops->scalar(d, scalar, 0)) != 0) goto err;
}
}
new_item = 0;
}
if (comm.len) {
if ((r = d->ops->comment(d, comm)) != 0) goto err;
}
//fprintf(stderr, ">%d> "BLOB_FMT"\n", indent, BLOB_PRINTF(l));
}
d->ops->end(d);
err:
if (r) apk_istream_error(is, r);
return apk_istream_close(is);
}

View File

@ -146,9 +146,27 @@ static apk_blob_t hexblob_tostring(struct adb *db, adb_val_t val, char *buf, siz
return APK_BLOB_PTR_LEN(buf, snprintf(buf, bufsz, "(%ld bytes)", b.len));
}
static adb_val_t hexblob_fromstring(struct adb *db, apk_blob_t val)
{
char buf[256];
if (val.len & 1)
return ADB_ERROR(EINVAL);
if (val.len > sizeof buf)
return ADB_ERROR(E2BIG);
apk_blob_t b = APK_BLOB_PTR_LEN(buf, val.len / 2);
apk_blob_pull_hexdump(&val, b);
if (APK_BLOB_IS_NULL(val))
return ADB_ERROR(EINVAL);
return adb_w_blob(db, b);
}
static struct adb_scalar_schema scalar_hexblob = {
.kind = ADB_KIND_BLOB,
.tostring = hexblob_tostring,
.fromstring = hexblob_fromstring,
};
static apk_blob_t int_tostring(struct adb *db, adb_val_t val, char *buf, size_t bufsz)
@ -184,9 +202,17 @@ static apk_blob_t oct_tostring(struct adb *db, adb_val_t val, char *buf, size_t
return APK_BLOB_PTR_LEN(buf, snprintf(buf, bufsz, "%o", adb_r_int(db, val)));
}
static adb_val_t oct_fromstring(struct adb *db, apk_blob_t val)
{
uint32_t n = apk_blob_pull_uint(&val, 8);
if (val.len) return ADB_ERROR(EINVAL);
return adb_w_int(db, n) ?: ADB_VAL_NULL;
}
static struct adb_scalar_schema scalar_oct = {
.kind = ADB_KIND_INT,
.tostring = oct_tostring,
.fromstring = oct_fromstring,
};
static apk_blob_t hsize_tostring(struct adb *db, adb_val_t val, char *buf, size_t bufsz)
@ -197,10 +223,23 @@ static apk_blob_t hsize_tostring(struct adb *db, adb_val_t val, char *buf, size_
return APK_BLOB_PTR_LEN(buf, snprintf(buf, bufsz, "%jd %s", (intmax_t)v, unit));
}
static adb_val_t hsize_fromstring(struct adb *db, apk_blob_t val)
{
apk_blob_t l, r;
if (!apk_blob_split(val, APK_BLOB_STR(" "), &l, &r))
return int_fromstring(db, val);
uint64_t n = apk_blob_pull_uint(&l, 10);
int sz = apk_get_human_size_unit(r);
n *= sz;
return adb_w_int(db, n);
}
static struct adb_scalar_schema scalar_hsize = {
.kind = ADB_KIND_INT,
.tostring = hsize_tostring,
.fromstring = int_fromstring,
.fromstring = hsize_fromstring,
.compare = int_compare,
};

View File

@ -97,6 +97,7 @@ unsigned long apk_blob_hash_seed(apk_blob_t, unsigned long seed);
unsigned long apk_blob_hash(apk_blob_t str);
int apk_blob_compare(apk_blob_t a, apk_blob_t b);
int apk_blob_sort(apk_blob_t a, apk_blob_t b);
int apk_blob_starts_with(apk_blob_t a, apk_blob_t b);
int apk_blob_ends_with(apk_blob_t str, apk_blob_t suffix);
int apk_blob_for_each_segment(apk_blob_t blob, const char *split,
apk_blob_cb cb, void *ctx);

View File

@ -81,6 +81,7 @@ struct apk_istream *apk_istream_from_file(int atfd, const char *file);
struct apk_istream *apk_istream_from_file_gz(int atfd, const char *file);
struct apk_istream *apk_istream_from_fd(int fd);
struct apk_istream *apk_istream_from_fd_url_if_modified(int atfd, const char *url, time_t since);
static inline int apk_istream_error(struct apk_istream *is, int err) { if (!is->err) is->err = err; return err; }
ssize_t apk_istream_read(struct apk_istream *is, void *ptr, size_t size);
apk_blob_t apk_istream_get(struct apk_istream *is, size_t len);
apk_blob_t apk_istream_get_max(struct apk_istream *is, size_t size);

View File

@ -13,6 +13,7 @@
#include "apk_blob.h"
const char *apk_error_str(int error);
int apk_get_human_size_unit(apk_blob_t b);
const char *apk_get_human_size(off_t size, off_t *dest);
struct apk_url_print {

View File

@ -53,3 +53,37 @@ static struct apk_applet apk_adbdump = {
};
APK_DEFINE_APPLET(apk_adbdump);
static int adbgen_main(void *pctx, struct apk_ctx *ac, struct apk_string_array *args)
{
struct apk_out *out = &ac->out;
char **arg;
int r;
struct adb_walk_genadb genadb = {
.d.ops = &adb_walk_genadb_ops,
.d.schemas = dbschemas,
};
adb_w_init_alloca(&genadb.db, 0, 1000);
adb_w_init_alloca(&genadb.idb[0], 0, 100);
foreach_array_item(arg, args) {
adb_reset(&genadb.db);
r = adb_walk_istream(&genadb.d, apk_istream_from_file(AT_FDCWD, *arg));
if (!r) {
r = adb_c_create(apk_ostream_to_fd(STDOUT_FILENO), &genadb.db,
apk_ctx_get_trust(ac));
}
adb_free(&genadb.db);
adb_free(&genadb.idb[0]);
if (r) apk_err(out, "%s: %s", *arg, apk_error_str(r));
}
return 0;
}
static struct apk_applet apk_adbgen = {
.name = "adbgen",
.main = adbgen_main,
};
APK_DEFINE_APPLET(apk_adbgen);

View File

@ -263,6 +263,12 @@ int apk_blob_sort(apk_blob_t a, apk_blob_t b)
return a.len - b.len;
}
int apk_blob_starts_with(apk_blob_t a, apk_blob_t b)
{
if (a.len < b.len) return 0;
return memcmp(a.ptr, b.ptr, b.len) == 0;
}
int apk_blob_ends_with(apk_blob_t a, apk_blob_t b)
{
if (a.len < b.len) return 0;

View File

@ -2,7 +2,9 @@ libapk_so_version = '2.99.0'
libapk_src = [
'adb.c',
'adb_walk_adb.c',
'adb_walk_genadb.c',
'adb_walk_gentext.c',
'adb_walk_istream.c',
'apk_adb.c',
'atom.c',
'blob.c',

View File

@ -65,9 +65,18 @@ const char *apk_error_str(int error)
}
}
static const char *size_units[] = {"B", "KiB", "MiB", "GiB", "TiB"};
int apk_get_human_size_unit(apk_blob_t b)
{
for (int i = 0, s = 1; i < ARRAY_SIZE(size_units); i++, s *= 1024)
if (apk_blob_compare(b, APK_BLOB_STR(size_units[i])) == 0)
return s;
return 1;
}
const char *apk_get_human_size(off_t size, off_t *dest)
{
static const char *size_units[] = {"B", "KiB", "MiB", "GiB", "TiB"};
size_t i;
off_t s;