Broken zfetch code for later reference.

src/main.zig

@@ -5,17 +5,30 @@ const curl = @cImport({
     @cInclude("curl/curl.h");
 });
 
+const zfetch = @import("zfetch");
+const tls = @import("iguanaTLS");
+const uri = @import("uri");
+const json = @import("json");
+
 const log = std.log.scoped(.derploader);
 
 const params = [_]clap.Param(clap.Help){
     clap.parseParam("-h, --help          Display this help and exit.") catch unreachable,
     clap.parseParam("-c, --create=<PATH> Create new database at PATH.") catch unreachable,
-    clap.parseParam("-i <ID>             Operate on ID.") catch unreachable,
+    clap.parseParam("-i <ID>...          Operate on ID.") catch unreachable,
     clap.parseParam("-a                  Operate on all IDs in the database.") catch unreachable,
+    clap.parseParam("-l <FILE>...        Operate on IDs from a file.") catch unreachable,
     clap.parseParam("-m                  Download metadata for ID image.") catch unreachable,
     clap.parseParam("-d                  Download image data of ID.") catch unreachable,
-    clap.parseParam("-e <PATH>           Extract image.") catch unreachable,
+    clap.parseParam("-e                  Extract image to local subfolder.") catch unreachable,
-    clap.parseParam("-t                  Time between requests.") catch unreachable,
+    clap.parseParam("-t <MILLIS>         Time between requests.") catch unreachable,
+    clap.parseParam("-r                  Register ID.") catch unreachable,
+    clap.parseParam("-k                  Include key in searches from environment variable DERPI_KEY.") catch unreachable,
+    clap.parseParam("-s <SEARCH>...      Iterate over the results of searches.") catch unreachable,
+    clap.parseParam("-o <ORDER>          Order searches by ORDER, descending.") catch unreachable,
+    clap.parseParam("-O <ORDER>          Order searches by ORDER, ascending.") catch unreachable,
+    clap.parseParam("-p <PAGE>           Start from page PAGE.") catch unreachable,
+    clap.parseParam("-P <PAGES>          Stop after PAGES pages.") catch unreachable,
 };
 
 fn printFullUsage(w: anytype) !void {
@@ -44,6 +57,8 @@ const create =
     \\    (json_extract(metadata, '$.image.representations.full')) VIRTUAL,
     \\  thumb_url TEXT GENERATED ALWAYS AS
     \\    (json_extract(metadata, '$.image.representations.thumb')) VIRTUAL,
+    \\  extension TEXT GENERATED ALWAYS AS
+    \\    (json_extract(metadata, '$.image.format')) VIRTUAL,
     \\  hash_full TEXT,
     \\  hash_thumb TEXT,
     \\  hash_meta TEXT
@@ -61,13 +76,13 @@ pub fn insertMeta(db: *sqlite.Db, id: u64, meta: []const u8) !void {
     const q =
         \\INSERT OR ROLLBACK INTO image (id, metadata) VALUES (?, ?);
     ;
-    try db.exec(q, .{ .id = id, .metadata = meta });
+    try db.exec(q, .{}, .{ .id = id, .metadata = meta });
 }
 
 const api_base = "https://derpibooru.org/api/v1/json";
 
 var urlbuf = [_:0]u8{0} ** 512;
-var curlerr = [_:0]u8{0} ** (curl.CURL_ERROR_SIZE + 1);
+var curlerr = [_:0]u8{0} ** (curl.CURL_ERROR_SIZE);
 
 const hash_prefix = "blake3-";
 var hash_buf = [_]u8{0} ** (std.crypto.hash.Blake3.digest_length);
@@ -82,6 +97,10 @@ fn hashit(input: []const u8) !void {
     );
 }
 
+var fetch_timer: ?std.time.Timer = null;
+var fetch_wait: u64 = 0;
+var first_fetch = true;
+
 pub fn main() anyerror!void {
     var gpa = std.heap.GeneralPurposeAllocator(.{}){};
     const alloc = &gpa.allocator;
@@ -116,7 +135,7 @@ pub fn main() anyerror!void {
         },
         .threading_mode = .Serialized,
     });
-    db.exec(create, .{}) catch sqliteErrorReport("Couldn't create table", &db);
+    db.exec(create, .{}, .{}) catch sqliteErrorReport("Couldn't create table", &db);
 
     var ret = curl.curl_global_init(curl.CURL_GLOBAL_ALL);
     if (ret != curl.CURLE_OK) {
@@ -131,175 +150,596 @@ pub fn main() anyerror!void {
 
     _ = curl.curl_easy_setopt(handle, curl.CURLOPT_ERRORBUFFER, &curlerr);
 
-    const maybe_id: ?u64 = if (args.option("-i")) |id_str| blk: {
+    const key = std.os.getenv("DERPI_KEY") orelse null;
-        break :blk std.fmt.parseInt(u64, id_str, 10) catch {
+
+    if (args.option("-t")) |millis_str| {
+        const millis = std.fmt.parseInt(u64, millis_str, 10) catch {
+            log.err("Fetch wait time must be a positive integer denoting some number of milliseconds.", .{});
+            return;
+        };
+        fetch_wait = millis;
+    }
+
+    for (args.options("-i")) |id_str| {
+        log.info("Iterating over all specified command line IDs.", .{});
+        const id = std.fmt.parseInt(u64, id_str, 10) catch {
             log.err("Image ID must be a positive integer.", .{});
-            return;
+            continue;
         };
-    } else null;
+        log.info("Operating on ID {d}, per -i.", .{id});
+        runActions(&db, id, &response_buffer, alloc, handle, &args, null);
+    }
 
-    if (args.flag("-m")) {
+    if (args.flag("-a")) {
-        const id = if (maybe_id) |id|
+        log.info("Iterating over all registered IDs.", .{});
-            id
+        var stmt = try db.prepare("SELECT id FROM image WHERE id IS NOT NULL");
-        else {
+        defer stmt.deinit();
-            log.err(
+        var iter = try stmt.iterator(u64, .{});
-                "Operation download metadata requires an ID (-i) argument.",
+        while (try iter.next(.{})) |id| {
-                .{},
+            log.info("Operating on ID {d}, per -a.", .{id});
-            );
+            runActions(&db, id, &response_buffer, alloc, handle, &args, null);
-            return;
-        };
-        const foobar = db.one(
-            bool,
-            "SELECT true FROM image WHERE id = ?",
-            .{},
-            .{ .id = id },
-        ) catch {
-            sqliteErrorReport("ID check read error", &db);
-            return;
-        };
-        if (foobar) |_| {
-            log.info("Info for id {d} already acquired.", .{id});
-            return;
         }
-        _ = try std.fmt.bufPrintZ(
+    }
-            urlbuf[0..],
-            api_base ++ "/images/{d}",
-            .{id},
-        );
-        easyFetch(handle, &urlbuf, &response_buffer) catch return;
-        //var w = std.io.getStdOut().writer();
-        const valid = std.json.validate(response_buffer.items);
 
-        if (valid) {
+    for (args.options("-l")) |path| {
-            try db.exec("BEGIN IMMEDIATE;", .{});
+        log.info("Iterating over IDs listed in {s}.", .{path});
-            errdefer db.exec("ROLLBACK;", .{}) catch {};
+        var file = std.fs.cwd().openFile(path, .{ .read = true }) catch |err| {
-            insertMeta(&db, id, response_buffer.items) catch {
+            log.err("Couldn't open file {s}: {}", .{ path, err });
-                sqliteErrorReport("Can't insert:", &db);
+            continue;
+        };
+        defer file.close();
+        var reader = file.reader();
+        var buffer: [128]u8 = undefined;
+        while (try reader.readUntilDelimiterOrEof(&buffer, '\n')) |id_str| {
+            const id = std.fmt.parseInt(u64, id_str, 10) catch {
+                log.err("Image ID must be a positive integer.", .{});
+                continue;
+            };
+            log.info("Operating on ID {d}, per -l.", .{id});
+            runActions(&db, id, &response_buffer, alloc, handle, &args, null);
+        }
+    }
+
+    const searches = args.options("-s");
+    if (searches.len > 0) {
+        //var reader = std.io.fixedBufferStream(
+        //    @embedFile("/etc/ssl/certs/ca-certificates.crt"),
+        //).reader();
+        //const trust = try tls.x509.CertificateChain.from_pem(alloc, reader);
+        // catch |a| {
+        //     log.err("Something dun fucked with the certs: {}", .{a});
+        //     return;
+        // };
+        const sort_ascending = args.option("-O");
+        const sort_descending = args.option("-o");
+        const sort_order = if (sort_ascending) |_| blk: {
+            if (sort_descending) |_| {
+                log.err("Can't sort up *and* down, dummy.", .{});
+                return;
+            } else {
+                break :blk "asc";
+            }
+        } else blk: {
+            if (sort_descending) |_| {
+                break :blk "desc";
+            } else {
+                break :blk "desc";
+            }
+        };
+        const sort_by = sort_descending orelse sort_ascending orelse "id";
+        var headers = zfetch.Headers.init(alloc);
+        try headers.appendValue("Accept", "application/json");
+        try headers.appendValue("User-Agent", "Derpiloader 0.1 (linux)");
+        var req = try zfetch.Request.init(
+            alloc,
+            "https://derpibooru.org",
+            null,
+        );
+        var page = if (args.option("-p")) |page| blk: {
+            break :blk std.fmt.parseInt(u64, page, 10) catch {
+                log.err("Page must be a positive integer.", .{});
                 return;
             };
-            try hashit(response_buffer.items);
+        } else 1;
+
+        var maxPages = if (args.option("-P")) |nr| blk: {
+            break :blk std.fmt.parseInt(u64, nr, 10) catch {
+                log.err("Pages maximum must be a positive integer.", .{});
+                return;
+            };
+        } else 0;
+
+        var buf = std.ArrayList(u8).init(alloc);
+        const kkey: []const u8 = key orelse "";
+        const aaaa: []const u8 = if (key) |_| "&key=" else "";
+        for (searches) |search| {
+            const esearch = try uri.escapeString(alloc, search);
+            var pages: u64 = 0;
+            log.info("Iterating over search \"{s}\", starting on page {d}.", .{ search, page });
+            while (true) foo: {
+                pages += 1;
+                if (maxPages > 0 and pages == maxPages) {
+                    return;
+                }
+                log.info("Doing page {d}, {d}/{d}.", .{ page, pages, maxPages });
+                buf.clearRetainingCapacity();
+                const url = try std.fmt.allocPrint(
+                    alloc,
+                    api_base ++ "/search/images?q={s}&page={d}&sd={s}&sf={s}&per_page=50{s}{s}",
+                    .{
+                        esearch, page, sort_order, sort_by, aaaa, kkey,
+                    },
+                );
+                try req.reset(url);
+                try req.do(.GET, headers, null);
+                const reader = req.reader();
+                try reader.readAllArrayList(&buf, 500 * 1024);
+                const val = try json.parse(alloc, buf.items);
+                if (val.get(.{"images"})) |aa| {
+                    if (unwrap(aa, .Array)) |images| {
+                        for (images) |i| {
+                            var buffer: [1024 * 10]u8 = undefined;
+                            const pid = unwrap(i.get("id") orelse {
+                                log.err("Malformed reply from Derpi.", .{});
+                                return;
+                            }, .Int) orelse {
+                                log.err("Malformed reply from Derpi, but in a different way.", .{});
+                                return;
+                            };
+                            const id = if (pid >= 0) @intCast(u64, pid) else {
+                                log.err("Malformed reply from Derpi, but in a third way.", .{});
+                                return;
+                            };
+                            var aaa = std.io.fixedBufferStream(buffer[0..]).writer();
+                            const data = json.Value{ .Object = &[_]json.Member{
+                                json.Member{
+                                    .key = "image",
+                                    .value = i,
+                                },
+                            } };
+                            try data.format("", .{}, aaa);
+                            const jason = buffer[0..aaa.context.pos];
+                            runActions(
+                                &db,
+                                id,
+                                &response_buffer,
+                                alloc,
+                                handle,
+                                &args,
+                                jason,
+                            );
+                        }
+                        if (images.len == 50) {
+                            page += 1;
+                        } else {
+                            break :foo;
+                        }
+                    }
+                }
+            }
+            page = 1;
+        }
+    }
+}
+
+pub fn unwrap(
+    un: anytype,
+    comptime tag: std.meta.Tag(@TypeOf(un)),
+) ?std.meta.TagPayload(@TypeOf(un), tag) {
+    if (un != tag) return null;
+    return @field(un, @tagName(tag));
+}
+
+fn runActions(
+    db: *sqlite.Db,
+    id: u64,
+    resp: *std.ArrayList(u8),
+    alloc: *std.mem.Allocator,
+    handle: *curl.CURL,
+    args: anytype,
+    meta: ?[]const u8,
+) void {
+    if (args.flag("-r")) {
+        registerID(db, id) catch |e| switch (e) {
+            error.GO_ON => {},
+            error.FATAL => {
+                db.deinit();
+                std.os.exit(1);
+            },
+            else => {
+                db.deinit();
+                std.os.exit(2);
+            },
+        };
+    }
+    if (args.flag("-m")) {
+        if (meta) |m| {
+            storeMetadata(db, id, m) catch |e| switch (e) {
+                error.GO_ON => {},
+                error.FATAL => {
+                    db.deinit();
+                    std.os.exit(1);
+                },
+                else => {
+                    db.deinit();
+                    std.os.exit(2);
+                },
+            };
+        } else {
+            getMetadata(db, id, resp, handle) catch |e| switch (e) {
+                error.GO_ON => {},
+                error.FATAL => {
+                    db.deinit();
+                    std.os.exit(1);
+                },
+                else => {
+                    db.deinit();
+                    std.os.exit(2);
+                },
+            };
+        }
+    }
+    resp.clearRetainingCapacity();
+    std.mem.set(u8, hash_buf[0..], 0);
+    std.mem.set(u8, hash_buf2[0..], 0);
+    if (args.flag("-d")) {
+        getImage(db, id, resp, alloc, handle) catch |e| switch (e) {
+            error.GO_ON => {},
+            error.FATAL => {
+                db.deinit();
+                std.os.exit(1);
+            },
+            else => {
+                db.deinit();
+                std.os.exit(2);
+            },
+        };
+    }
+    resp.clearRetainingCapacity();
+    std.mem.set(u8, hash_buf[0..], 0);
+    std.mem.set(u8, hash_buf2[0..], 0);
+    if (args.flag("-e")) {
+        extractImage(db, id, alloc) catch |e| switch (e) {
+            error.GO_ON => {},
+            else => {
+                db.deinit();
+                std.os.exit(2);
+            },
+        };
+    }
+}
+
+fn registerID(db: *sqlite.Db, id: u64) !void {
+    log.info("Registering ID {d}.", .{id});
+    const foo = db.one(
+        bool,
+        "SELECT true FROM image WHERE id = ?;",
+        .{},
+        .{ .id = id },
+    ) catch {
+        sqliteErrorReport("SQLite error while checking if ID already present", db);
+        return error.GO_ON;
+    };
+    if (foo) |_| {
+        log.info("ID {d} already registered.", .{id});
+        return;
+    }
+    try db.exec("BEGIN IMMEDIATE", .{}, .{});
+    errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
+    db.exec(
+        \\INSERT OR ROLLBACK
+        \\  INTO image (id)
+        \\  VALUES (?);
+    , .{}, .{ .id = id }) catch {
+        sqliteErrorReport("Couldn't insert ID into database.", db);
+        return error.GO_ON;
+    };
+
+    db.exec("COMMIT", .{}, .{}) catch {
+        sqliteErrorReport("FATAL: couldn't commit database", db);
+        return error.FATAL;
+    };
+}
+
+fn storeMetadata(
+    db: *sqlite.Db,
+    id: u64,
+    metadata: []const u8,
+) !void {
+    log.info("Storing metadata for ID {d}.", .{id});
+    const foobar = db.one(
+        bool,
+        "SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;",
+        .{},
+        .{ .id = id },
+    ) catch {
+        sqliteErrorReport("SQLite error while checking for metadata precence.", db);
+        return error.GO_ON;
+    };
+    if (foobar) |_| {
+        log.info("Metadata for ID {d} already acquired. Use -u to replace.", .{id});
+        return;
+    }
+    const valid = std.json.validate(metadata);
+
+    if (valid) {
+        try db.exec("BEGIN IMMEDIATE;", .{}, .{});
+        errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
+        db.exec(
+            \\INSERT OR ROLLBACK
+            \\  INTO
+            \\    image (id, metadata)
+            \\    VALUES (?, ?)
+            \\  ON CONFLICT (id)
+            \\    DO UPDATE
+            \\    SET metadata=excluded.metadata;
+        , .{}, .{ .id = id, .metadata = metadata }) catch {
+            sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db);
+            return error.GO_ON;
+        };
+        hashit(metadata) catch |err| {
+            log.err("Couldn't hash metadata for ID {d}: {s}", .{ id, err });
+            return error.GO_ON;
+        };
+        db.exec(
+            "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
+            .{},
+            .{ hash_buf2[0..], id },
+        ) catch {
+            sqliteErrorReport("Couldn't set metadata hash", db);
+            return error.GO_ON;
+        };
+        db.exec("COMMIT", .{}, .{}) catch {
+            sqliteErrorReport("FATAL: couldn't commit database", db);
+            return error.FATAL;
+        };
+    }
+}
+
+fn getMetadata(
+    db: *sqlite.Db,
+    id: u64,
+    resp: *std.ArrayList(u8),
+    handle: *curl.CURL,
+) !void {
+    log.info("Downloading metadata for ID {d}.", .{id});
+    const foobar = db.one(
+        bool,
+        "SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;",
+        .{},
+        .{ .id = id },
+    ) catch {
+        sqliteErrorReport("SQLite error while checking for metadata precence.", db);
+        return error.GO_ON;
+    };
+    if (foobar) |_| {
+        log.info("Metadata for ID {d} already acquired. Use -u to replace.", .{id});
+        return;
+    }
+    _ = try std.fmt.bufPrintZ(
+        urlbuf[0..],
+        api_base ++ "/images/{d}",
+        .{id},
+    );
+    easyFetch(handle, &urlbuf, resp) catch {
+        log.info("Failed to download metadata for ID {d}.", .{id});
+        return error.GO_ON;
+    };
+    const valid = std.json.validate(resp.items);
+
+    if (valid) {
+        try db.exec("BEGIN IMMEDIATE;", .{}, .{});
+        errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
+        db.exec(
+            \\INSERT OR ROLLBACK
+            \\  INTO
+            \\    image (id, metadata)
+            \\    VALUES (?, ?)
+            \\  ON CONFLICT (id)
+            \\    DO UPDATE
+            \\    SET metadata=excluded.metadata;
+        , .{}, .{ .id = id, .metadata = resp.items }) catch {
+            sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db);
+            return error.GO_ON;
+        };
+        hashit(resp.items) catch |err| {
+            log.err("Couldn't hash metadata for ID {d}: {s}", .{ id, err });
+            return error.GO_ON;
+        };
+        db.exec(
+            "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
+            .{},
+            .{ hash_buf2[0..], id },
+        ) catch {
+            sqliteErrorReport("Couldn't set metadata hash", db);
+            return error.GO_ON;
+        };
+        db.exec("COMMIT", .{}, .{}) catch {
+            sqliteErrorReport("FATAL: couldn't commit database", db);
+            return error.FATAL;
+        };
+    } else {
+        log.err("Invalid metadata for ID {d}", .{id});
+        return error.FATAL;
+    }
+}
+
+fn getImage(
+    db: *sqlite.Db,
+    id: u64,
+    resp: *std.ArrayList(u8),
+    alloc: *std.mem.Allocator,
+    handle: *curl.CURL,
+) !void {
+    log.info("Downloading image and thumbnail for ID {d}.", .{id});
+    const foobar = db.oneAlloc(
+        struct {
+            full_url: ?[:0]u8,
+            thumb_url: ?[:0]u8,
+        },
+        alloc,
+        "SELECT full_url, thumb_url FROM image WHERE id = ?",
+        .{},
+        .{ .id = id },
+    ) catch {
+        sqliteErrorReport("SQLite error while getting image URLs", db);
+        return error.GO_ON;
+    };
+    if (foobar) |res| {
+        if (res.full_url) |url| blk: {
+            defer alloc.free(url);
+            const skipper = db.one(bool,
+                \\SELECT true FROM image
+                \\  WHERE id = ? AND image IS NOT NULL;
+            , .{}, .{id}) catch {
+                sqliteErrorReport("SQLite error while checking if image is already downloaded", db);
+                return error.GO_ON;
+            };
+            if (skipper) |_| {
+                log.info("Image for ID {d} already downloaded.", .{id});
+                break :blk;
+            }
+            easyFetch(handle, url, resp) catch {
+                log.info("Failed to download fullsize image for ID {d}", .{id});
+                return error.FATAL;
+            };
+            try db.exec("BEGIN IMMEDIATE;", .{}, .{});
+            errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
             db.exec(
-                "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
+                "UPDATE OR ROLLBACK image SET image = ? WHERE id = ?",
+                .{},
+                .{
+                    .image = resp.items,
+                    .id = id,
+                },
+            ) catch {
+                sqliteErrorReport("Couldn't add image to DB.", db);
+                return error.GO_ON;
+            };
+            hashit(resp.items) catch |err| {
+                log.err("Couldn't hash image for ID {d}: {s}", .{ id, err });
+                return error.GO_ON;
+            };
+            db.exec(
+                "UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?",
+                .{},
                 .{ hash_buf2[0..], id },
             ) catch {
-                sqliteErrorReport("Couldn't insert", &db);
+                sqliteErrorReport("Couldn't set iamge hash", db);
-                return;
+                return error.GO_ON;
             };
-            try db.exec("COMMIT", .{});
+            db.exec("COMMIT", .{}, .{}) catch {
+                sqliteErrorReport("FATAL: couldn't commit database", db);
+                return error.FATAL;
+            };
+            resp.clearRetainingCapacity();
+            std.mem.set(u8, hash_buf[0..], 0);
+            std.mem.set(u8, hash_buf2[0..], 0);
+        }
+        if (res.thumb_url) |url| blk: {
+            defer alloc.free(url);
+            const skipper = db.one(bool,
+                \\SELECT true FROM image
+                \\  WHERE id = ? AND thumb IS NOT NULL;
+            , .{}, .{id}) catch {
+                sqliteErrorReport("SQLite error while checking if thumb is already downloaded", db);
+                return error.GO_ON;
+            };
+            if (skipper) |_| {
+                log.info("Thumb for ID {d} already downloaded.", .{id});
+                break :blk;
+            }
+            easyFetch(handle, url, resp) catch {
+                log.info("Failed to download thumbnail image for ID {d}", .{id});
+                return error.GO_ON;
+            };
+            try db.exec("BEGIN IMMEDIATE;", .{}, .{});
+            errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
+            db.exec(
+                "UPDATE OR ROLLBACK image SET thumb = ? WHERE id = ?",
+                .{},
+                .{
+                    .thumb = resp.items,
+                    .id = id,
+                },
+            ) catch {
+                sqliteErrorReport("Couldn't add thumb to DB", db);
+                return error.GO_ON;
+            };
+            hashit(resp.items) catch |err| {
+                log.err("Couldn't hash thumb for ID {d}: {s}", .{ id, err });
+                return error.GO_ON;
+            };
+            db.exec(
+                "UPDATE OR ROLLBACK image SET hash_thumb = ? WHERE id = ?",
+                .{},
+                .{ hash_buf2[0..], id },
+            ) catch {
+                sqliteErrorReport("Couldn't add thumb hash", db);
+                return error.GO_ON;
+            };
+            db.exec("COMMIT", .{}, .{}) catch {
+                sqliteErrorReport("FATAL: couldn't commit database", db);
+                return error.FATAL;
+            };
+        }
+    } else {
+        log.err("No metadata for id {d} available", .{id});
+        return;
+    }
+}
+
+fn extractImage(db: *sqlite.Db, id: u64, alloc: *std.mem.Allocator) !void {
+    log.info("Extracting image for ID {d}.", .{id});
+    const foo = db.oneAlloc(
+        struct {
+            image: ?[:0]u8,
+            extension: ?[:0]u8,
+        },
+        alloc,
+        "SELECT image, extension FROM image WHERE id = ?",
+        .{},
+        .{ .id = id },
+    ) catch {
+        sqliteErrorReport("SQLite error while reading image", db);
+        return error.GO_ON;
+    };
+    defer {
+        if (foo) |f| {
+            if (f.image) |i| {
+                alloc.free(i);
+            }
+            if (f.extension) |e| {
+                alloc.free(e);
+            }
         }
     }
-
+    if (foo) |res| {
-    if (args.flag("-d")) {
+        const bar = comptime @as([]const u8, "unknown");
-        const id = if (maybe_id) |id|
+        const baz = if (res.extension) |e| e else bar;
-            id
+        var buf = [_]u8{0} ** 64;
-        else {
+        const buf2 = try std.fmt.bufPrint(
-            log.err(
+            buf[0..],
-                "Operation download image requires an ID (-i) argument.",
+            "{d:0>10}.{s}",
-                .{},
+            .{
-            );
+                id,
-            return;
+                baz,
-        };
-        const foobar = db.oneAlloc(
-            struct {
-                full_url: ?[:0]u8,
-                thumb_url: ?[:0]u8,
             },
-            alloc,
+        );
-            "SELECT full_url, thumb_url FROM image WHERE id = ?",
+        if (res.image) |image| {
-            .{},
+            var dir = try std.fs.cwd().makeOpenPath("images", .{
-            .{ .id = id },
+                .access_sub_paths = true,
-        ) catch {
+            });
-            sqliteErrorReport("ID check read error", &db);
+            var file = try dir.createFile(
-            return;
+                buf2,
-        };
-        if (foobar) |res| {
-            if (res.full_url) |url| {
-                easyFetch(handle, url, &response_buffer) catch return;
-                try db.exec("BEGIN IMMEDIATE;", .{});
-                errdefer db.exec("ROLLBACK;", .{}) catch {};
-                db.exec(
-                    "UPDATE OR ROLLBACK image SET image = ? WHERE id = ?",
-                    .{
-                        .image = response_buffer.items,
-                        .id = id,
-                    },
-                ) catch {
-                    sqliteErrorReport("Couldn't add image to DB.", &db);
-                    return;
-                };
-                try hashit(response_buffer.items);
-                db.exec(
-                    "UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?",
-                    .{ hash_buf2[0..], id },
-                ) catch {
-                    sqliteErrorReport("Couldn't add iamge hash", &db);
-                    return;
-                };
-                try db.exec("COMMIT", .{});
-                response_buffer.clearRetainingCapacity();
-                std.mem.set(u8, hash_buf[0..], 0);
-                std.mem.set(u8, hash_buf2[0..], 0);
-            }
-            if (res.thumb_url) |url| {
-                easyFetch(handle, url, &response_buffer) catch return;
-                try db.exec("BEGIN IMMEDIATE;", .{});
-                errdefer db.exec("ROLLBACK;", .{}) catch {};
-                db.exec(
-                    "UPDATE OR ROLLBACK image SET thumb = ? WHERE id = ?",
-                    .{
-                        .thumb = response_buffer.items,
-                        .id = id,
-                    },
-                ) catch {
-                    sqliteErrorReport("Couldn't add thumb to DB", &db);
-                    return;
-                };
-                try hashit(response_buffer.items);
-                db.exec(
-                    "UPDATE OR ROLLBACK image SET hash_thumb = ? WHERE id = ?",
-                    .{ hash_buf2[0..], id },
-                ) catch {
-                    sqliteErrorReport("Couldn't add thumb hash", &db);
-                    return;
-                };
-                try db.exec("COMMIT", .{});
-            }
-        } else {
-            log.err("No metadata for id {d} available", .{id});
-            return;
-        }
-    }
-
-    if (args.option("-e")) |path| {
-        const id = if (maybe_id) |id|
-            id
-        else {
-            log.err(
-                "Operation extract image requires an ID (-i) argument.",
-                .{},
-            );
-            return;
-        };
-        const maybe_image = db.oneAlloc(
-            []u8,
-            alloc,
-            "SELECT image FROM image WHERE id = ?",
-            .{},
-            .{ .id = id },
-        ) catch {
-            sqliteErrorReport("ID check read error", &db);
-            return;
-        };
-        if (maybe_image) |image| {
-            var file = try std.fs.cwd().createFile(
-                path,
                 .{
                     .read = false,
                     .truncate = true,
                 },
             );
+            defer file.close();
             try file.writeAll(image);
-            file.close();
+            log.info("Extracted image for ID {d}.", .{id});
         } else {
             log.info("No image data for ID {d}.", .{id});
         }
@@ -307,6 +747,17 @@ pub fn main() anyerror!void {
 }
 
 fn easyFetch(handle: *curl.CURL, url: [*:0]const u8, resp: *std.ArrayList(u8)) !void {
+    if (fetch_wait > 0) {
+        if (fetch_timer) |*timer| {
+            const cur = timer.read() / (1000 * 1000);
+            if (cur < fetch_wait) {
+                std.time.sleep((fetch_wait - cur) * 1000 * 1000);
+            }
+            timer.reset();
+        } else {
+            fetch_timer = try std.time.Timer.start();
+        }
+    }
     var ret = curl.curl_easy_setopt(handle, curl.CURLOPT_URL, url);
     if (ret != curl.CURLE_OK) {
         curlErrorReport("cURL set url:", ret);
@@ -332,6 +783,7 @@ fn easyFetch(handle: *curl.CURL, url: [*:0]const u8, resp: *std.ArrayList(u8)) !
         curlErrorReport("cURL perform:", ret);
         return error.FUCK;
     }
+
     log.info("Got {d} bytes", .{resp.items.len});
 }
 

zig.mod

@@ -2,7 +2,11 @@ id: 1m1xcon88qpj9vi17d8yo2743g0pqswk91ep21b0v139bu5s
 name: derploader
 main: src/main.zig
 dev_dependencies:
-    - src: git https://github.com/Hejsil/zig-clap branch-zig-master
+  - src: git https://github.com/Hejsil/zig-clap branch-zig-master
-    - src: git https://github.com/vrischmann/zig-sqlite
+  - src: git https://github.com/vrischmann/zig-sqlite
-      c_source_flags:
+    c_source_flags:
-          - -DSQLITE_ENABLE_JSON1
+      - -DSQLITE_ENABLE_JSON1
+  - src: git https://github.com/nektro/zig-json
+  - src: git https://github.com/truemedian/zfetch
+  - src: git https://github.com/alexnask/iguanaTLS
+  - src: git https://github.com/MasterQ32/zig-uri/

zigmod.lock

@@ -1,3 +1,9 @@
 2
 git https://github.com/Hejsil/zig-clap commit-ed90e560d9b1144a27562ef62ff7686eb3af029a
-git https://github.com/vrischmann/zig-sqlite commit-fafe666f22590faf82d7c7ce1087858002c004b6
+git https://github.com/vrischmann/zig-sqlite commit-1adb900dcc816a419a4b67ccae0555ffe33f72c4
+git https://github.com/nektro/zig-json commit-72e555fbc0776f2600aee19b01e5ab1855ebec7a
+git https://github.com/truemedian/zfetch commit-8bbc7b34cd417794841e1432585334bc969dfe83
+git https://github.com/truemedian/hzzp commit-2d30bddae3bf1eaecde5144490307604efe76f2a
+git https://github.com/alexnask/iguanaTLS commit-0d39a361639ad5469f8e4dcdaea35446bbe54b48
+git https://github.com/MasterQ32/zig-network commit-b9c91769d8ebd626c8e45b2abb05cbc28ccc50da
+git https://github.com/MasterQ32/zig-uri commit-52cdd2061bec0579519f0d30280597f3a1db8b75

zigmod.sum

@@ -1,2 +1,8 @@
 blake3-e59408623c62ac2b958cf33f848cb9dccc87631e8ca98e479cf9870694a4771d v/git/github.com/Hejsil/zig-clap/branch-zig-master
-blake3-605156fee0fdf3be71878c38e7bb1e1934ba4bafeaf5fde535685bfa924345ea git/github.com/vrischmann/zig-sqlite
+blake3-ce341d796da35b5db8663df0a35cbce6c9cc39ead510e805ea8a04593731b303 git/github.com/vrischmann/zig-sqlite
+blake3-1893709ffc6359c5f9cd2f9409abccf78a94ed37bb2c6dd075c603356d17c94b git/github.com/nektro/zig-json
+blake3-e3072f7fb47e86d53c9a1879e254ba1af55941153fd5f6752ec659b2f14854c9 git/github.com/truemedian/zfetch
+blake3-98982125d0fbedc62e179e62081d2797a2b8a3623c42f9fd5d72cd56d6350714 git/github.com/truemedian/hzzp
+blake3-e6901bd7432450d5b22b01880cc7fa3fa2433e766a527206f18b29c67c1349bb git/github.com/alexnask/iguanaTLS
+blake3-21f91e48333ac0ca7f4704c96352831c25216e7056d02ce24de95d03fc942246 git/github.com/MasterQ32/zig-network
+blake3-030ebb03f1ed21122e681b06786bea6f2f1b810e8eb9f2029d0eee4f4fb3103f git/github.com/MasterQ32/zig-uri