Whoa, signed!

build.zig

@@ -30,6 +30,7 @@ pub fn build(b: *std.build.Builder) void {
     }
     deps.addAllTo(exe);
     exe.linkLibC();
+    exe.single_threaded = true;
     exe.linkSystemLibrary("libcurl");
     exe.setTarget(target);
     exe.setBuildMode(mode);

src/main.zig

@@ -29,6 +29,8 @@ const params = [_]clap.Param(clap.Help){
     clap.parseParam("-O <ORDER>          Order searches by ORDER, ascending.") catch unreachable,
     clap.parseParam("-p <PAGE>           Start from page PAGE.") catch unreachable,
     clap.parseParam("-P <PAGES>          Stop after PAGES pages.") catch unreachable,
+    clap.parseParam("--migrate           Update an old database.") catch unreachable,
+    clap.parseParam("--yolo              PRAGMA synchronous = OFF") catch unreachable,
 };
 
 fn printFullUsage(w: anytype) !void {
@@ -49,19 +51,18 @@ fn curlErrorReport(str: []const u8, code: curl.CURLcode) void {
 
 const create =
     \\CREATE TABLE IF NOT EXISTS image(
-    \\  id INTEGER UNIQUE,
+    \\  iid INTEGER PRIMARY KEY,
+    \\  eid INTEGER UNIQUE,
     \\  metadata TEXT,
-    \\  image BLOB,
-    \\  thumb BLOB,
     \\  full_url TEXT GENERATED ALWAYS AS
     \\    (json_extract(metadata, '$.image.representations.full')) VIRTUAL,
     \\  thumb_url TEXT GENERATED ALWAYS AS
     \\    (json_extract(metadata, '$.image.representations.thumb')) VIRTUAL,
     \\  extension TEXT GENERATED ALWAYS AS
     \\    (json_extract(metadata, '$.image.format')) VIRTUAL,
-    \\  hash_full TEXT,
+    \\  hash_meta TEXT,
-    \\  hash_thumb TEXT,
+    \\  image_id INTEGER,
-    \\  hash_meta TEXT
+    \\  thumb_id INTEGER
     \\);
 ;
 
@@ -125,18 +126,50 @@ pub fn main() anyerror!void {
         return;
     }
 
-    var db: sqlite.Db = undefined;
     const filename = "test.db3";
-    try db.init(.{
+    var db = try sqlite.Db.init(.{
         .mode = sqlite.Db.Mode{ .File = filename },
         .open_flags = .{
             .write = true,
             .create = true,
         },
-        .threading_mode = .Serialized,
+        .threading_mode = .SingleThread,
     });
     db.exec(create, .{}, .{}) catch sqliteErrorReport("Couldn't create table", &db);
 
+    if (args.flag("--migrate")) {
+        if (args.flag("--yolo")) try db.exec("PRAGMA synchronous=0;", .{}, .{});
+        try db.exec("BEGIN IMMEDIATE;", .{}, .{});
+        errdefer db.exec("ROLLBACK;", .{}, .{}) catch std.debug.panic("SQLite database errored trying to roll back. Have fun!", .{});
+        var stmt = try db.prepare("SELECT ROWID FROM image;");
+        defer stmt.deinit();
+        var iter = try stmt.iterator(u64, .{});
+        while (try iter.next(.{})) |id| {
+            try db.exec(
+                \\INSERT INTO blob (data, hash)
+                \\  SELECT image, hash_full
+                \\  FROM image WHERE ROWID = ?;
+            , .{}, .{id});
+            try db.exec(
+                \\UPDATE image
+                \\  SET image_id = last_insert_rowid()
+                \\  WHERE ROWID = ?
+            , .{}, .{id});
+            try db.exec(
+                \\INSERT INTO blob (data, hash)
+                \\  SELECT thumb, hash_thumb
+                \\  FROM image WHERE ROWID = ?;
+            , .{}, .{id});
+            try db.exec(
+                \\UPDATE image
+                \\  SET thumb_id = last_insert_rowid()
+                \\  WHERE ROWID = ?
+            , .{}, .{id});
+        }
+        db.exec("COMMIT;", .{}, .{}) catch std.debug.panic("SQLite database errored trying to commit. Not *terrible*, I guess.", .{});
+        return;
+    }
+
     var ret = curl.curl_global_init(curl.CURL_GLOBAL_ALL);
     if (ret != curl.CURLE_OK) {
         log.err("cURL global init failure: {s}", .{curl.curl_easy_strerror(ret)});
@@ -227,14 +260,6 @@ pub fn main() anyerror!void {
             }
         };
         const sort_by = sort_descending orelse sort_ascending orelse "id";
-        var headers = zfetch.Headers.init(alloc);
-        try headers.appendValue("Accept", "application/json");
-        try headers.appendValue("User-Agent", "Derpiloader 0.1 (linux)");
-        var req = try zfetch.Request.init(
-            alloc,
-            "https://derpibooru.org",
-            null,
-        );
         var page = if (args.option("-p")) |page| blk: {
             break :blk std.fmt.parseInt(u64, page, 10) catch {
                 log.err("Page must be a positive integer.", .{});
@@ -249,36 +274,34 @@ pub fn main() anyerror!void {
             };
         } else 0;
 
-        var buf = std.ArrayList(u8).init(alloc);
         const kkey: []const u8 = key orelse "";
         const aaaa: []const u8 = if (key) |_| "&key=" else "";
+        var fuckme = std.ArrayList(u8).init(alloc);
+        defer fuckme.deinit();
         for (searches) |search| {
             const esearch = try uri.escapeString(alloc, search);
             var pages: u64 = 0;
             log.info("Iterating over search \"{s}\", starting on page {d}.", .{ search, page });
-            while (true) foo: {
+            fuck: while (true) {
                 pages += 1;
                 if (maxPages > 0 and pages == maxPages) {
                     return;
                 }
                 log.info("Doing page {d}, {d}/{d}.", .{ page, pages, maxPages });
-                buf.clearRetainingCapacity();
+                _ = try std.fmt.bufPrintZ(
-                const url = try std.fmt.allocPrint(
+                    urlbuf[0..],
-                    alloc,
                     api_base ++ "/search/images?q={s}&page={d}&sd={s}&sf={s}&per_page=50{s}{s}",
                     .{
                         esearch, page, sort_order, sort_by, aaaa, kkey,
                     },
                 );
-                try req.reset(url);
+                fuckme.clearRetainingCapacity();
-                try req.do(.GET, headers, null);
+                try easyFetch(handle, &urlbuf, &fuckme);
-                const reader = req.reader();
+                const val = try json.parse(alloc, fuckme.items);
-                try reader.readAllArrayList(&buf, 500 * 1024);
-                const val = try json.parse(alloc, buf.items);
                 if (val.get(.{"images"})) |aa| {
                     if (unwrap(aa, .Array)) |images| {
                         for (images) |i| {
-                            var buffer: [1024 * 10]u8 = undefined;
+                            var buffer: [1024 * 20]u8 = undefined;
                             const pid = unwrap(i.get("id") orelse {
                                 log.err("Malformed reply from Derpi.", .{});
                                 return;
@@ -312,7 +335,7 @@ pub fn main() anyerror!void {
                         if (images.len == 50) {
                             page += 1;
                         } else {
-                            break :foo;
+                            break :fuck;
                         }
                     }
                 }
@@ -413,9 +436,9 @@ fn registerID(db: *sqlite.Db, id: u64) !void {
     log.info("Registering ID {d}.", .{id});
     const foo = db.one(
         bool,
-        "SELECT true FROM image WHERE id = ?;",
+        "SELECT true FROM image WHERE eid = ?;",
         .{},
-        .{ .id = id },
+        .{ .eid = id },
     ) catch {
         sqliteErrorReport("SQLite error while checking if ID already present", db);
         return error.GO_ON;
@@ -428,7 +451,7 @@ fn registerID(db: *sqlite.Db, id: u64) !void {
     errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
     db.exec(
         \\INSERT OR ROLLBACK
-        \\  INTO image (id)
+        \\  INTO image (eid)
         \\  VALUES (?);
     , .{}, .{ .id = id }) catch {
         sqliteErrorReport("Couldn't insert ID into database.", db);
@@ -449,9 +472,9 @@ fn storeMetadata(
     log.info("Storing metadata for ID {d}.", .{id});
     const foobar = db.one(
         bool,
-        "SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;",
+        "SELECT true FROM image WHERE eid = ? AND metadata IS NOT NULL;",
         .{},
-        .{ .id = id },
+        .{ .eid = id },
     ) catch {
         sqliteErrorReport("SQLite error while checking for metadata precence.", db);
         return error.GO_ON;
@@ -468,12 +491,12 @@ fn storeMetadata(
         db.exec(
             \\INSERT OR ROLLBACK
             \\  INTO
-            \\    image (id, metadata)
+            \\    image (eid, metadata)
             \\    VALUES (?, ?)
-            \\  ON CONFLICT (id)
+            \\  ON CONFLICT (eid)
             \\    DO UPDATE
             \\    SET metadata=excluded.metadata;
-        , .{}, .{ .id = id, .metadata = metadata }) catch {
+        , .{}, .{ .eid = id, .metadata = metadata }) catch {
             sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db);
             return error.GO_ON;
         };
@@ -482,7 +505,7 @@ fn storeMetadata(
             return error.GO_ON;
         };
         db.exec(
-            "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
+            "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE eid = ?",
             .{},
             .{ hash_buf2[0..], id },
         ) catch {
@@ -505,9 +528,9 @@ fn getMetadata(
     log.info("Downloading metadata for ID {d}.", .{id});
     const foobar = db.one(
         bool,
-        "SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;",
+        "SELECT true FROM image WHERE eid = ? AND metadata IS NOT NULL;",
         .{},
-        .{ .id = id },
+        .{ .eid = id },
     ) catch {
         sqliteErrorReport("SQLite error while checking for metadata precence.", db);
         return error.GO_ON;
@@ -523,7 +546,7 @@ fn getMetadata(
     );
     easyFetch(handle, &urlbuf, resp) catch {
         log.info("Failed to download metadata for ID {d}.", .{id});
-        return error.GO_ON;
+        return error.FATAL;
     };
     const valid = std.json.validate(resp.items);
 
@@ -533,12 +556,12 @@ fn getMetadata(
         db.exec(
             \\INSERT OR ROLLBACK
             \\  INTO
-            \\    image (id, metadata)
+            \\    image (eid, metadata)
             \\    VALUES (?, ?)
-            \\  ON CONFLICT (id)
+            \\  ON CONFLICT (eid)
             \\    DO UPDATE
             \\    SET metadata=excluded.metadata;
-        , .{}, .{ .id = id, .metadata = resp.items }) catch {
+        , .{}, .{ .eid = id, .metadata = resp.items }) catch {
             sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db);
             return error.GO_ON;
         };
@@ -547,7 +570,7 @@ fn getMetadata(
             return error.GO_ON;
         };
         db.exec(
-            "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
+            "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE eid = ?",
             .{},
             .{ hash_buf2[0..], id },
         ) catch {
@@ -578,9 +601,9 @@ fn getImage(
             thumb_url: ?[:0]u8,
         },
         alloc,
-        "SELECT full_url, thumb_url FROM image WHERE id = ?",
+        "SELECT full_url, thumb_url FROM image WHERE eid = ?",
         .{},
-        .{ .id = id },
+        .{ .eid = id },
     ) catch {
         sqliteErrorReport("SQLite error while getting image URLs", db);
         return error.GO_ON;
@@ -590,7 +613,54 @@ fn getImage(
             defer alloc.free(url);
             const skipper = db.one(bool,
                 \\SELECT true FROM image
-                \\  WHERE id = ? AND image IS NOT NULL;
+                \\  WHERE eid = ? AND image_id IS NOT NULL;
+            , .{}, .{id}) catch {
+                sqliteErrorReport("SQLite error while checking if image is already downloaded", db);
+                return error.GO_ON;
+            };
+            if (skipper) |_| {
+                log.info("Image for ID {d} already downloaded.", .{id});
+                break :blk;
+            }
+            defer {
+                resp.clearRetainingCapacity();
+                std.mem.set(u8, hash_buf[0..], 0);
+                std.mem.set(u8, hash_buf2[0..], 0);
+            }
+            easyFetch(handle, url, resp) catch {
+                log.info("Failed to download fullsize image for ID {d}", .{id});
+                return error.FATAL;
+            };
+            hashit(resp.items) catch |err| {
+                log.err("Couldn't hash image for ID {d}: {s}", .{ id, err });
+                return error.GO_ON;
+            };
+            try db.exec("BEGIN IMMEDIATE;", .{}, .{});
+            errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
+            try db.exec(
+                \\INSERT INTO blob (data, hash)
+                \\  VALUES (?, ?);
+            , .{}, .{ resp.items, hash_buf2[0..] });
+            db.exec(
+                \\UPDATE OR ROLLBACK image
+                \\  SET image_id = last_insert_rowid() WHERE eid = ?
+            ,
+                .{},
+                .{id},
+            ) catch {
+                sqliteErrorReport("Couldn't add image to DB.", db);
+                return error.GO_ON;
+            };
+            db.exec("COMMIT", .{}, .{}) catch {
+                sqliteErrorReport("FATAL: couldn't commit database", db);
+                return error.FATAL;
+            };
+        }
+        if (res.thumb_url) |url| blk: {
+            defer alloc.free(url);
+            const skipper = db.one(bool,
+                \\SELECT true FROM image
+                \\  WHERE eid = ? AND thumb_id IS NOT NULL;
             , .{}, .{id}) catch {
                 sqliteErrorReport("SQLite error while checking if image is already downloaded", db);
                 return error.GO_ON;
@@ -603,79 +673,24 @@ fn getImage(
                 log.info("Failed to download fullsize image for ID {d}", .{id});
                 return error.FATAL;
             };
-            try db.exec("BEGIN IMMEDIATE;", .{}, .{});
-            errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
-            db.exec(
-                "UPDATE OR ROLLBACK image SET image = ? WHERE id = ?",
-                .{},
-                .{
-                    .image = resp.items,
-                    .id = id,
-                },
-            ) catch {
-                sqliteErrorReport("Couldn't add image to DB.", db);
-                return error.GO_ON;
-            };
             hashit(resp.items) catch |err| {
                 log.err("Couldn't hash image for ID {d}: {s}", .{ id, err });
                 return error.GO_ON;
             };
-            db.exec(
-                "UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?",
-                .{},
-                .{ hash_buf2[0..], id },
-            ) catch {
-                sqliteErrorReport("Couldn't set iamge hash", db);
-                return error.GO_ON;
-            };
-            db.exec("COMMIT", .{}, .{}) catch {
-                sqliteErrorReport("FATAL: couldn't commit database", db);
-                return error.FATAL;
-            };
-            resp.clearRetainingCapacity();
-            std.mem.set(u8, hash_buf[0..], 0);
-            std.mem.set(u8, hash_buf2[0..], 0);
-        }
-        if (res.thumb_url) |url| blk: {
-            defer alloc.free(url);
-            const skipper = db.one(bool,
-                \\SELECT true FROM image
-                \\  WHERE id = ? AND thumb IS NOT NULL;
-            , .{}, .{id}) catch {
-                sqliteErrorReport("SQLite error while checking if thumb is already downloaded", db);
-                return error.GO_ON;
-            };
-            if (skipper) |_| {
-                log.info("Thumb for ID {d} already downloaded.", .{id});
-                break :blk;
-            }
-            easyFetch(handle, url, resp) catch {
-                log.info("Failed to download thumbnail image for ID {d}", .{id});
-                return error.GO_ON;
-            };
             try db.exec("BEGIN IMMEDIATE;", .{}, .{});
             errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
+            try db.exec(
+                \\INSERT INTO blob (data, hash)
+                \\  VALUES (?, ?);
+            , .{}, .{ resp.items, hash_buf2[0..] });
             db.exec(
-                "UPDATE OR ROLLBACK image SET thumb = ? WHERE id = ?",
+                \\UPDATE OR ROLLBACK image
+                \\  SET thumb_id = last_insert_rowid() WHERE eid = ?
+            ,
                 .{},
-                .{
+                .{id},
-                    .thumb = resp.items,
-                    .id = id,
-                },
             ) catch {
-                sqliteErrorReport("Couldn't add thumb to DB", db);
+                sqliteErrorReport("Couldn't add image to DB.", db);
-                return error.GO_ON;
-            };
-            hashit(resp.items) catch |err| {
-                log.err("Couldn't hash thumb for ID {d}: {s}", .{ id, err });
-                return error.GO_ON;
-            };
-            db.exec(
-                "UPDATE OR ROLLBACK image SET hash_thumb = ? WHERE id = ?",
-                .{},
-                .{ hash_buf2[0..], id },
-            ) catch {
-                sqliteErrorReport("Couldn't add thumb hash", db);
                 return error.GO_ON;
             };
             db.exec("COMMIT", .{}, .{}) catch {
@@ -693,13 +708,13 @@ fn extractImage(db: *sqlite.Db, id: u64, alloc: *std.mem.Allocator) !void {
     log.info("Extracting image for ID {d}.", .{id});
     const foo = db.oneAlloc(
         struct {
-            image: ?[:0]u8,
+            image: ?[]u8,
-            extension: ?[:0]u8,
+            extension: ?[]u8,
         },
         alloc,
-        "SELECT image, extension FROM image WHERE id = ?",
+        "SELECT blob.data, image.extension FROM blob, image WHERE eid = ? AND image.image_id = blob.id;",
         .{},
-        .{ .id = id },
+        .{id},
     ) catch {
         sqliteErrorReport("SQLite error while reading image", db);
         return error.GO_ON;

zig.mod

@@ -6,6 +6,29 @@ dev_dependencies:
   - src: git https://github.com/vrischmann/zig-sqlite
     c_source_flags:
       - -DSQLITE_ENABLE_JSON1
+      - -DSQLITE_ENABLE_DQS=0
+      - -DSQLITE_THREADSAFE=0
+      - -DSQLITE_DEFAULT_MEMSTATUS=0
+      - -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1
+      - -DSQLITE_LIKE_DOESNT_MATCH_BLOBS
+      - -DSQLITE_MAX_EXPR_DEPTH=0
+      - -DSQLITE_OMIT_DEPRECATED
+      - -DSQLITE_OMIT_SHARED_CACHE
+      - -DSQLITE_DEFAULT_FOREIGN_KEYS
+      - -DHAVE_FDATASYNC
+      - -DHAVE_ISNAN
+      - -DSQLITE_USE_URI
+      - -DSQLITE_ALLOW_URI_AUTHORITY
+      - -DSQLITE_ENABLE_BYTECODE_VTAB
+      - -DSQLITE_ENABLE_COLUMN_METADATA
+      - -DSQLITE_ENABLE_DBPAGE_VTAB
+      - -DSQLITE_ENABLE_DBSTAT_VTAB
+      - -DSQLITE_ENABLE_EXPLAIN_COMMENTS
+      - -DSQLITE_ENABLE_FTS5
+      - -DSQLITE_ENABLE_MATH_FUNCTIONS
+      - -DSQLITE_ENABLE_BATCH_ATOMIC_WRITE=1
+    c_source_files:
+      - ../../../../../../sqlite-amalgamation-3360000/sqlite3.c
   - src: git https://github.com/nektro/zig-json
   - src: git https://github.com/truemedian/zfetch
   - src: git https://github.com/alexnask/iguanaTLS

zigmod.lock

@@ -1,9 +1,9 @@
 2
-git https://github.com/Hejsil/zig-clap commit-ed90e560d9b1144a27562ef62ff7686eb3af029a
+git https://github.com/Hejsil/zig-clap commit-c5fb22823a9a4a699acaefc1e9febfee0b8e506c
-git https://github.com/vrischmann/zig-sqlite commit-1adb900dcc816a419a4b67ccae0555ffe33f72c4
+git https://github.com/vrischmann/zig-sqlite commit-4954c419d379ffbb637904b41d25ef910c6bb02b
 git https://github.com/nektro/zig-json commit-72e555fbc0776f2600aee19b01e5ab1855ebec7a
-git https://github.com/truemedian/zfetch commit-8bbc7b34cd417794841e1432585334bc969dfe83
+git https://github.com/truemedian/zfetch commit-6ba2ba136ec7cfc887811039cd4a7d8a43ba725b
-git https://github.com/truemedian/hzzp commit-2d30bddae3bf1eaecde5144490307604efe76f2a
+git https://github.com/truemedian/hzzp commit-492107d44caa2676c7b5aa4e934e1e937232d652
 git https://github.com/alexnask/iguanaTLS commit-0d39a361639ad5469f8e4dcdaea35446bbe54b48
 git https://github.com/MasterQ32/zig-network commit-b9c91769d8ebd626c8e45b2abb05cbc28ccc50da
 git https://github.com/MasterQ32/zig-uri commit-52cdd2061bec0579519f0d30280597f3a1db8b75