From c1a646d38fc9046705181256f1b81dbb85241b2c Mon Sep 17 00:00:00 2001 From: Lonnie Date: Wed, 17 Nov 2021 00:28:40 +0100 Subject: [PATCH] Whoa, signed! --- build.zig | 1 + src/main.zig | 247 +++++++++++++++++++++++++++------------------------ zig.mod | 23 +++++ zigmod.lock | 8 +- 4 files changed, 159 insertions(+), 120 deletions(-) diff --git a/build.zig b/build.zig index cff9bdb..339e711 100644 --- a/build.zig +++ b/build.zig @@ -30,6 +30,7 @@ pub fn build(b: *std.build.Builder) void { } deps.addAllTo(exe); exe.linkLibC(); + exe.single_threaded = true; exe.linkSystemLibrary("libcurl"); exe.setTarget(target); exe.setBuildMode(mode); diff --git a/src/main.zig b/src/main.zig index 9793c35..db6040d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -29,6 +29,8 @@ const params = [_]clap.Param(clap.Help){ clap.parseParam("-O Order searches by ORDER, ascending.") catch unreachable, clap.parseParam("-p Start from page PAGE.") catch unreachable, clap.parseParam("-P Stop after PAGES pages.") catch unreachable, + clap.parseParam("--migrate Update an old database.") catch unreachable, + clap.parseParam("--yolo PRAGMA synchronous = OFF") catch unreachable, }; fn printFullUsage(w: anytype) !void { @@ -49,19 +51,18 @@ fn curlErrorReport(str: []const u8, code: curl.CURLcode) void { const create = \\CREATE TABLE IF NOT EXISTS image( - \\ id INTEGER UNIQUE, + \\ iid INTEGER PRIMARY KEY, + \\ eid INTEGER UNIQUE, \\ metadata TEXT, - \\ image BLOB, - \\ thumb BLOB, \\ full_url TEXT GENERATED ALWAYS AS \\ (json_extract(metadata, '$.image.representations.full')) VIRTUAL, \\ thumb_url TEXT GENERATED ALWAYS AS \\ (json_extract(metadata, '$.image.representations.thumb')) VIRTUAL, \\ extension TEXT GENERATED ALWAYS AS \\ (json_extract(metadata, '$.image.format')) VIRTUAL, - \\ hash_full TEXT, - \\ hash_thumb TEXT, - \\ hash_meta TEXT + \\ hash_meta TEXT, + \\ image_id INTEGER, + \\ thumb_id INTEGER \\); ; @@ -125,18 +126,50 @@ pub fn main() anyerror!void { return; } - var db: sqlite.Db = undefined; const filename = "test.db3"; - try db.init(.{ + var db = try sqlite.Db.init(.{ .mode = sqlite.Db.Mode{ .File = filename }, .open_flags = .{ .write = true, .create = true, }, - .threading_mode = .Serialized, + .threading_mode = .SingleThread, }); db.exec(create, .{}, .{}) catch sqliteErrorReport("Couldn't create table", &db); + if (args.flag("--migrate")) { + if (args.flag("--yolo")) try db.exec("PRAGMA synchronous=0;", .{}, .{}); + try db.exec("BEGIN IMMEDIATE;", .{}, .{}); + errdefer db.exec("ROLLBACK;", .{}, .{}) catch std.debug.panic("SQLite database errored trying to roll back. Have fun!", .{}); + var stmt = try db.prepare("SELECT ROWID FROM image;"); + defer stmt.deinit(); + var iter = try stmt.iterator(u64, .{}); + while (try iter.next(.{})) |id| { + try db.exec( + \\INSERT INTO blob (data, hash) + \\ SELECT image, hash_full + \\ FROM image WHERE ROWID = ?; + , .{}, .{id}); + try db.exec( + \\UPDATE image + \\ SET image_id = last_insert_rowid() + \\ WHERE ROWID = ? + , .{}, .{id}); + try db.exec( + \\INSERT INTO blob (data, hash) + \\ SELECT thumb, hash_thumb + \\ FROM image WHERE ROWID = ?; + , .{}, .{id}); + try db.exec( + \\UPDATE image + \\ SET thumb_id = last_insert_rowid() + \\ WHERE ROWID = ? + , .{}, .{id}); + } + db.exec("COMMIT;", .{}, .{}) catch std.debug.panic("SQLite database errored trying to commit. Not *terrible*, I guess.", .{}); + return; + } + var ret = curl.curl_global_init(curl.CURL_GLOBAL_ALL); if (ret != curl.CURLE_OK) { log.err("cURL global init failure: {s}", .{curl.curl_easy_strerror(ret)}); @@ -227,14 +260,6 @@ pub fn main() anyerror!void { } }; const sort_by = sort_descending orelse sort_ascending orelse "id"; - var headers = zfetch.Headers.init(alloc); - try headers.appendValue("Accept", "application/json"); - try headers.appendValue("User-Agent", "Derpiloader 0.1 (linux)"); - var req = try zfetch.Request.init( - alloc, - "https://derpibooru.org", - null, - ); var page = if (args.option("-p")) |page| blk: { break :blk std.fmt.parseInt(u64, page, 10) catch { log.err("Page must be a positive integer.", .{}); @@ -249,36 +274,34 @@ pub fn main() anyerror!void { }; } else 0; - var buf = std.ArrayList(u8).init(alloc); const kkey: []const u8 = key orelse ""; const aaaa: []const u8 = if (key) |_| "&key=" else ""; + var fuckme = std.ArrayList(u8).init(alloc); + defer fuckme.deinit(); for (searches) |search| { const esearch = try uri.escapeString(alloc, search); var pages: u64 = 0; log.info("Iterating over search \"{s}\", starting on page {d}.", .{ search, page }); - while (true) foo: { + fuck: while (true) { pages += 1; if (maxPages > 0 and pages == maxPages) { return; } log.info("Doing page {d}, {d}/{d}.", .{ page, pages, maxPages }); - buf.clearRetainingCapacity(); - const url = try std.fmt.allocPrint( - alloc, + _ = try std.fmt.bufPrintZ( + urlbuf[0..], api_base ++ "/search/images?q={s}&page={d}&sd={s}&sf={s}&per_page=50{s}{s}", .{ esearch, page, sort_order, sort_by, aaaa, kkey, }, ); - try req.reset(url); - try req.do(.GET, headers, null); - const reader = req.reader(); - try reader.readAllArrayList(&buf, 500 * 1024); - const val = try json.parse(alloc, buf.items); + fuckme.clearRetainingCapacity(); + try easyFetch(handle, &urlbuf, &fuckme); + const val = try json.parse(alloc, fuckme.items); if (val.get(.{"images"})) |aa| { if (unwrap(aa, .Array)) |images| { for (images) |i| { - var buffer: [1024 * 10]u8 = undefined; + var buffer: [1024 * 20]u8 = undefined; const pid = unwrap(i.get("id") orelse { log.err("Malformed reply from Derpi.", .{}); return; @@ -312,7 +335,7 @@ pub fn main() anyerror!void { if (images.len == 50) { page += 1; } else { - break :foo; + break :fuck; } } } @@ -413,9 +436,9 @@ fn registerID(db: *sqlite.Db, id: u64) !void { log.info("Registering ID {d}.", .{id}); const foo = db.one( bool, - "SELECT true FROM image WHERE id = ?;", + "SELECT true FROM image WHERE eid = ?;", .{}, - .{ .id = id }, + .{ .eid = id }, ) catch { sqliteErrorReport("SQLite error while checking if ID already present", db); return error.GO_ON; @@ -428,7 +451,7 @@ fn registerID(db: *sqlite.Db, id: u64) !void { errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; db.exec( \\INSERT OR ROLLBACK - \\ INTO image (id) + \\ INTO image (eid) \\ VALUES (?); , .{}, .{ .id = id }) catch { sqliteErrorReport("Couldn't insert ID into database.", db); @@ -449,9 +472,9 @@ fn storeMetadata( log.info("Storing metadata for ID {d}.", .{id}); const foobar = db.one( bool, - "SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;", + "SELECT true FROM image WHERE eid = ? AND metadata IS NOT NULL;", .{}, - .{ .id = id }, + .{ .eid = id }, ) catch { sqliteErrorReport("SQLite error while checking for metadata precence.", db); return error.GO_ON; @@ -468,12 +491,12 @@ fn storeMetadata( db.exec( \\INSERT OR ROLLBACK \\ INTO - \\ image (id, metadata) + \\ image (eid, metadata) \\ VALUES (?, ?) - \\ ON CONFLICT (id) + \\ ON CONFLICT (eid) \\ DO UPDATE \\ SET metadata=excluded.metadata; - , .{}, .{ .id = id, .metadata = metadata }) catch { + , .{}, .{ .eid = id, .metadata = metadata }) catch { sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db); return error.GO_ON; }; @@ -482,7 +505,7 @@ fn storeMetadata( return error.GO_ON; }; db.exec( - "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?", + "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE eid = ?", .{}, .{ hash_buf2[0..], id }, ) catch { @@ -505,9 +528,9 @@ fn getMetadata( log.info("Downloading metadata for ID {d}.", .{id}); const foobar = db.one( bool, - "SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;", + "SELECT true FROM image WHERE eid = ? AND metadata IS NOT NULL;", .{}, - .{ .id = id }, + .{ .eid = id }, ) catch { sqliteErrorReport("SQLite error while checking for metadata precence.", db); return error.GO_ON; @@ -523,7 +546,7 @@ fn getMetadata( ); easyFetch(handle, &urlbuf, resp) catch { log.info("Failed to download metadata for ID {d}.", .{id}); - return error.GO_ON; + return error.FATAL; }; const valid = std.json.validate(resp.items); @@ -533,12 +556,12 @@ fn getMetadata( db.exec( \\INSERT OR ROLLBACK \\ INTO - \\ image (id, metadata) + \\ image (eid, metadata) \\ VALUES (?, ?) - \\ ON CONFLICT (id) + \\ ON CONFLICT (eid) \\ DO UPDATE \\ SET metadata=excluded.metadata; - , .{}, .{ .id = id, .metadata = resp.items }) catch { + , .{}, .{ .eid = id, .metadata = resp.items }) catch { sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db); return error.GO_ON; }; @@ -547,7 +570,7 @@ fn getMetadata( return error.GO_ON; }; db.exec( - "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?", + "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE eid = ?", .{}, .{ hash_buf2[0..], id }, ) catch { @@ -578,9 +601,9 @@ fn getImage( thumb_url: ?[:0]u8, }, alloc, - "SELECT full_url, thumb_url FROM image WHERE id = ?", + "SELECT full_url, thumb_url FROM image WHERE eid = ?", .{}, - .{ .id = id }, + .{ .eid = id }, ) catch { sqliteErrorReport("SQLite error while getting image URLs", db); return error.GO_ON; @@ -590,7 +613,54 @@ fn getImage( defer alloc.free(url); const skipper = db.one(bool, \\SELECT true FROM image - \\ WHERE id = ? AND image IS NOT NULL; + \\ WHERE eid = ? AND image_id IS NOT NULL; + , .{}, .{id}) catch { + sqliteErrorReport("SQLite error while checking if image is already downloaded", db); + return error.GO_ON; + }; + if (skipper) |_| { + log.info("Image for ID {d} already downloaded.", .{id}); + break :blk; + } + defer { + resp.clearRetainingCapacity(); + std.mem.set(u8, hash_buf[0..], 0); + std.mem.set(u8, hash_buf2[0..], 0); + } + easyFetch(handle, url, resp) catch { + log.info("Failed to download fullsize image for ID {d}", .{id}); + return error.FATAL; + }; + hashit(resp.items) catch |err| { + log.err("Couldn't hash image for ID {d}: {s}", .{ id, err }); + return error.GO_ON; + }; + try db.exec("BEGIN IMMEDIATE;", .{}, .{}); + errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; + try db.exec( + \\INSERT INTO blob (data, hash) + \\ VALUES (?, ?); + , .{}, .{ resp.items, hash_buf2[0..] }); + db.exec( + \\UPDATE OR ROLLBACK image + \\ SET image_id = last_insert_rowid() WHERE eid = ? + , + .{}, + .{id}, + ) catch { + sqliteErrorReport("Couldn't add image to DB.", db); + return error.GO_ON; + }; + db.exec("COMMIT", .{}, .{}) catch { + sqliteErrorReport("FATAL: couldn't commit database", db); + return error.FATAL; + }; + } + if (res.thumb_url) |url| blk: { + defer alloc.free(url); + const skipper = db.one(bool, + \\SELECT true FROM image + \\ WHERE eid = ? AND thumb_id IS NOT NULL; , .{}, .{id}) catch { sqliteErrorReport("SQLite error while checking if image is already downloaded", db); return error.GO_ON; @@ -603,79 +673,24 @@ fn getImage( log.info("Failed to download fullsize image for ID {d}", .{id}); return error.FATAL; }; - try db.exec("BEGIN IMMEDIATE;", .{}, .{}); - errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; - db.exec( - "UPDATE OR ROLLBACK image SET image = ? WHERE id = ?", - .{}, - .{ - .image = resp.items, - .id = id, - }, - ) catch { - sqliteErrorReport("Couldn't add image to DB.", db); - return error.GO_ON; - }; hashit(resp.items) catch |err| { log.err("Couldn't hash image for ID {d}: {s}", .{ id, err }); return error.GO_ON; }; - db.exec( - "UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?", - .{}, - .{ hash_buf2[0..], id }, - ) catch { - sqliteErrorReport("Couldn't set iamge hash", db); - return error.GO_ON; - }; - db.exec("COMMIT", .{}, .{}) catch { - sqliteErrorReport("FATAL: couldn't commit database", db); - return error.FATAL; - }; - resp.clearRetainingCapacity(); - std.mem.set(u8, hash_buf[0..], 0); - std.mem.set(u8, hash_buf2[0..], 0); - } - if (res.thumb_url) |url| blk: { - defer alloc.free(url); - const skipper = db.one(bool, - \\SELECT true FROM image - \\ WHERE id = ? AND thumb IS NOT NULL; - , .{}, .{id}) catch { - sqliteErrorReport("SQLite error while checking if thumb is already downloaded", db); - return error.GO_ON; - }; - if (skipper) |_| { - log.info("Thumb for ID {d} already downloaded.", .{id}); - break :blk; - } - easyFetch(handle, url, resp) catch { - log.info("Failed to download thumbnail image for ID {d}", .{id}); - return error.GO_ON; - }; try db.exec("BEGIN IMMEDIATE;", .{}, .{}); errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; + try db.exec( + \\INSERT INTO blob (data, hash) + \\ VALUES (?, ?); + , .{}, .{ resp.items, hash_buf2[0..] }); db.exec( - "UPDATE OR ROLLBACK image SET thumb = ? WHERE id = ?", + \\UPDATE OR ROLLBACK image + \\ SET thumb_id = last_insert_rowid() WHERE eid = ? + , .{}, - .{ - .thumb = resp.items, - .id = id, - }, + .{id}, ) catch { - sqliteErrorReport("Couldn't add thumb to DB", db); - return error.GO_ON; - }; - hashit(resp.items) catch |err| { - log.err("Couldn't hash thumb for ID {d}: {s}", .{ id, err }); - return error.GO_ON; - }; - db.exec( - "UPDATE OR ROLLBACK image SET hash_thumb = ? WHERE id = ?", - .{}, - .{ hash_buf2[0..], id }, - ) catch { - sqliteErrorReport("Couldn't add thumb hash", db); + sqliteErrorReport("Couldn't add image to DB.", db); return error.GO_ON; }; db.exec("COMMIT", .{}, .{}) catch { @@ -693,13 +708,13 @@ fn extractImage(db: *sqlite.Db, id: u64, alloc: *std.mem.Allocator) !void { log.info("Extracting image for ID {d}.", .{id}); const foo = db.oneAlloc( struct { - image: ?[:0]u8, - extension: ?[:0]u8, + image: ?[]u8, + extension: ?[]u8, }, alloc, - "SELECT image, extension FROM image WHERE id = ?", + "SELECT blob.data, image.extension FROM blob, image WHERE eid = ? AND image.image_id = blob.id;", .{}, - .{ .id = id }, + .{id}, ) catch { sqliteErrorReport("SQLite error while reading image", db); return error.GO_ON; diff --git a/zig.mod b/zig.mod index df61022..f6f5b11 100644 --- a/zig.mod +++ b/zig.mod @@ -6,6 +6,29 @@ dev_dependencies: - src: git https://github.com/vrischmann/zig-sqlite c_source_flags: - -DSQLITE_ENABLE_JSON1 + - -DSQLITE_ENABLE_DQS=0 + - -DSQLITE_THREADSAFE=0 + - -DSQLITE_DEFAULT_MEMSTATUS=0 + - -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 + - -DSQLITE_LIKE_DOESNT_MATCH_BLOBS + - -DSQLITE_MAX_EXPR_DEPTH=0 + - -DSQLITE_OMIT_DEPRECATED + - -DSQLITE_OMIT_SHARED_CACHE + - -DSQLITE_DEFAULT_FOREIGN_KEYS + - -DHAVE_FDATASYNC + - -DHAVE_ISNAN + - -DSQLITE_USE_URI + - -DSQLITE_ALLOW_URI_AUTHORITY + - -DSQLITE_ENABLE_BYTECODE_VTAB + - -DSQLITE_ENABLE_COLUMN_METADATA + - -DSQLITE_ENABLE_DBPAGE_VTAB + - -DSQLITE_ENABLE_DBSTAT_VTAB + - -DSQLITE_ENABLE_EXPLAIN_COMMENTS + - -DSQLITE_ENABLE_FTS5 + - -DSQLITE_ENABLE_MATH_FUNCTIONS + - -DSQLITE_ENABLE_BATCH_ATOMIC_WRITE=1 + c_source_files: + - ../../../../../../sqlite-amalgamation-3360000/sqlite3.c - src: git https://github.com/nektro/zig-json - src: git https://github.com/truemedian/zfetch - src: git https://github.com/alexnask/iguanaTLS diff --git a/zigmod.lock b/zigmod.lock index 12770e3..e1e8ad9 100644 --- a/zigmod.lock +++ b/zigmod.lock @@ -1,9 +1,9 @@ 2 -git https://github.com/Hejsil/zig-clap commit-ed90e560d9b1144a27562ef62ff7686eb3af029a -git https://github.com/vrischmann/zig-sqlite commit-1adb900dcc816a419a4b67ccae0555ffe33f72c4 +git https://github.com/Hejsil/zig-clap commit-c5fb22823a9a4a699acaefc1e9febfee0b8e506c +git https://github.com/vrischmann/zig-sqlite commit-4954c419d379ffbb637904b41d25ef910c6bb02b git https://github.com/nektro/zig-json commit-72e555fbc0776f2600aee19b01e5ab1855ebec7a -git https://github.com/truemedian/zfetch commit-8bbc7b34cd417794841e1432585334bc969dfe83 -git https://github.com/truemedian/hzzp commit-2d30bddae3bf1eaecde5144490307604efe76f2a +git https://github.com/truemedian/zfetch commit-6ba2ba136ec7cfc887811039cd4a7d8a43ba725b +git https://github.com/truemedian/hzzp commit-492107d44caa2676c7b5aa4e934e1e937232d652 git https://github.com/alexnask/iguanaTLS commit-0d39a361639ad5469f8e4dcdaea35446bbe54b48 git https://github.com/MasterQ32/zig-network commit-b9c91769d8ebd626c8e45b2abb05cbc28ccc50da git https://github.com/MasterQ32/zig-uri commit-52cdd2061bec0579519f0d30280597f3a1db8b75