From 1213adfc1fe4dd2bac580a55a9ddf9bf9822f95a Mon Sep 17 00:00:00 2001 From: Lonnie Date: Sun, 8 Aug 2021 07:20:21 +0200 Subject: [PATCH] Broken zfetch code for later reference. --- src/main.zig | 762 ++++++++++++++++++++++++++++++++++++++++----------- zig.mod | 12 +- zigmod.lock | 8 +- zigmod.sum | 8 +- 4 files changed, 629 insertions(+), 161 deletions(-) diff --git a/src/main.zig b/src/main.zig index 72050f5..9793c35 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5,17 +5,30 @@ const curl = @cImport({ @cInclude("curl/curl.h"); }); +const zfetch = @import("zfetch"); +const tls = @import("iguanaTLS"); +const uri = @import("uri"); +const json = @import("json"); + const log = std.log.scoped(.derploader); const params = [_]clap.Param(clap.Help){ clap.parseParam("-h, --help Display this help and exit.") catch unreachable, clap.parseParam("-c, --create= Create new database at PATH.") catch unreachable, - clap.parseParam("-i Operate on ID.") catch unreachable, + clap.parseParam("-i ... Operate on ID.") catch unreachable, clap.parseParam("-a Operate on all IDs in the database.") catch unreachable, + clap.parseParam("-l ... Operate on IDs from a file.") catch unreachable, clap.parseParam("-m Download metadata for ID image.") catch unreachable, clap.parseParam("-d Download image data of ID.") catch unreachable, - clap.parseParam("-e Extract image.") catch unreachable, - clap.parseParam("-t Time between requests.") catch unreachable, + clap.parseParam("-e Extract image to local subfolder.") catch unreachable, + clap.parseParam("-t Time between requests.") catch unreachable, + clap.parseParam("-r Register ID.") catch unreachable, + clap.parseParam("-k Include key in searches from environment variable DERPI_KEY.") catch unreachable, + clap.parseParam("-s ... Iterate over the results of searches.") catch unreachable, + clap.parseParam("-o Order searches by ORDER, descending.") catch unreachable, + clap.parseParam("-O Order searches by ORDER, ascending.") catch unreachable, + clap.parseParam("-p Start from page PAGE.") catch unreachable, + clap.parseParam("-P Stop after PAGES pages.") catch unreachable, }; fn printFullUsage(w: anytype) !void { @@ -44,6 +57,8 @@ const create = \\ (json_extract(metadata, '$.image.representations.full')) VIRTUAL, \\ thumb_url TEXT GENERATED ALWAYS AS \\ (json_extract(metadata, '$.image.representations.thumb')) VIRTUAL, + \\ extension TEXT GENERATED ALWAYS AS + \\ (json_extract(metadata, '$.image.format')) VIRTUAL, \\ hash_full TEXT, \\ hash_thumb TEXT, \\ hash_meta TEXT @@ -61,13 +76,13 @@ pub fn insertMeta(db: *sqlite.Db, id: u64, meta: []const u8) !void { const q = \\INSERT OR ROLLBACK INTO image (id, metadata) VALUES (?, ?); ; - try db.exec(q, .{ .id = id, .metadata = meta }); + try db.exec(q, .{}, .{ .id = id, .metadata = meta }); } const api_base = "https://derpibooru.org/api/v1/json"; var urlbuf = [_:0]u8{0} ** 512; -var curlerr = [_:0]u8{0} ** (curl.CURL_ERROR_SIZE + 1); +var curlerr = [_:0]u8{0} ** (curl.CURL_ERROR_SIZE); const hash_prefix = "blake3-"; var hash_buf = [_]u8{0} ** (std.crypto.hash.Blake3.digest_length); @@ -82,6 +97,10 @@ fn hashit(input: []const u8) !void { ); } +var fetch_timer: ?std.time.Timer = null; +var fetch_wait: u64 = 0; +var first_fetch = true; + pub fn main() anyerror!void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; const alloc = &gpa.allocator; @@ -116,7 +135,7 @@ pub fn main() anyerror!void { }, .threading_mode = .Serialized, }); - db.exec(create, .{}) catch sqliteErrorReport("Couldn't create table", &db); + db.exec(create, .{}, .{}) catch sqliteErrorReport("Couldn't create table", &db); var ret = curl.curl_global_init(curl.CURL_GLOBAL_ALL); if (ret != curl.CURLE_OK) { @@ -131,175 +150,596 @@ pub fn main() anyerror!void { _ = curl.curl_easy_setopt(handle, curl.CURLOPT_ERRORBUFFER, &curlerr); - const maybe_id: ?u64 = if (args.option("-i")) |id_str| blk: { - break :blk std.fmt.parseInt(u64, id_str, 10) catch { + const key = std.os.getenv("DERPI_KEY") orelse null; + + if (args.option("-t")) |millis_str| { + const millis = std.fmt.parseInt(u64, millis_str, 10) catch { + log.err("Fetch wait time must be a positive integer denoting some number of milliseconds.", .{}); + return; + }; + fetch_wait = millis; + } + + for (args.options("-i")) |id_str| { + log.info("Iterating over all specified command line IDs.", .{}); + const id = std.fmt.parseInt(u64, id_str, 10) catch { log.err("Image ID must be a positive integer.", .{}); - return; + continue; }; - } else null; + log.info("Operating on ID {d}, per -i.", .{id}); + runActions(&db, id, &response_buffer, alloc, handle, &args, null); + } - if (args.flag("-m")) { - const id = if (maybe_id) |id| - id - else { - log.err( - "Operation download metadata requires an ID (-i) argument.", - .{}, - ); - return; - }; - const foobar = db.one( - bool, - "SELECT true FROM image WHERE id = ?", - .{}, - .{ .id = id }, - ) catch { - sqliteErrorReport("ID check read error", &db); - return; - }; - if (foobar) |_| { - log.info("Info for id {d} already acquired.", .{id}); - return; + if (args.flag("-a")) { + log.info("Iterating over all registered IDs.", .{}); + var stmt = try db.prepare("SELECT id FROM image WHERE id IS NOT NULL"); + defer stmt.deinit(); + var iter = try stmt.iterator(u64, .{}); + while (try iter.next(.{})) |id| { + log.info("Operating on ID {d}, per -a.", .{id}); + runActions(&db, id, &response_buffer, alloc, handle, &args, null); } - _ = try std.fmt.bufPrintZ( - urlbuf[0..], - api_base ++ "/images/{d}", - .{id}, - ); - easyFetch(handle, &urlbuf, &response_buffer) catch return; - //var w = std.io.getStdOut().writer(); - const valid = std.json.validate(response_buffer.items); + } - if (valid) { - try db.exec("BEGIN IMMEDIATE;", .{}); - errdefer db.exec("ROLLBACK;", .{}) catch {}; - insertMeta(&db, id, response_buffer.items) catch { - sqliteErrorReport("Can't insert:", &db); + for (args.options("-l")) |path| { + log.info("Iterating over IDs listed in {s}.", .{path}); + var file = std.fs.cwd().openFile(path, .{ .read = true }) catch |err| { + log.err("Couldn't open file {s}: {}", .{ path, err }); + continue; + }; + defer file.close(); + var reader = file.reader(); + var buffer: [128]u8 = undefined; + while (try reader.readUntilDelimiterOrEof(&buffer, '\n')) |id_str| { + const id = std.fmt.parseInt(u64, id_str, 10) catch { + log.err("Image ID must be a positive integer.", .{}); + continue; + }; + log.info("Operating on ID {d}, per -l.", .{id}); + runActions(&db, id, &response_buffer, alloc, handle, &args, null); + } + } + + const searches = args.options("-s"); + if (searches.len > 0) { + //var reader = std.io.fixedBufferStream( + // @embedFile("/etc/ssl/certs/ca-certificates.crt"), + //).reader(); + //const trust = try tls.x509.CertificateChain.from_pem(alloc, reader); + // catch |a| { + // log.err("Something dun fucked with the certs: {}", .{a}); + // return; + // }; + const sort_ascending = args.option("-O"); + const sort_descending = args.option("-o"); + const sort_order = if (sort_ascending) |_| blk: { + if (sort_descending) |_| { + log.err("Can't sort up *and* down, dummy.", .{}); + return; + } else { + break :blk "asc"; + } + } else blk: { + if (sort_descending) |_| { + break :blk "desc"; + } else { + break :blk "desc"; + } + }; + const sort_by = sort_descending orelse sort_ascending orelse "id"; + var headers = zfetch.Headers.init(alloc); + try headers.appendValue("Accept", "application/json"); + try headers.appendValue("User-Agent", "Derpiloader 0.1 (linux)"); + var req = try zfetch.Request.init( + alloc, + "https://derpibooru.org", + null, + ); + var page = if (args.option("-p")) |page| blk: { + break :blk std.fmt.parseInt(u64, page, 10) catch { + log.err("Page must be a positive integer.", .{}); return; }; - try hashit(response_buffer.items); + } else 1; + + var maxPages = if (args.option("-P")) |nr| blk: { + break :blk std.fmt.parseInt(u64, nr, 10) catch { + log.err("Pages maximum must be a positive integer.", .{}); + return; + }; + } else 0; + + var buf = std.ArrayList(u8).init(alloc); + const kkey: []const u8 = key orelse ""; + const aaaa: []const u8 = if (key) |_| "&key=" else ""; + for (searches) |search| { + const esearch = try uri.escapeString(alloc, search); + var pages: u64 = 0; + log.info("Iterating over search \"{s}\", starting on page {d}.", .{ search, page }); + while (true) foo: { + pages += 1; + if (maxPages > 0 and pages == maxPages) { + return; + } + log.info("Doing page {d}, {d}/{d}.", .{ page, pages, maxPages }); + buf.clearRetainingCapacity(); + const url = try std.fmt.allocPrint( + alloc, + api_base ++ "/search/images?q={s}&page={d}&sd={s}&sf={s}&per_page=50{s}{s}", + .{ + esearch, page, sort_order, sort_by, aaaa, kkey, + }, + ); + try req.reset(url); + try req.do(.GET, headers, null); + const reader = req.reader(); + try reader.readAllArrayList(&buf, 500 * 1024); + const val = try json.parse(alloc, buf.items); + if (val.get(.{"images"})) |aa| { + if (unwrap(aa, .Array)) |images| { + for (images) |i| { + var buffer: [1024 * 10]u8 = undefined; + const pid = unwrap(i.get("id") orelse { + log.err("Malformed reply from Derpi.", .{}); + return; + }, .Int) orelse { + log.err("Malformed reply from Derpi, but in a different way.", .{}); + return; + }; + const id = if (pid >= 0) @intCast(u64, pid) else { + log.err("Malformed reply from Derpi, but in a third way.", .{}); + return; + }; + var aaa = std.io.fixedBufferStream(buffer[0..]).writer(); + const data = json.Value{ .Object = &[_]json.Member{ + json.Member{ + .key = "image", + .value = i, + }, + } }; + try data.format("", .{}, aaa); + const jason = buffer[0..aaa.context.pos]; + runActions( + &db, + id, + &response_buffer, + alloc, + handle, + &args, + jason, + ); + } + if (images.len == 50) { + page += 1; + } else { + break :foo; + } + } + } + } + page = 1; + } + } +} + +pub fn unwrap( + un: anytype, + comptime tag: std.meta.Tag(@TypeOf(un)), +) ?std.meta.TagPayload(@TypeOf(un), tag) { + if (un != tag) return null; + return @field(un, @tagName(tag)); +} + +fn runActions( + db: *sqlite.Db, + id: u64, + resp: *std.ArrayList(u8), + alloc: *std.mem.Allocator, + handle: *curl.CURL, + args: anytype, + meta: ?[]const u8, +) void { + if (args.flag("-r")) { + registerID(db, id) catch |e| switch (e) { + error.GO_ON => {}, + error.FATAL => { + db.deinit(); + std.os.exit(1); + }, + else => { + db.deinit(); + std.os.exit(2); + }, + }; + } + if (args.flag("-m")) { + if (meta) |m| { + storeMetadata(db, id, m) catch |e| switch (e) { + error.GO_ON => {}, + error.FATAL => { + db.deinit(); + std.os.exit(1); + }, + else => { + db.deinit(); + std.os.exit(2); + }, + }; + } else { + getMetadata(db, id, resp, handle) catch |e| switch (e) { + error.GO_ON => {}, + error.FATAL => { + db.deinit(); + std.os.exit(1); + }, + else => { + db.deinit(); + std.os.exit(2); + }, + }; + } + } + resp.clearRetainingCapacity(); + std.mem.set(u8, hash_buf[0..], 0); + std.mem.set(u8, hash_buf2[0..], 0); + if (args.flag("-d")) { + getImage(db, id, resp, alloc, handle) catch |e| switch (e) { + error.GO_ON => {}, + error.FATAL => { + db.deinit(); + std.os.exit(1); + }, + else => { + db.deinit(); + std.os.exit(2); + }, + }; + } + resp.clearRetainingCapacity(); + std.mem.set(u8, hash_buf[0..], 0); + std.mem.set(u8, hash_buf2[0..], 0); + if (args.flag("-e")) { + extractImage(db, id, alloc) catch |e| switch (e) { + error.GO_ON => {}, + else => { + db.deinit(); + std.os.exit(2); + }, + }; + } +} + +fn registerID(db: *sqlite.Db, id: u64) !void { + log.info("Registering ID {d}.", .{id}); + const foo = db.one( + bool, + "SELECT true FROM image WHERE id = ?;", + .{}, + .{ .id = id }, + ) catch { + sqliteErrorReport("SQLite error while checking if ID already present", db); + return error.GO_ON; + }; + if (foo) |_| { + log.info("ID {d} already registered.", .{id}); + return; + } + try db.exec("BEGIN IMMEDIATE", .{}, .{}); + errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; + db.exec( + \\INSERT OR ROLLBACK + \\ INTO image (id) + \\ VALUES (?); + , .{}, .{ .id = id }) catch { + sqliteErrorReport("Couldn't insert ID into database.", db); + return error.GO_ON; + }; + + db.exec("COMMIT", .{}, .{}) catch { + sqliteErrorReport("FATAL: couldn't commit database", db); + return error.FATAL; + }; +} + +fn storeMetadata( + db: *sqlite.Db, + id: u64, + metadata: []const u8, +) !void { + log.info("Storing metadata for ID {d}.", .{id}); + const foobar = db.one( + bool, + "SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;", + .{}, + .{ .id = id }, + ) catch { + sqliteErrorReport("SQLite error while checking for metadata precence.", db); + return error.GO_ON; + }; + if (foobar) |_| { + log.info("Metadata for ID {d} already acquired. Use -u to replace.", .{id}); + return; + } + const valid = std.json.validate(metadata); + + if (valid) { + try db.exec("BEGIN IMMEDIATE;", .{}, .{}); + errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; + db.exec( + \\INSERT OR ROLLBACK + \\ INTO + \\ image (id, metadata) + \\ VALUES (?, ?) + \\ ON CONFLICT (id) + \\ DO UPDATE + \\ SET metadata=excluded.metadata; + , .{}, .{ .id = id, .metadata = metadata }) catch { + sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db); + return error.GO_ON; + }; + hashit(metadata) catch |err| { + log.err("Couldn't hash metadata for ID {d}: {s}", .{ id, err }); + return error.GO_ON; + }; + db.exec( + "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?", + .{}, + .{ hash_buf2[0..], id }, + ) catch { + sqliteErrorReport("Couldn't set metadata hash", db); + return error.GO_ON; + }; + db.exec("COMMIT", .{}, .{}) catch { + sqliteErrorReport("FATAL: couldn't commit database", db); + return error.FATAL; + }; + } +} + +fn getMetadata( + db: *sqlite.Db, + id: u64, + resp: *std.ArrayList(u8), + handle: *curl.CURL, +) !void { + log.info("Downloading metadata for ID {d}.", .{id}); + const foobar = db.one( + bool, + "SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;", + .{}, + .{ .id = id }, + ) catch { + sqliteErrorReport("SQLite error while checking for metadata precence.", db); + return error.GO_ON; + }; + if (foobar) |_| { + log.info("Metadata for ID {d} already acquired. Use -u to replace.", .{id}); + return; + } + _ = try std.fmt.bufPrintZ( + urlbuf[0..], + api_base ++ "/images/{d}", + .{id}, + ); + easyFetch(handle, &urlbuf, resp) catch { + log.info("Failed to download metadata for ID {d}.", .{id}); + return error.GO_ON; + }; + const valid = std.json.validate(resp.items); + + if (valid) { + try db.exec("BEGIN IMMEDIATE;", .{}, .{}); + errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; + db.exec( + \\INSERT OR ROLLBACK + \\ INTO + \\ image (id, metadata) + \\ VALUES (?, ?) + \\ ON CONFLICT (id) + \\ DO UPDATE + \\ SET metadata=excluded.metadata; + , .{}, .{ .id = id, .metadata = resp.items }) catch { + sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db); + return error.GO_ON; + }; + hashit(resp.items) catch |err| { + log.err("Couldn't hash metadata for ID {d}: {s}", .{ id, err }); + return error.GO_ON; + }; + db.exec( + "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?", + .{}, + .{ hash_buf2[0..], id }, + ) catch { + sqliteErrorReport("Couldn't set metadata hash", db); + return error.GO_ON; + }; + db.exec("COMMIT", .{}, .{}) catch { + sqliteErrorReport("FATAL: couldn't commit database", db); + return error.FATAL; + }; + } else { + log.err("Invalid metadata for ID {d}", .{id}); + return error.FATAL; + } +} + +fn getImage( + db: *sqlite.Db, + id: u64, + resp: *std.ArrayList(u8), + alloc: *std.mem.Allocator, + handle: *curl.CURL, +) !void { + log.info("Downloading image and thumbnail for ID {d}.", .{id}); + const foobar = db.oneAlloc( + struct { + full_url: ?[:0]u8, + thumb_url: ?[:0]u8, + }, + alloc, + "SELECT full_url, thumb_url FROM image WHERE id = ?", + .{}, + .{ .id = id }, + ) catch { + sqliteErrorReport("SQLite error while getting image URLs", db); + return error.GO_ON; + }; + if (foobar) |res| { + if (res.full_url) |url| blk: { + defer alloc.free(url); + const skipper = db.one(bool, + \\SELECT true FROM image + \\ WHERE id = ? AND image IS NOT NULL; + , .{}, .{id}) catch { + sqliteErrorReport("SQLite error while checking if image is already downloaded", db); + return error.GO_ON; + }; + if (skipper) |_| { + log.info("Image for ID {d} already downloaded.", .{id}); + break :blk; + } + easyFetch(handle, url, resp) catch { + log.info("Failed to download fullsize image for ID {d}", .{id}); + return error.FATAL; + }; + try db.exec("BEGIN IMMEDIATE;", .{}, .{}); + errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; db.exec( - "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?", + "UPDATE OR ROLLBACK image SET image = ? WHERE id = ?", + .{}, + .{ + .image = resp.items, + .id = id, + }, + ) catch { + sqliteErrorReport("Couldn't add image to DB.", db); + return error.GO_ON; + }; + hashit(resp.items) catch |err| { + log.err("Couldn't hash image for ID {d}: {s}", .{ id, err }); + return error.GO_ON; + }; + db.exec( + "UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?", + .{}, .{ hash_buf2[0..], id }, ) catch { - sqliteErrorReport("Couldn't insert", &db); - return; + sqliteErrorReport("Couldn't set iamge hash", db); + return error.GO_ON; }; - try db.exec("COMMIT", .{}); + db.exec("COMMIT", .{}, .{}) catch { + sqliteErrorReport("FATAL: couldn't commit database", db); + return error.FATAL; + }; + resp.clearRetainingCapacity(); + std.mem.set(u8, hash_buf[0..], 0); + std.mem.set(u8, hash_buf2[0..], 0); + } + if (res.thumb_url) |url| blk: { + defer alloc.free(url); + const skipper = db.one(bool, + \\SELECT true FROM image + \\ WHERE id = ? AND thumb IS NOT NULL; + , .{}, .{id}) catch { + sqliteErrorReport("SQLite error while checking if thumb is already downloaded", db); + return error.GO_ON; + }; + if (skipper) |_| { + log.info("Thumb for ID {d} already downloaded.", .{id}); + break :blk; + } + easyFetch(handle, url, resp) catch { + log.info("Failed to download thumbnail image for ID {d}", .{id}); + return error.GO_ON; + }; + try db.exec("BEGIN IMMEDIATE;", .{}, .{}); + errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; + db.exec( + "UPDATE OR ROLLBACK image SET thumb = ? WHERE id = ?", + .{}, + .{ + .thumb = resp.items, + .id = id, + }, + ) catch { + sqliteErrorReport("Couldn't add thumb to DB", db); + return error.GO_ON; + }; + hashit(resp.items) catch |err| { + log.err("Couldn't hash thumb for ID {d}: {s}", .{ id, err }); + return error.GO_ON; + }; + db.exec( + "UPDATE OR ROLLBACK image SET hash_thumb = ? WHERE id = ?", + .{}, + .{ hash_buf2[0..], id }, + ) catch { + sqliteErrorReport("Couldn't add thumb hash", db); + return error.GO_ON; + }; + db.exec("COMMIT", .{}, .{}) catch { + sqliteErrorReport("FATAL: couldn't commit database", db); + return error.FATAL; + }; + } + } else { + log.err("No metadata for id {d} available", .{id}); + return; + } +} + +fn extractImage(db: *sqlite.Db, id: u64, alloc: *std.mem.Allocator) !void { + log.info("Extracting image for ID {d}.", .{id}); + const foo = db.oneAlloc( + struct { + image: ?[:0]u8, + extension: ?[:0]u8, + }, + alloc, + "SELECT image, extension FROM image WHERE id = ?", + .{}, + .{ .id = id }, + ) catch { + sqliteErrorReport("SQLite error while reading image", db); + return error.GO_ON; + }; + defer { + if (foo) |f| { + if (f.image) |i| { + alloc.free(i); + } + if (f.extension) |e| { + alloc.free(e); + } } } - - if (args.flag("-d")) { - const id = if (maybe_id) |id| - id - else { - log.err( - "Operation download image requires an ID (-i) argument.", - .{}, - ); - return; - }; - const foobar = db.oneAlloc( - struct { - full_url: ?[:0]u8, - thumb_url: ?[:0]u8, + if (foo) |res| { + const bar = comptime @as([]const u8, "unknown"); + const baz = if (res.extension) |e| e else bar; + var buf = [_]u8{0} ** 64; + const buf2 = try std.fmt.bufPrint( + buf[0..], + "{d:0>10}.{s}", + .{ + id, + baz, }, - alloc, - "SELECT full_url, thumb_url FROM image WHERE id = ?", - .{}, - .{ .id = id }, - ) catch { - sqliteErrorReport("ID check read error", &db); - return; - }; - if (foobar) |res| { - if (res.full_url) |url| { - easyFetch(handle, url, &response_buffer) catch return; - try db.exec("BEGIN IMMEDIATE;", .{}); - errdefer db.exec("ROLLBACK;", .{}) catch {}; - db.exec( - "UPDATE OR ROLLBACK image SET image = ? WHERE id = ?", - .{ - .image = response_buffer.items, - .id = id, - }, - ) catch { - sqliteErrorReport("Couldn't add image to DB.", &db); - return; - }; - try hashit(response_buffer.items); - db.exec( - "UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?", - .{ hash_buf2[0..], id }, - ) catch { - sqliteErrorReport("Couldn't add iamge hash", &db); - return; - }; - try db.exec("COMMIT", .{}); - response_buffer.clearRetainingCapacity(); - std.mem.set(u8, hash_buf[0..], 0); - std.mem.set(u8, hash_buf2[0..], 0); - } - if (res.thumb_url) |url| { - easyFetch(handle, url, &response_buffer) catch return; - try db.exec("BEGIN IMMEDIATE;", .{}); - errdefer db.exec("ROLLBACK;", .{}) catch {}; - db.exec( - "UPDATE OR ROLLBACK image SET thumb = ? WHERE id = ?", - .{ - .thumb = response_buffer.items, - .id = id, - }, - ) catch { - sqliteErrorReport("Couldn't add thumb to DB", &db); - return; - }; - try hashit(response_buffer.items); - db.exec( - "UPDATE OR ROLLBACK image SET hash_thumb = ? WHERE id = ?", - .{ hash_buf2[0..], id }, - ) catch { - sqliteErrorReport("Couldn't add thumb hash", &db); - return; - }; - try db.exec("COMMIT", .{}); - } - } else { - log.err("No metadata for id {d} available", .{id}); - return; - } - } - - if (args.option("-e")) |path| { - const id = if (maybe_id) |id| - id - else { - log.err( - "Operation extract image requires an ID (-i) argument.", - .{}, - ); - return; - }; - const maybe_image = db.oneAlloc( - []u8, - alloc, - "SELECT image FROM image WHERE id = ?", - .{}, - .{ .id = id }, - ) catch { - sqliteErrorReport("ID check read error", &db); - return; - }; - if (maybe_image) |image| { - var file = try std.fs.cwd().createFile( - path, + ); + if (res.image) |image| { + var dir = try std.fs.cwd().makeOpenPath("images", .{ + .access_sub_paths = true, + }); + var file = try dir.createFile( + buf2, .{ .read = false, .truncate = true, }, ); + defer file.close(); try file.writeAll(image); - file.close(); + log.info("Extracted image for ID {d}.", .{id}); } else { log.info("No image data for ID {d}.", .{id}); } @@ -307,6 +747,17 @@ pub fn main() anyerror!void { } fn easyFetch(handle: *curl.CURL, url: [*:0]const u8, resp: *std.ArrayList(u8)) !void { + if (fetch_wait > 0) { + if (fetch_timer) |*timer| { + const cur = timer.read() / (1000 * 1000); + if (cur < fetch_wait) { + std.time.sleep((fetch_wait - cur) * 1000 * 1000); + } + timer.reset(); + } else { + fetch_timer = try std.time.Timer.start(); + } + } var ret = curl.curl_easy_setopt(handle, curl.CURLOPT_URL, url); if (ret != curl.CURLE_OK) { curlErrorReport("cURL set url:", ret); @@ -332,6 +783,7 @@ fn easyFetch(handle: *curl.CURL, url: [*:0]const u8, resp: *std.ArrayList(u8)) ! curlErrorReport("cURL perform:", ret); return error.FUCK; } + log.info("Got {d} bytes", .{resp.items.len}); } diff --git a/zig.mod b/zig.mod index 8009d0e..df61022 100644 --- a/zig.mod +++ b/zig.mod @@ -2,7 +2,11 @@ id: 1m1xcon88qpj9vi17d8yo2743g0pqswk91ep21b0v139bu5s name: derploader main: src/main.zig dev_dependencies: - - src: git https://github.com/Hejsil/zig-clap branch-zig-master - - src: git https://github.com/vrischmann/zig-sqlite - c_source_flags: - - -DSQLITE_ENABLE_JSON1 + - src: git https://github.com/Hejsil/zig-clap branch-zig-master + - src: git https://github.com/vrischmann/zig-sqlite + c_source_flags: + - -DSQLITE_ENABLE_JSON1 + - src: git https://github.com/nektro/zig-json + - src: git https://github.com/truemedian/zfetch + - src: git https://github.com/alexnask/iguanaTLS + - src: git https://github.com/MasterQ32/zig-uri/ diff --git a/zigmod.lock b/zigmod.lock index 4d62cdf..12770e3 100644 --- a/zigmod.lock +++ b/zigmod.lock @@ -1,3 +1,9 @@ 2 git https://github.com/Hejsil/zig-clap commit-ed90e560d9b1144a27562ef62ff7686eb3af029a -git https://github.com/vrischmann/zig-sqlite commit-fafe666f22590faf82d7c7ce1087858002c004b6 +git https://github.com/vrischmann/zig-sqlite commit-1adb900dcc816a419a4b67ccae0555ffe33f72c4 +git https://github.com/nektro/zig-json commit-72e555fbc0776f2600aee19b01e5ab1855ebec7a +git https://github.com/truemedian/zfetch commit-8bbc7b34cd417794841e1432585334bc969dfe83 +git https://github.com/truemedian/hzzp commit-2d30bddae3bf1eaecde5144490307604efe76f2a +git https://github.com/alexnask/iguanaTLS commit-0d39a361639ad5469f8e4dcdaea35446bbe54b48 +git https://github.com/MasterQ32/zig-network commit-b9c91769d8ebd626c8e45b2abb05cbc28ccc50da +git https://github.com/MasterQ32/zig-uri commit-52cdd2061bec0579519f0d30280597f3a1db8b75 diff --git a/zigmod.sum b/zigmod.sum index 534f098..8831c35 100644 --- a/zigmod.sum +++ b/zigmod.sum @@ -1,2 +1,8 @@ blake3-e59408623c62ac2b958cf33f848cb9dccc87631e8ca98e479cf9870694a4771d v/git/github.com/Hejsil/zig-clap/branch-zig-master -blake3-605156fee0fdf3be71878c38e7bb1e1934ba4bafeaf5fde535685bfa924345ea git/github.com/vrischmann/zig-sqlite +blake3-ce341d796da35b5db8663df0a35cbce6c9cc39ead510e805ea8a04593731b303 git/github.com/vrischmann/zig-sqlite +blake3-1893709ffc6359c5f9cd2f9409abccf78a94ed37bb2c6dd075c603356d17c94b git/github.com/nektro/zig-json +blake3-e3072f7fb47e86d53c9a1879e254ba1af55941153fd5f6752ec659b2f14854c9 git/github.com/truemedian/zfetch +blake3-98982125d0fbedc62e179e62081d2797a2b8a3623c42f9fd5d72cd56d6350714 git/github.com/truemedian/hzzp +blake3-e6901bd7432450d5b22b01880cc7fa3fa2433e766a527206f18b29c67c1349bb git/github.com/alexnask/iguanaTLS +blake3-21f91e48333ac0ca7f4704c96352831c25216e7056d02ce24de95d03fc942246 git/github.com/MasterQ32/zig-network +blake3-030ebb03f1ed21122e681b06786bea6f2f1b810e8eb9f2029d0eee4f4fb3103f git/github.com/MasterQ32/zig-uri