From e42b042481080d1ce1cd6c3e23ba2166819723c2 Mon Sep 17 00:00:00 2001 From: Lonnie Date: Sun, 1 Aug 2021 03:48:44 +0200 Subject: [PATCH] Currently dysfunctional on AMD64 Linux due to bus error during `zig build`. --- .gitignore | 2 + build.zig | 38 ++++++++ source-me.sh | 1 + src/main.zig | 265 +++++++++++++++++++++++++++++++++++++++++++++++++++ zig.mod | 8 ++ zigmod.lock | 3 + zigmod.sum | 2 + 7 files changed, 319 insertions(+) create mode 100644 .gitignore create mode 100644 build.zig create mode 100644 source-me.sh create mode 100644 src/main.zig create mode 100644 zig.mod create mode 100644 zigmod.lock create mode 100644 zigmod.sum diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8e6c279 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +secrets/ +*.db3 diff --git a/build.zig b/build.zig new file mode 100644 index 0000000..b214631 --- /dev/null +++ b/build.zig @@ -0,0 +1,38 @@ +const std = @import("std"); +const deps = @import("./deps.zig"); + +pub fn build(b: *std.build.Builder) void { + // Standard target options allows the person running `zig build` to choose + // what target to build for. Here we do not override the defaults, which + // means any target is allowed, and the default is native. Other options + // for restricting supported target set are available. + var target = b.standardTargetOptions( + .{ + //.default_target = .{ + // .abi = .musl, + //}, + }, + ); + if (target.isGnuLibC()) target.setGnuLibCVersion(2, 28, 0); + + // Standard release options allow the person running `zig build` to select + // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. + const mode = b.standardReleaseOptions(); + + const exe = b.addExecutable("derploader", "src/main.zig"); + deps.addAllTo(exe); + exe.linkLibC(); + exe.linkSystemLibrary("libcurl"); + exe.setTarget(target); + exe.setBuildMode(mode); + exe.install(); + + const run_cmd = exe.run(); + run_cmd.step.dependOn(b.getInstallStep()); + if (b.args) |args| { + run_cmd.addArgs(args); + } + + const run_step = b.step("run", "Run the app"); + run_step.dependOn(&run_cmd.step); +} diff --git a/source-me.sh b/source-me.sh new file mode 100644 index 0000000..69a9b80 --- /dev/null +++ b/source-me.sh @@ -0,0 +1 @@ +export PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 diff --git a/src/main.zig b/src/main.zig new file mode 100644 index 0000000..0ea4169 --- /dev/null +++ b/src/main.zig @@ -0,0 +1,265 @@ +const std = @import("std"); +const sqlite = @import("sqlite"); +const clap = @import("clap"); +const curl = @cImport({ + @cInclude("curl/curl.h"); +}); + +const log = std.log.scoped(.derploader); + +const params = [_]clap.Param(clap.Help){ + clap.parseParam("-h, --help Display this help and exit.") catch unreachable, + clap.parseParam("-c, --create= Create new database at PATH.") catch unreachable, + clap.parseParam("-m Download metadata for ID image.") catch unreachable, + clap.parseParam("-d Download image data of ID.") catch unreachable, +}; + +fn printFullUsage(w: anytype) !void { + _ = try w.print("{s} ", .{std.os.argv[0]}); + try clap.usage(w, ¶ms); + _ = try w.writeByte('\n'); + try clap.help(w, ¶ms); + return; +} + +fn sqliteErrorReport(str: []const u8, db: *sqlite.Db) void { + log.err("{s}: {}", .{ str, db.getDetailedError() }); +} + +fn curlErrorReport(str: []const u8, code: curl.CURLcode) void { + log.err("{s}: {s} {s}", .{ str, curl.curl_easy_strerror(code), curlerr[0.. :0] }); +} + +const create = + \\CREATE TABLE IF NOT EXISTS image( + \\ id INTEGER UNIQUE, + \\ metadata TEXT, + \\ image BLOB, + \\ thumb BLOB, + \\ full_url TEXT GENERATED ALWAYS AS + \\ (json_extract(metadata, '$.image.representations.full')) VIRTUAL, + \\ thumb_url TEXT GENERATED ALWAYS AS + \\ (json_extract(metadata, '$.image.representations.thumb')) VIRTUAL, + \\ hash_full TEXT, + \\ hash_thumb TEXT, + \\ hash_meta TEXT + \\); +; + +const metatable = + \\CREATE TABLE IF NOT EXISTS derpiloader( + \\ name TEXT, + \\ value + \\); +; + +pub fn insertMeta(db: *sqlite.Db, id: u64, meta: []const u8) !void { + const q = + \\INSERT OR ROLLBACK INTO image (id, metadata) VALUES (?, ?); + ; + try db.exec(q, .{ .id = id, .metadata = meta }); +} + +const api_base = "https://derpibooru.org/api/v1/json"; + +var urlbuf = [_:0]u8{0} ** 512; +var curlerr = [_:0]u8{0} ** (curl.CURL_ERROR_SIZE + 1); + +const hash_prefix = "blake3-"; +var hash_buf = [_]u8{0} ** (std.crypto.hash.Blake3.digest_length); +var hash_buf2 = [_]u8{0} ** (std.crypto.hash.Blake3.digest_length * 2 + hash_prefix[0..].len); + +fn hashit(input: []const u8) !void { + std.crypto.hash.Blake3.hash(input, hash_buf[0..], .{}); + _ = try std.fmt.bufPrint( + hash_buf2[0..], + hash_prefix ++ "{s}", + .{std.fmt.fmtSliceHexLower(hash_buf[0..])}, + ); +} + +pub fn main() anyerror!void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const alloc = &gpa.allocator; + + //const key = try std.process.getEnvVarOwned(alloc, "derpikey"); + + var diag = clap.Diagnostic{}; + var args = clap.parse( + clap.Help, + ¶ms, + .{ .diagnostic = &diag, .allocator = alloc }, + ) catch |err| { + // Report useful error and exit + diag.report(std.io.getStdErr().writer(), err) catch {}; + return; + }; + defer args.deinit(); + + if (args.flag("-h")) { + var w = std.io.getStdOut().writer(); + try printFullUsage(w); + return; + } + + var db: sqlite.Db = undefined; + const filename = "test.db3"; + try db.init(.{ + .mode = sqlite.Db.Mode{ .File = filename }, + .open_flags = .{ + .write = true, + .create = true, + }, + .threading_mode = .Serialized, + }); + db.exec(create, .{}) catch sqliteErrorReport("Couldn't create table", &db); + + var ret = curl.curl_global_init(curl.CURL_GLOBAL_ALL); + if (ret != curl.CURLE_OK) { + log.err("cURL global init failure: {s}", .{curl.curl_easy_strerror(ret)}); + return; + } + defer curl.curl_global_cleanup(); + const handle = curl.curl_easy_init() orelse return error.CURLHandleInitFailed; + defer curl.curl_easy_cleanup(handle); + var response_buffer = std.ArrayList(u8).init(alloc); + defer response_buffer.deinit(); + + _ = curl.curl_easy_setopt(handle, curl.CURLOPT_ERRORBUFFER, &curlerr); + + if (args.option("-m")) |id_str| { + const id = std.fmt.parseInt(u64, id_str, 10) catch { + log.err("Image ID must be a positive integer.", .{}); + return; + }; + const foobar = db.one( + bool, + "SELECT true FROM image WHERE id = ?", + .{}, + .{ .id = id }, + ) catch { + sqliteErrorReport("ID check read error", &db); + return; + }; + if (foobar) |_| { + log.info("Info for id {d} already acquired.", .{id}); + return; + } + _ = try std.fmt.bufPrintZ( + urlbuf[0..], + api_base ++ "/images/{d}", + .{id}, + ); + easyFetch(handle, &urlbuf, &response_buffer) catch return; + //var w = std.io.getStdOut().writer(); + const valid = std.json.validate(response_buffer.items); + + if (valid) { + try db.exec("BEGIN IMMEDIATE;", .{}); + errdefer db.exec("ROLLBACK;", .{}) catch {}; + insertMeta(&db, id, response_buffer.items) catch { + sqliteErrorReport("Can't insert:", &db); + return; + }; + try hashit(response_buffer.items); + db.exec( + "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?", + .{ hash_buf2[0..], id }, + ) catch { + sqliteErrorReport("Couldn't insert", &db); + return; + }; + try db.exec("COMMIT", .{}); + } + } + + if (args.option("-d")) |id_str| { + const id = std.fmt.parseInt(u64, id_str, 10) catch { + log.err("Image ID must be a positive integer.", .{}); + return; + }; + const foobar = db.oneAlloc( + struct { + full_url: ?[:0]u8, + thumb_url: ?[:0]u8, + }, + alloc, + "SELECT full_url, thumb_url FROM image WHERE id = ?", + .{}, + .{ .id = id }, + ) catch { + sqliteErrorReport("ID check read error", &db); + return; + }; + if (foobar) |res| { + if (res.full_url) |url| { + easyFetch(handle, url, &response_buffer) catch return; + try db.exec("BEGIN IMMEDIATE;", .{}); + errdefer db.exec("ROLLBACK;", .{}) catch {}; + db.exec( + "UPDATE OR ROLLBACK image SET image = ? WHERE id = ?", + .{ + .image = sqlite.Blob{ .data = response_buffer.items }, + .id = id, + }, + ) catch { + sqliteErrorReport("Couldn't add image to DB.", &db); + return; + }; + try hashit(response_buffer.items); + db.exec( + "UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?", + .{ hash_buf2[0..], id }, + ) catch { + sqliteErrorReport("Couldn't insert", &db); + return; + }; + try db.exec("COMMIT", .{}); + } + } else { + log.err("No metadata for id {d} available.", .{id}); + return; + } + } +} + +fn easyFetch(handle: *curl.CURL, url: [*:0]const u8, resp: *std.ArrayList(u8)) !void { + var ret = curl.curl_easy_setopt(handle, curl.CURLOPT_URL, url); + if (ret != curl.CURLE_OK) { + curlErrorReport("cURL set url:", ret); + return error.FUCK; + } + ret = curl.curl_easy_setopt(handle, curl.CURLOPT_WRITEFUNCTION, writeToArrayListCallback); + if (ret != curl.CURLE_OK) { + curlErrorReport("cURL set writefunction:", ret); + return error.FUCK; + } + ret = curl.curl_easy_setopt(handle, curl.CURLOPT_WRITEDATA, resp); + if (ret != curl.CURLE_OK) { + curlErrorReport("cURL set writedata:", ret); + return error.FUCK; + } + ret = curl.curl_easy_setopt(handle, curl.CURLOPT_USERAGENT, "Derpiloader 0.1 (linux)"); + if (ret != curl.CURLE_OK) { + curlErrorReport("cURL set user agent:", ret); + return error.FUCK; + } + ret = curl.curl_easy_perform(handle); + if (ret != curl.CURLE_OK) { + curlErrorReport("cURL perform:", ret); + return error.FUCK; + } + log.info("Got {d} bytes", .{resp.items.len}); +} + +fn writeToArrayListCallback( + data: *c_void, + size: c_uint, + nmemb: c_uint, + user_data: *c_void, +) callconv(.C) c_uint { + var buffer = @intToPtr(*std.ArrayList(u8), @ptrToInt(user_data)); + var typed_data = @intToPtr([*]u8, @ptrToInt(data)); + buffer.appendSlice(typed_data[0 .. nmemb * size]) catch return 0; + return nmemb * size; +} diff --git a/zig.mod b/zig.mod new file mode 100644 index 0000000..8009d0e --- /dev/null +++ b/zig.mod @@ -0,0 +1,8 @@ +id: 1m1xcon88qpj9vi17d8yo2743g0pqswk91ep21b0v139bu5s +name: derploader +main: src/main.zig +dev_dependencies: + - src: git https://github.com/Hejsil/zig-clap branch-zig-master + - src: git https://github.com/vrischmann/zig-sqlite + c_source_flags: + - -DSQLITE_ENABLE_JSON1 diff --git a/zigmod.lock b/zigmod.lock new file mode 100644 index 0000000..ff07e15 --- /dev/null +++ b/zigmod.lock @@ -0,0 +1,3 @@ +2 +git https://github.com/Hejsil/zig-clap branch-zig-master +git https://github.com/vrischmann/zig-sqlite commit-fafe666f22590faf82d7c7ce1087858002c004b6 diff --git a/zigmod.sum b/zigmod.sum new file mode 100644 index 0000000..534f098 --- /dev/null +++ b/zigmod.sum @@ -0,0 +1,2 @@ +blake3-e59408623c62ac2b958cf33f848cb9dccc87631e8ca98e479cf9870694a4771d v/git/github.com/Hejsil/zig-clap/branch-zig-master +blake3-605156fee0fdf3be71878c38e7bb1e1934ba4bafeaf5fde535685bfa924345ea git/github.com/vrischmann/zig-sqlite