Currently dysfunctional on AMD64 Linux due to bus error during `zig build`.
Lonnie
commit
e42b042481
|
|
@ -0,0 +1,2 @@
|
|||
secrets/
|
||||
*.db3
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
const std = @import("std");
|
||||
const deps = @import("./deps.zig");
|
||||
|
||||
pub fn build(b: *std.build.Builder) void {
|
||||
// Standard target options allows the person running `zig build` to choose
|
||||
// what target to build for. Here we do not override the defaults, which
|
||||
// means any target is allowed, and the default is native. Other options
|
||||
// for restricting supported target set are available.
|
||||
var target = b.standardTargetOptions(
|
||||
.{
|
||||
//.default_target = .{
|
||||
// .abi = .musl,
|
||||
//},
|
||||
},
|
||||
);
|
||||
if (target.isGnuLibC()) target.setGnuLibCVersion(2, 28, 0);
|
||||
|
||||
// Standard release options allow the person running `zig build` to select
|
||||
// between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall.
|
||||
const mode = b.standardReleaseOptions();
|
||||
|
||||
const exe = b.addExecutable("derploader", "src/main.zig");
|
||||
deps.addAllTo(exe);
|
||||
exe.linkLibC();
|
||||
exe.linkSystemLibrary("libcurl");
|
||||
exe.setTarget(target);
|
||||
exe.setBuildMode(mode);
|
||||
exe.install();
|
||||
|
||||
const run_cmd = exe.run();
|
||||
run_cmd.step.dependOn(b.getInstallStep());
|
||||
if (b.args) |args| {
|
||||
run_cmd.addArgs(args);
|
||||
}
|
||||
|
||||
const run_step = b.step("run", "Run the app");
|
||||
run_step.dependOn(&run_cmd.step);
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
export PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1
|
||||
|
|
@ -0,0 +1,265 @@
|
|||
const std = @import("std");
|
||||
const sqlite = @import("sqlite");
|
||||
const clap = @import("clap");
|
||||
const curl = @cImport({
|
||||
@cInclude("curl/curl.h");
|
||||
});
|
||||
|
||||
const log = std.log.scoped(.derploader);
|
||||
|
||||
const params = [_]clap.Param(clap.Help){
|
||||
clap.parseParam("-h, --help Display this help and exit.") catch unreachable,
|
||||
clap.parseParam("-c, --create=<PATH> Create new database at PATH.") catch unreachable,
|
||||
clap.parseParam("-m <ID> Download metadata for ID image.") catch unreachable,
|
||||
clap.parseParam("-d <ID> Download image data of ID.") catch unreachable,
|
||||
};
|
||||
|
||||
fn printFullUsage(w: anytype) !void {
|
||||
_ = try w.print("{s} ", .{std.os.argv[0]});
|
||||
try clap.usage(w, ¶ms);
|
||||
_ = try w.writeByte('\n');
|
||||
try clap.help(w, ¶ms);
|
||||
return;
|
||||
}
|
||||
|
||||
fn sqliteErrorReport(str: []const u8, db: *sqlite.Db) void {
|
||||
log.err("{s}: {}", .{ str, db.getDetailedError() });
|
||||
}
|
||||
|
||||
fn curlErrorReport(str: []const u8, code: curl.CURLcode) void {
|
||||
log.err("{s}: {s} {s}", .{ str, curl.curl_easy_strerror(code), curlerr[0.. :0] });
|
||||
}
|
||||
|
||||
const create =
|
||||
\\CREATE TABLE IF NOT EXISTS image(
|
||||
\\ id INTEGER UNIQUE,
|
||||
\\ metadata TEXT,
|
||||
\\ image BLOB,
|
||||
\\ thumb BLOB,
|
||||
\\ full_url TEXT GENERATED ALWAYS AS
|
||||
\\ (json_extract(metadata, '$.image.representations.full')) VIRTUAL,
|
||||
\\ thumb_url TEXT GENERATED ALWAYS AS
|
||||
\\ (json_extract(metadata, '$.image.representations.thumb')) VIRTUAL,
|
||||
\\ hash_full TEXT,
|
||||
\\ hash_thumb TEXT,
|
||||
\\ hash_meta TEXT
|
||||
\\);
|
||||
;
|
||||
|
||||
const metatable =
|
||||
\\CREATE TABLE IF NOT EXISTS derpiloader(
|
||||
\\ name TEXT,
|
||||
\\ value
|
||||
\\);
|
||||
;
|
||||
|
||||
pub fn insertMeta(db: *sqlite.Db, id: u64, meta: []const u8) !void {
|
||||
const q =
|
||||
\\INSERT OR ROLLBACK INTO image (id, metadata) VALUES (?, ?);
|
||||
;
|
||||
try db.exec(q, .{ .id = id, .metadata = meta });
|
||||
}
|
||||
|
||||
const api_base = "https://derpibooru.org/api/v1/json";
|
||||
|
||||
var urlbuf = [_:0]u8{0} ** 512;
|
||||
var curlerr = [_:0]u8{0} ** (curl.CURL_ERROR_SIZE + 1);
|
||||
|
||||
const hash_prefix = "blake3-";
|
||||
var hash_buf = [_]u8{0} ** (std.crypto.hash.Blake3.digest_length);
|
||||
var hash_buf2 = [_]u8{0} ** (std.crypto.hash.Blake3.digest_length * 2 + hash_prefix[0..].len);
|
||||
|
||||
fn hashit(input: []const u8) !void {
|
||||
std.crypto.hash.Blake3.hash(input, hash_buf[0..], .{});
|
||||
_ = try std.fmt.bufPrint(
|
||||
hash_buf2[0..],
|
||||
hash_prefix ++ "{s}",
|
||||
.{std.fmt.fmtSliceHexLower(hash_buf[0..])},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn main() anyerror!void {
|
||||
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
|
||||
const alloc = &gpa.allocator;
|
||||
|
||||
//const key = try std.process.getEnvVarOwned(alloc, "derpikey");
|
||||
|
||||
var diag = clap.Diagnostic{};
|
||||
var args = clap.parse(
|
||||
clap.Help,
|
||||
¶ms,
|
||||
.{ .diagnostic = &diag, .allocator = alloc },
|
||||
) catch |err| {
|
||||
// Report useful error and exit
|
||||
diag.report(std.io.getStdErr().writer(), err) catch {};
|
||||
return;
|
||||
};
|
||||
defer args.deinit();
|
||||
|
||||
if (args.flag("-h")) {
|
||||
var w = std.io.getStdOut().writer();
|
||||
try printFullUsage(w);
|
||||
return;
|
||||
}
|
||||
|
||||
var db: sqlite.Db = undefined;
|
||||
const filename = "test.db3";
|
||||
try db.init(.{
|
||||
.mode = sqlite.Db.Mode{ .File = filename },
|
||||
.open_flags = .{
|
||||
.write = true,
|
||||
.create = true,
|
||||
},
|
||||
.threading_mode = .Serialized,
|
||||
});
|
||||
db.exec(create, .{}) catch sqliteErrorReport("Couldn't create table", &db);
|
||||
|
||||
var ret = curl.curl_global_init(curl.CURL_GLOBAL_ALL);
|
||||
if (ret != curl.CURLE_OK) {
|
||||
log.err("cURL global init failure: {s}", .{curl.curl_easy_strerror(ret)});
|
||||
return;
|
||||
}
|
||||
defer curl.curl_global_cleanup();
|
||||
const handle = curl.curl_easy_init() orelse return error.CURLHandleInitFailed;
|
||||
defer curl.curl_easy_cleanup(handle);
|
||||
var response_buffer = std.ArrayList(u8).init(alloc);
|
||||
defer response_buffer.deinit();
|
||||
|
||||
_ = curl.curl_easy_setopt(handle, curl.CURLOPT_ERRORBUFFER, &curlerr);
|
||||
|
||||
if (args.option("-m")) |id_str| {
|
||||
const id = std.fmt.parseInt(u64, id_str, 10) catch {
|
||||
log.err("Image ID must be a positive integer.", .{});
|
||||
return;
|
||||
};
|
||||
const foobar = db.one(
|
||||
bool,
|
||||
"SELECT true FROM image WHERE id = ?",
|
||||
.{},
|
||||
.{ .id = id },
|
||||
) catch {
|
||||
sqliteErrorReport("ID check read error", &db);
|
||||
return;
|
||||
};
|
||||
if (foobar) |_| {
|
||||
log.info("Info for id {d} already acquired.", .{id});
|
||||
return;
|
||||
}
|
||||
_ = try std.fmt.bufPrintZ(
|
||||
urlbuf[0..],
|
||||
api_base ++ "/images/{d}",
|
||||
.{id},
|
||||
);
|
||||
easyFetch(handle, &urlbuf, &response_buffer) catch return;
|
||||
//var w = std.io.getStdOut().writer();
|
||||
const valid = std.json.validate(response_buffer.items);
|
||||
|
||||
if (valid) {
|
||||
try db.exec("BEGIN IMMEDIATE;", .{});
|
||||
errdefer db.exec("ROLLBACK;", .{}) catch {};
|
||||
insertMeta(&db, id, response_buffer.items) catch {
|
||||
sqliteErrorReport("Can't insert:", &db);
|
||||
return;
|
||||
};
|
||||
try hashit(response_buffer.items);
|
||||
db.exec(
|
||||
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
|
||||
.{ hash_buf2[0..], id },
|
||||
) catch {
|
||||
sqliteErrorReport("Couldn't insert", &db);
|
||||
return;
|
||||
};
|
||||
try db.exec("COMMIT", .{});
|
||||
}
|
||||
}
|
||||
|
||||
if (args.option("-d")) |id_str| {
|
||||
const id = std.fmt.parseInt(u64, id_str, 10) catch {
|
||||
log.err("Image ID must be a positive integer.", .{});
|
||||
return;
|
||||
};
|
||||
const foobar = db.oneAlloc(
|
||||
struct {
|
||||
full_url: ?[:0]u8,
|
||||
thumb_url: ?[:0]u8,
|
||||
},
|
||||
alloc,
|
||||
"SELECT full_url, thumb_url FROM image WHERE id = ?",
|
||||
.{},
|
||||
.{ .id = id },
|
||||
) catch {
|
||||
sqliteErrorReport("ID check read error", &db);
|
||||
return;
|
||||
};
|
||||
if (foobar) |res| {
|
||||
if (res.full_url) |url| {
|
||||
easyFetch(handle, url, &response_buffer) catch return;
|
||||
try db.exec("BEGIN IMMEDIATE;", .{});
|
||||
errdefer db.exec("ROLLBACK;", .{}) catch {};
|
||||
db.exec(
|
||||
"UPDATE OR ROLLBACK image SET image = ? WHERE id = ?",
|
||||
.{
|
||||
.image = sqlite.Blob{ .data = response_buffer.items },
|
||||
.id = id,
|
||||
},
|
||||
) catch {
|
||||
sqliteErrorReport("Couldn't add image to DB.", &db);
|
||||
return;
|
||||
};
|
||||
try hashit(response_buffer.items);
|
||||
db.exec(
|
||||
"UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?",
|
||||
.{ hash_buf2[0..], id },
|
||||
) catch {
|
||||
sqliteErrorReport("Couldn't insert", &db);
|
||||
return;
|
||||
};
|
||||
try db.exec("COMMIT", .{});
|
||||
}
|
||||
} else {
|
||||
log.err("No metadata for id {d} available.", .{id});
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn easyFetch(handle: *curl.CURL, url: [*:0]const u8, resp: *std.ArrayList(u8)) !void {
|
||||
var ret = curl.curl_easy_setopt(handle, curl.CURLOPT_URL, url);
|
||||
if (ret != curl.CURLE_OK) {
|
||||
curlErrorReport("cURL set url:", ret);
|
||||
return error.FUCK;
|
||||
}
|
||||
ret = curl.curl_easy_setopt(handle, curl.CURLOPT_WRITEFUNCTION, writeToArrayListCallback);
|
||||
if (ret != curl.CURLE_OK) {
|
||||
curlErrorReport("cURL set writefunction:", ret);
|
||||
return error.FUCK;
|
||||
}
|
||||
ret = curl.curl_easy_setopt(handle, curl.CURLOPT_WRITEDATA, resp);
|
||||
if (ret != curl.CURLE_OK) {
|
||||
curlErrorReport("cURL set writedata:", ret);
|
||||
return error.FUCK;
|
||||
}
|
||||
ret = curl.curl_easy_setopt(handle, curl.CURLOPT_USERAGENT, "Derpiloader 0.1 (linux)");
|
||||
if (ret != curl.CURLE_OK) {
|
||||
curlErrorReport("cURL set user agent:", ret);
|
||||
return error.FUCK;
|
||||
}
|
||||
ret = curl.curl_easy_perform(handle);
|
||||
if (ret != curl.CURLE_OK) {
|
||||
curlErrorReport("cURL perform:", ret);
|
||||
return error.FUCK;
|
||||
}
|
||||
log.info("Got {d} bytes", .{resp.items.len});
|
||||
}
|
||||
|
||||
fn writeToArrayListCallback(
|
||||
data: *c_void,
|
||||
size: c_uint,
|
||||
nmemb: c_uint,
|
||||
user_data: *c_void,
|
||||
) callconv(.C) c_uint {
|
||||
var buffer = @intToPtr(*std.ArrayList(u8), @ptrToInt(user_data));
|
||||
var typed_data = @intToPtr([*]u8, @ptrToInt(data));
|
||||
buffer.appendSlice(typed_data[0 .. nmemb * size]) catch return 0;
|
||||
return nmemb * size;
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
id: 1m1xcon88qpj9vi17d8yo2743g0pqswk91ep21b0v139bu5s
|
||||
name: derploader
|
||||
main: src/main.zig
|
||||
dev_dependencies:
|
||||
- src: git https://github.com/Hejsil/zig-clap branch-zig-master
|
||||
- src: git https://github.com/vrischmann/zig-sqlite
|
||||
c_source_flags:
|
||||
- -DSQLITE_ENABLE_JSON1
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
2
|
||||
git https://github.com/Hejsil/zig-clap branch-zig-master
|
||||
git https://github.com/vrischmann/zig-sqlite commit-fafe666f22590faf82d7c7ce1087858002c004b6
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
blake3-e59408623c62ac2b958cf33f848cb9dccc87631e8ca98e479cf9870694a4771d v/git/github.com/Hejsil/zig-clap/branch-zig-master
|
||||
blake3-605156fee0fdf3be71878c38e7bb1e1934ba4bafeaf5fde535685bfa924345ea git/github.com/vrischmann/zig-sqlite
|
||||
Loading…
Reference in New Issue