lonjil
/
derploader
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Whoa, signed!

frunk
lonjil 2021-11-17 00:28:40 +01:00
parent 1213adfc1f
commit c1a646d38f
No known key found for this signature in database
4 changed files with 159 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
build.zig
View File

@ -30,6 +30,7 @@ pub fn build(b: *std.build.Builder) void {
}
deps.addAllTo(exe);
exe.linkLibC();
exe.single_threaded = true;
exe.linkSystemLibrary("libcurl");
exe.setTarget(target);
exe.setBuildMode(mode);

247
src/main.zig
View File

@ -29,6 +29,8 @@ const params = [_]clap.Param(clap.Help){
clap.parseParam("-O <ORDER> Order searches by ORDER, ascending.") catch unreachable,
clap.parseParam("-p <PAGE> Start from page PAGE.") catch unreachable,
clap.parseParam("-P <PAGES> Stop after PAGES pages.") catch unreachable,
clap.parseParam("--migrate Update an old database.") catch unreachable,
clap.parseParam("--yolo PRAGMA synchronous = OFF") catch unreachable,
};
fn printFullUsage(w: anytype) !void {
@ -49,19 +51,18 @@ fn curlErrorReport(str: []const u8, code: curl.CURLcode) void {
const create =
\\CREATE TABLE IF NOT EXISTS image(
\\ id INTEGER UNIQUE,
\\ iid INTEGER PRIMARY KEY,
\\ eid INTEGER UNIQUE,
\\ metadata TEXT,
\\ image BLOB,
\\ thumb BLOB,
\\ full_url TEXT GENERATED ALWAYS AS
\\ (json_extract(metadata, '$.image.representations.full')) VIRTUAL,
\\ thumb_url TEXT GENERATED ALWAYS AS
\\ (json_extract(metadata, '$.image.representations.thumb')) VIRTUAL,
\\ extension TEXT GENERATED ALWAYS AS
\\ (json_extract(metadata, '$.image.format')) VIRTUAL,
\\ hash_full TEXT,
\\ hash_thumb TEXT,
\\ hash_meta TEXT
\\ hash_meta TEXT,
\\ image_id INTEGER,
\\ thumb_id INTEGER
\\);
;
@ -125,18 +126,50 @@ pub fn main() anyerror!void {
return;
}
var db: sqlite.Db = undefined;
const filename = "test.db3";
try db.init(.{
var db = try sqlite.Db.init(.{
.mode = sqlite.Db.Mode{ .File = filename },
.open_flags = .{
.write = true,
.create = true,
},
.threading_mode = .Serialized,
.threading_mode = .SingleThread,
});
db.exec(create, .{}, .{}) catch sqliteErrorReport("Couldn't create table", &db);
if (args.flag("--migrate")) {
if (args.flag("--yolo")) try db.exec("PRAGMA synchronous=0;", .{}, .{});
try db.exec("BEGIN IMMEDIATE;", .{}, .{});
errdefer db.exec("ROLLBACK;", .{}, .{}) catch std.debug.panic("SQLite database errored trying to roll back. Have fun!", .{});
var stmt = try db.prepare("SELECT ROWID FROM image;");
defer stmt.deinit();
var iter = try stmt.iterator(u64, .{});
while (try iter.next(.{})) |id| {
try db.exec(
\\INSERT INTO blob (data, hash)
\\ SELECT image, hash_full
\\ FROM image WHERE ROWID = ?;
, .{}, .{id});
try db.exec(
\\UPDATE image
\\ SET image_id = last_insert_rowid()
\\ WHERE ROWID = ?
, .{}, .{id});
try db.exec(
\\INSERT INTO blob (data, hash)
\\ SELECT thumb, hash_thumb
\\ FROM image WHERE ROWID = ?;
, .{}, .{id});
try db.exec(
\\UPDATE image
\\ SET thumb_id = last_insert_rowid()
\\ WHERE ROWID = ?
, .{}, .{id});
}
db.exec("COMMIT;", .{}, .{}) catch std.debug.panic("SQLite database errored trying to commit. Not *terrible*, I guess.", .{});
return;
}
var ret = curl.curl_global_init(curl.CURL_GLOBAL_ALL);
if (ret != curl.CURLE_OK) {
log.err("cURL global init failure: {s}", .{curl.curl_easy_strerror(ret)});
@ -227,14 +260,6 @@ pub fn main() anyerror!void {
}
};
const sort_by = sort_descending orelse sort_ascending orelse "id";
var headers = zfetch.Headers.init(alloc);
try headers.appendValue("Accept", "application/json");
try headers.appendValue("User-Agent", "Derpiloader 0.1 (linux)");
var req = try zfetch.Request.init(
alloc,
"https://derpibooru.org",
null,
);
var page = if (args.option("-p")) |page| blk: {
break :blk std.fmt.parseInt(u64, page, 10) catch {
log.err("Page must be a positive integer.", .{});
@ -249,36 +274,34 @@ pub fn main() anyerror!void {
};
} else 0;
var buf = std.ArrayList(u8).init(alloc);
const kkey: []const u8 = key orelse "";
const aaaa: []const u8 = if (key) |_| "&key=" else "";
var fuckme = std.ArrayList(u8).init(alloc);
defer fuckme.deinit();
for (searches) |search| {
const esearch = try uri.escapeString(alloc, search);
var pages: u64 = 0;
log.info("Iterating over search \"{s}\", starting on page {d}.", .{ search, page });
while (true) foo: {
fuck: while (true) {
pages += 1;
if (maxPages > 0 and pages == maxPages) {
return;
}
log.info("Doing page {d}, {d}/{d}.", .{ page, pages, maxPages });
buf.clearRetainingCapacity();
const url = try std.fmt.allocPrint(
alloc,
_ = try std.fmt.bufPrintZ(
urlbuf[0..],
api_base ++ "/search/images?q={s}&page={d}&sd={s}&sf={s}&per_page=50{s}{s}",
.{
esearch, page, sort_order, sort_by, aaaa, kkey,
},
);
try req.reset(url);
try req.do(.GET, headers, null);
const reader = req.reader();
try reader.readAllArrayList(&buf, 500 * 1024);
const val = try json.parse(alloc, buf.items);
fuckme.clearRetainingCapacity();
try easyFetch(handle, &urlbuf, &fuckme);
const val = try json.parse(alloc, fuckme.items);
if (val.get(.{"images"})) |aa| {
if (unwrap(aa, .Array)) |images| {
for (images) |i| {
var buffer: [1024 * 10]u8 = undefined;
var buffer: [1024 * 20]u8 = undefined;
const pid = unwrap(i.get("id") orelse {
log.err("Malformed reply from Derpi.", .{});
return;
@ -312,7 +335,7 @@ pub fn main() anyerror!void {
if (images.len == 50) {
page += 1;
} else {
break :foo;
break :fuck;
}
}
}
@ -413,9 +436,9 @@ fn registerID(db: *sqlite.Db, id: u64) !void {
log.info("Registering ID {d}.", .{id});
const foo = db.one(
bool,
"SELECT true FROM image WHERE id = ?;",
"SELECT true FROM image WHERE eid = ?;",
.{},
.{ .id = id },
.{ .eid = id },
) catch {
sqliteErrorReport("SQLite error while checking if ID already present", db);
return error.GO_ON;
@ -428,7 +451,7 @@ fn registerID(db: *sqlite.Db, id: u64) !void {
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
db.exec(
\\INSERT OR ROLLBACK
\\ INTO image (id)
\\ INTO image (eid)
\\ VALUES (?);
, .{}, .{ .id = id }) catch {
sqliteErrorReport("Couldn't insert ID into database.", db);
@ -449,9 +472,9 @@ fn storeMetadata(
log.info("Storing metadata for ID {d}.", .{id});
const foobar = db.one(
bool,
"SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;",
"SELECT true FROM image WHERE eid = ? AND metadata IS NOT NULL;",
.{},
.{ .id = id },
.{ .eid = id },
) catch {
sqliteErrorReport("SQLite error while checking for metadata precence.", db);
return error.GO_ON;
@ -468,12 +491,12 @@ fn storeMetadata(
db.exec(
\\INSERT OR ROLLBACK
\\ INTO
\\ image (id, metadata)
\\ image (eid, metadata)
\\ VALUES (?, ?)
\\ ON CONFLICT (id)
\\ ON CONFLICT (eid)
\\ DO UPDATE
\\ SET metadata=excluded.metadata;
, .{}, .{ .id = id, .metadata = metadata }) catch {
, .{}, .{ .eid = id, .metadata = metadata }) catch {
sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db);
return error.GO_ON;
};
@ -482,7 +505,7 @@ fn storeMetadata(
return error.GO_ON;
};
db.exec(
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE eid = ?",
.{},
.{ hash_buf2[0..], id },
) catch {
@ -505,9 +528,9 @@ fn getMetadata(
log.info("Downloading metadata for ID {d}.", .{id});
const foobar = db.one(
bool,
"SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;",
"SELECT true FROM image WHERE eid = ? AND metadata IS NOT NULL;",
.{},
.{ .id = id },
.{ .eid = id },
) catch {
sqliteErrorReport("SQLite error while checking for metadata precence.", db);
return error.GO_ON;
@ -523,7 +546,7 @@ fn getMetadata(
);
easyFetch(handle, &urlbuf, resp) catch {
log.info("Failed to download metadata for ID {d}.", .{id});
return error.GO_ON;
return error.FATAL;
};
const valid = std.json.validate(resp.items);
@ -533,12 +556,12 @@ fn getMetadata(
db.exec(
\\INSERT OR ROLLBACK
\\ INTO
\\ image (id, metadata)
\\ image (eid, metadata)
\\ VALUES (?, ?)
\\ ON CONFLICT (id)
\\ ON CONFLICT (eid)
\\ DO UPDATE
\\ SET metadata=excluded.metadata;
, .{}, .{ .id = id, .metadata = resp.items }) catch {
, .{}, .{ .eid = id, .metadata = resp.items }) catch {
sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db);
return error.GO_ON;
};
@ -547,7 +570,7 @@ fn getMetadata(
return error.GO_ON;
};
db.exec(
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE eid = ?",
.{},
.{ hash_buf2[0..], id },
) catch {
@ -578,9 +601,9 @@ fn getImage(
thumb_url: ?[:0]u8,
},
alloc,
"SELECT full_url, thumb_url FROM image WHERE id = ?",
"SELECT full_url, thumb_url FROM image WHERE eid = ?",
.{},
.{ .id = id },
.{ .eid = id },
) catch {
sqliteErrorReport("SQLite error while getting image URLs", db);
return error.GO_ON;
@ -590,7 +613,54 @@ fn getImage(
defer alloc.free(url);
const skipper = db.one(bool,
\\SELECT true FROM image
\\ WHERE id = ? AND image IS NOT NULL;
\\ WHERE eid = ? AND image_id IS NOT NULL;
, .{}, .{id}) catch {
sqliteErrorReport("SQLite error while checking if image is already downloaded", db);
return error.GO_ON;
};
if (skipper) |_| {
log.info("Image for ID {d} already downloaded.", .{id});
break :blk;
}
defer {
resp.clearRetainingCapacity();
std.mem.set(u8, hash_buf[0..], 0);
std.mem.set(u8, hash_buf2[0..], 0);
}
easyFetch(handle, url, resp) catch {
log.info("Failed to download fullsize image for ID {d}", .{id});
return error.FATAL;
};
hashit(resp.items) catch |err| {
log.err("Couldn't hash image for ID {d}: {s}", .{ id, err });
return error.GO_ON;
};
try db.exec("BEGIN IMMEDIATE;", .{}, .{});
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
try db.exec(
\\INSERT INTO blob (data, hash)
\\ VALUES (?, ?);
, .{}, .{ resp.items, hash_buf2[0..] });
db.exec(
\\UPDATE OR ROLLBACK image
\\ SET image_id = last_insert_rowid() WHERE eid = ?
,
.{},
.{id},
) catch {
sqliteErrorReport("Couldn't add image to DB.", db);
return error.GO_ON;
};
db.exec("COMMIT", .{}, .{}) catch {
sqliteErrorReport("FATAL: couldn't commit database", db);
return error.FATAL;
};
}
if (res.thumb_url) |url| blk: {
defer alloc.free(url);
const skipper = db.one(bool,
\\SELECT true FROM image
\\ WHERE eid = ? AND thumb_id IS NOT NULL;
, .{}, .{id}) catch {
sqliteErrorReport("SQLite error while checking if image is already downloaded", db);
return error.GO_ON;
@ -603,79 +673,24 @@ fn getImage(
log.info("Failed to download fullsize image for ID {d}", .{id});
return error.FATAL;
};
try db.exec("BEGIN IMMEDIATE;", .{}, .{});
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
db.exec(
"UPDATE OR ROLLBACK image SET image = ? WHERE id = ?",
.{},
.{
.image = resp.items,
.id = id,
},
) catch {
sqliteErrorReport("Couldn't add image to DB.", db);
return error.GO_ON;
};
hashit(resp.items) catch |err| {
log.err("Couldn't hash image for ID {d}: {s}", .{ id, err });
return error.GO_ON;
};
db.exec(
"UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?",
.{},
.{ hash_buf2[0..], id },
) catch {
sqliteErrorReport("Couldn't set iamge hash", db);
return error.GO_ON;
};
db.exec("COMMIT", .{}, .{}) catch {
sqliteErrorReport("FATAL: couldn't commit database", db);
return error.FATAL;
};
resp.clearRetainingCapacity();
std.mem.set(u8, hash_buf[0..], 0);
std.mem.set(u8, hash_buf2[0..], 0);
}
if (res.thumb_url) |url| blk: {
defer alloc.free(url);
const skipper = db.one(bool,
\\SELECT true FROM image
\\ WHERE id = ? AND thumb IS NOT NULL;
, .{}, .{id}) catch {
sqliteErrorReport("SQLite error while checking if thumb is already downloaded", db);
return error.GO_ON;
};
if (skipper) |_| {
log.info("Thumb for ID {d} already downloaded.", .{id});
break :blk;
}
easyFetch(handle, url, resp) catch {
log.info("Failed to download thumbnail image for ID {d}", .{id});
return error.GO_ON;
};
try db.exec("BEGIN IMMEDIATE;", .{}, .{});
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
try db.exec(
\\INSERT INTO blob (data, hash)
\\ VALUES (?, ?);
, .{}, .{ resp.items, hash_buf2[0..] });
db.exec(
"UPDATE OR ROLLBACK image SET thumb = ? WHERE id = ?",
\\UPDATE OR ROLLBACK image
\\ SET thumb_id = last_insert_rowid() WHERE eid = ?
,
.{},
.{
.thumb = resp.items,
.id = id,
},
.{id},
) catch {
sqliteErrorReport("Couldn't add thumb to DB", db);
return error.GO_ON;
};
hashit(resp.items) catch |err| {
log.err("Couldn't hash thumb for ID {d}: {s}", .{ id, err });
return error.GO_ON;
};
db.exec(
"UPDATE OR ROLLBACK image SET hash_thumb = ? WHERE id = ?",
.{},
.{ hash_buf2[0..], id },
) catch {
sqliteErrorReport("Couldn't add thumb hash", db);
sqliteErrorReport("Couldn't add image to DB.", db);
return error.GO_ON;
};
db.exec("COMMIT", .{}, .{}) catch {
@ -693,13 +708,13 @@ fn extractImage(db: *sqlite.Db, id: u64, alloc: *std.mem.Allocator) !void {
log.info("Extracting image for ID {d}.", .{id});
const foo = db.oneAlloc(
struct {
image: ?[:0]u8,
extension: ?[:0]u8,
image: ?[]u8,
extension: ?[]u8,
},
alloc,
"SELECT image, extension FROM image WHERE id = ?",
"SELECT blob.data, image.extension FROM blob, image WHERE eid = ? AND image.image_id = blob.id;",
.{},
.{ .id = id },
.{id},
) catch {
sqliteErrorReport("SQLite error while reading image", db);
return error.GO_ON;

23
zig.mod
View File

@ -6,6 +6,29 @@ dev_dependencies:
- src: git https://github.com/vrischmann/zig-sqlite
c_source_flags:
- -DSQLITE_ENABLE_JSON1
- -DSQLITE_ENABLE_DQS=0
- -DSQLITE_THREADSAFE=0
- -DSQLITE_DEFAULT_MEMSTATUS=0
- -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1
- -DSQLITE_LIKE_DOESNT_MATCH_BLOBS
- -DSQLITE_MAX_EXPR_DEPTH=0
- -DSQLITE_OMIT_DEPRECATED
- -DSQLITE_OMIT_SHARED_CACHE
- -DSQLITE_DEFAULT_FOREIGN_KEYS
- -DHAVE_FDATASYNC
- -DHAVE_ISNAN
- -DSQLITE_USE_URI
- -DSQLITE_ALLOW_URI_AUTHORITY
- -DSQLITE_ENABLE_BYTECODE_VTAB
- -DSQLITE_ENABLE_COLUMN_METADATA
- -DSQLITE_ENABLE_DBPAGE_VTAB
- -DSQLITE_ENABLE_DBSTAT_VTAB
- -DSQLITE_ENABLE_EXPLAIN_COMMENTS
- -DSQLITE_ENABLE_FTS5
- -DSQLITE_ENABLE_MATH_FUNCTIONS
- -DSQLITE_ENABLE_BATCH_ATOMIC_WRITE=1
c_source_files:
- ../../../../../../sqlite-amalgamation-3360000/sqlite3.c
- src: git https://github.com/nektro/zig-json
- src: git https://github.com/truemedian/zfetch
- src: git https://github.com/alexnask/iguanaTLS

8
zigmod.lock
View File

@ -1,9 +1,9 @@
2
git https://github.com/Hejsil/zig-clap commit-ed90e560d9b1144a27562ef62ff7686eb3af029a
git https://github.com/vrischmann/zig-sqlite commit-1adb900dcc816a419a4b67ccae0555ffe33f72c4
git https://github.com/Hejsil/zig-clap commit-c5fb22823a9a4a699acaefc1e9febfee0b8e506c
git https://github.com/vrischmann/zig-sqlite commit-4954c419d379ffbb637904b41d25ef910c6bb02b
git https://github.com/nektro/zig-json commit-72e555fbc0776f2600aee19b01e5ab1855ebec7a
git https://github.com/truemedian/zfetch commit-8bbc7b34cd417794841e1432585334bc969dfe83
git https://github.com/truemedian/hzzp commit-2d30bddae3bf1eaecde5144490307604efe76f2a
git https://github.com/truemedian/zfetch commit-6ba2ba136ec7cfc887811039cd4a7d8a43ba725b
git https://github.com/truemedian/hzzp commit-492107d44caa2676c7b5aa4e934e1e937232d652
git https://github.com/alexnask/iguanaTLS commit-0d39a361639ad5469f8e4dcdaea35446bbe54b48
git https://github.com/MasterQ32/zig-network commit-b9c91769d8ebd626c8e45b2abb05cbc28ccc50da
git https://github.com/MasterQ32/zig-uri commit-52cdd2061bec0579519f0d30280597f3a1db8b75