lonjil
/
derploader
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Stuff.

frunk
lonjil 2021-12-02 02:46:59 +01:00
parent c1a646d38f
commit 3038309758
No known key found for this signature in database
4 changed files with 89 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
build.zig
View File

@ -1,6 +1,10 @@
const std = @import("std"); const std = @import("std");
const deps = @import("./deps.zig"); const deps = @import("./deps.zig");
fn blah(val: anytype, val2: anytype) bool {
return val != null and val.? == val2;
}
pub fn build(b: *std.build.Builder) void { pub fn build(b: *std.build.Builder) void {
// Standard target options allows the person running `zig build` to choose // Standard target options allows the person running `zig build` to choose
// what target to build for. Here we do not override the defaults, which // what target to build for. Here we do not override the defaults, which
@ -29,9 +33,21 @@ pub fn build(b: *std.build.Builder) void {
exe.linkage = .static; exe.linkage = .static;
} }
deps.addAllTo(exe); deps.addAllTo(exe);
exe.linkLibC(); //exe.linkLibC();
exe.single_threaded = true; exe.single_threaded = true;
if (blah(static, true)) {
exe.addIncludeDir("/usr/include/");
exe.addObjectFile("/usr/lib/libcurl.a");
exe.addObjectFile("/usr/lib/libssl.a");
exe.addObjectFile("/usr/lib/libcrypto.a");
exe.addObjectFile("/usr/lib/libnghttp2.a");
exe.addObjectFile("/usr/lib/libbrotlicommon.a");
exe.addObjectFile("/usr/lib/libbrotlidec.a");
exe.addObjectFile("/usr/lib/libbrotlienc.a");
exe.addObjectFile("/lib/libz.a");
} else {
exe.linkSystemLibrary("libcurl"); exe.linkSystemLibrary("libcurl");
}
exe.setTarget(target); exe.setTarget(target);
exe.setBuildMode(mode); exe.setBuildMode(mode);
exe.install(); exe.install();

124
src/main.zig
View File

@ -5,8 +5,6 @@ const curl = @cImport({
@cInclude("curl/curl.h"); @cInclude("curl/curl.h");
}); });
const zfetch = @import("zfetch");
const tls = @import("iguanaTLS");
const uri = @import("uri"); const uri = @import("uri");
const json = @import("json"); const json = @import("json");
@ -106,8 +104,6 @@ pub fn main() anyerror!void {
var gpa = std.heap.GeneralPurposeAllocator(.{}){}; var gpa = std.heap.GeneralPurposeAllocator(.{}){};
const alloc = &gpa.allocator; const alloc = &gpa.allocator;
//const key = try std.process.getEnvVarOwned(alloc, "derpikey");
var diag = clap.Diagnostic{}; var diag = clap.Diagnostic{};
var args = clap.parse( var args = clap.parse(
clap.Help, clap.Help,
@ -135,7 +131,7 @@ pub fn main() anyerror!void {
}, },
.threading_mode = .SingleThread, .threading_mode = .SingleThread,
}); });
db.exec(create, .{}, .{}) catch sqliteErrorReport("Couldn't create table", &db); db.exec(create, .{}, .{ 0, 0, 0 }) catch sqliteErrorReport("Couldn't create table", &db);
if (args.flag("--migrate")) { if (args.flag("--migrate")) {
if (args.flag("--yolo")) try db.exec("PRAGMA synchronous=0;", .{}, .{}); if (args.flag("--yolo")) try db.exec("PRAGMA synchronous=0;", .{}, .{});
@ -436,9 +432,9 @@ fn registerID(db: *sqlite.Db, id: u64) !void {
log.info("Registering ID {d}.", .{id}); log.info("Registering ID {d}.", .{id});
const foo = db.one( const foo = db.one(
bool, bool,
"SELECT true FROM image WHERE eid = ?;", "SELECT true FROM image WHERE id = ?;",
.{}, .{},
.{ .eid = id }, .{ .id = id },
) catch { ) catch {
sqliteErrorReport("SQLite error while checking if ID already present", db); sqliteErrorReport("SQLite error while checking if ID already present", db);
return error.GO_ON; return error.GO_ON;
@ -451,7 +447,7 @@ fn registerID(db: *sqlite.Db, id: u64) !void {
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
db.exec( db.exec(
\\INSERT OR ROLLBACK \\INSERT OR ROLLBACK
\\ INTO image (eid) \\ INTO image (id)
\\ VALUES (?); \\ VALUES (?);
, .{}, .{ .id = id }) catch { , .{}, .{ .id = id }) catch {
sqliteErrorReport("Couldn't insert ID into database.", db); sqliteErrorReport("Couldn't insert ID into database.", db);
@ -472,9 +468,9 @@ fn storeMetadata(
log.info("Storing metadata for ID {d}.", .{id}); log.info("Storing metadata for ID {d}.", .{id});
const foobar = db.one( const foobar = db.one(
bool, bool,
"SELECT true FROM image WHERE eid = ? AND metadata IS NOT NULL;", "SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;",
.{}, .{},
.{ .eid = id }, .{ .id = id },
) catch { ) catch {
sqliteErrorReport("SQLite error while checking for metadata precence.", db); sqliteErrorReport("SQLite error while checking for metadata precence.", db);
return error.GO_ON; return error.GO_ON;
@ -491,12 +487,12 @@ fn storeMetadata(
db.exec( db.exec(
\\INSERT OR ROLLBACK \\INSERT OR ROLLBACK
\\ INTO \\ INTO
\\ image (eid, metadata) \\ image (id, metadata)
\\ VALUES (?, ?) \\ VALUES (?, ?)
\\ ON CONFLICT (eid) \\ ON CONFLICT (id)
\\ DO UPDATE \\ DO UPDATE
\\ SET metadata=excluded.metadata; \\ SET metadata=excluded.metadata;
, .{}, .{ .eid = id, .metadata = metadata }) catch { , .{}, .{ .id = id, .metadata = metadata }) catch {
sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db); sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db);
return error.GO_ON; return error.GO_ON;
}; };
@ -505,7 +501,7 @@ fn storeMetadata(
return error.GO_ON; return error.GO_ON;
}; };
db.exec( db.exec(
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE eid = ?", "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
.{}, .{},
.{ hash_buf2[0..], id }, .{ hash_buf2[0..], id },
) catch { ) catch {
@ -528,9 +524,9 @@ fn getMetadata(
log.info("Downloading metadata for ID {d}.", .{id}); log.info("Downloading metadata for ID {d}.", .{id});
const foobar = db.one( const foobar = db.one(
bool, bool,
"SELECT true FROM image WHERE eid = ? AND metadata IS NOT NULL;", "SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;",
.{}, .{},
.{ .eid = id }, .{ .id = id },
) catch { ) catch {
sqliteErrorReport("SQLite error while checking for metadata precence.", db); sqliteErrorReport("SQLite error while checking for metadata precence.", db);
return error.GO_ON; return error.GO_ON;
@ -556,12 +552,12 @@ fn getMetadata(
db.exec( db.exec(
\\INSERT OR ROLLBACK \\INSERT OR ROLLBACK
\\ INTO \\ INTO
\\ image (eid, metadata) \\ image (id, metadata)
\\ VALUES (?, ?) \\ VALUES (?, ?)
\\ ON CONFLICT (eid) \\ ON CONFLICT (id)
\\ DO UPDATE \\ DO UPDATE
\\ SET metadata=excluded.metadata; \\ SET metadata=excluded.metadata;
, .{}, .{ .eid = id, .metadata = resp.items }) catch { , .{}, .{ .id = id, .metadata = resp.items }) catch {
sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db); sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db);
return error.GO_ON; return error.GO_ON;
}; };
@ -570,7 +566,7 @@ fn getMetadata(
return error.GO_ON; return error.GO_ON;
}; };
db.exec( db.exec(
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE eid = ?", "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
.{}, .{},
.{ hash_buf2[0..], id }, .{ hash_buf2[0..], id },
) catch { ) catch {
@ -601,9 +597,9 @@ fn getImage(
thumb_url: ?[:0]u8, thumb_url: ?[:0]u8,
}, },
alloc, alloc,
"SELECT full_url, thumb_url FROM image WHERE eid = ?", "SELECT full_url, thumb_url FROM image WHERE id = ?",
.{}, .{},
.{ .eid = id }, .{ .id = id },
) catch { ) catch {
sqliteErrorReport("SQLite error while getting image URLs", db); sqliteErrorReport("SQLite error while getting image URLs", db);
return error.GO_ON; return error.GO_ON;
@ -613,7 +609,7 @@ fn getImage(
defer alloc.free(url); defer alloc.free(url);
const skipper = db.one(bool, const skipper = db.one(bool,
\\SELECT true FROM image \\SELECT true FROM image
\\ WHERE eid = ? AND image_id IS NOT NULL; \\ WHERE id = ? AND image IS NOT NULL;
, .{}, .{id}) catch { , .{}, .{id}) catch {
sqliteErrorReport("SQLite error while checking if image is already downloaded", db); sqliteErrorReport("SQLite error while checking if image is already downloaded", db);
return error.GO_ON; return error.GO_ON;
@ -622,75 +618,83 @@ fn getImage(
log.info("Image for ID {d} already downloaded.", .{id}); log.info("Image for ID {d} already downloaded.", .{id});
break :blk; break :blk;
} }
defer {
resp.clearRetainingCapacity();
std.mem.set(u8, hash_buf[0..], 0);
std.mem.set(u8, hash_buf2[0..], 0);
}
easyFetch(handle, url, resp) catch { easyFetch(handle, url, resp) catch {
log.info("Failed to download fullsize image for ID {d}", .{id}); log.info("Failed to download fullsize image for ID {d}", .{id});
return error.FATAL; return error.FATAL;
}; };
try db.exec("BEGIN IMMEDIATE;", .{}, .{});
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
db.exec(
"UPDATE OR ROLLBACK image SET image = ? WHERE id = ?",
.{},
.{
.image = resp.items,
.id = id,
},
) catch {
sqliteErrorReport("Couldn't add image to DB.", db);
return error.GO_ON;
};
hashit(resp.items) catch |err| { hashit(resp.items) catch |err| {
log.err("Couldn't hash image for ID {d}: {s}", .{ id, err }); log.err("Couldn't hash image for ID {d}: {s}", .{ id, err });
return error.GO_ON; return error.GO_ON;
}; };
try db.exec("BEGIN IMMEDIATE;", .{}, .{});
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
try db.exec(
\\INSERT INTO blob (data, hash)
\\ VALUES (?, ?);
, .{}, .{ resp.items, hash_buf2[0..] });
db.exec( db.exec(
\\UPDATE OR ROLLBACK image "UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?",
\\ SET image_id = last_insert_rowid() WHERE eid = ?
,
.{}, .{},
.{id}, .{ hash_buf2[0..], id },
) catch { ) catch {
sqliteErrorReport("Couldn't add image to DB.", db); sqliteErrorReport("Couldn't set iamge hash", db);
return error.GO_ON; return error.GO_ON;
}; };
db.exec("COMMIT", .{}, .{}) catch { db.exec("COMMIT", .{}, .{}) catch {
sqliteErrorReport("FATAL: couldn't commit database", db); sqliteErrorReport("FATAL: couldn't commit database", db);
return error.FATAL; return error.FATAL;
}; };
resp.clearRetainingCapacity();
std.mem.set(u8, hash_buf[0..], 0);
std.mem.set(u8, hash_buf2[0..], 0);
} }
if (res.thumb_url) |url| blk: { if (res.thumb_url) |url| blk: {
defer alloc.free(url); defer alloc.free(url);
const skipper = db.one(bool, const skipper = db.one(bool,
\\SELECT true FROM image \\SELECT true FROM image
\\ WHERE eid = ? AND thumb_id IS NOT NULL; \\ WHERE id = ? AND thumb IS NOT NULL;
, .{}, .{id}) catch { , .{}, .{id}) catch {
sqliteErrorReport("SQLite error while checking if image is already downloaded", db); sqliteErrorReport("SQLite error while checking if thumb is already downloaded", db);
return error.GO_ON; return error.GO_ON;
}; };
if (skipper) |_| { if (skipper) |_| {
log.info("Image for ID {d} already downloaded.", .{id}); log.info("Thumb for ID {d} already downloaded.", .{id});
break :blk; break :blk;
} }
easyFetch(handle, url, resp) catch { easyFetch(handle, url, resp) catch {
log.info("Failed to download fullsize image for ID {d}", .{id}); log.info("Failed to download thumbnail image for ID {d}", .{id});
return error.FATAL;
};
hashit(resp.items) catch |err| {
log.err("Couldn't hash image for ID {d}: {s}", .{ id, err });
return error.GO_ON; return error.GO_ON;
}; };
try db.exec("BEGIN IMMEDIATE;", .{}, .{}); try db.exec("BEGIN IMMEDIATE;", .{}, .{});
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {}; errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
try db.exec(
\\INSERT INTO blob (data, hash)
\\ VALUES (?, ?);
, .{}, .{ resp.items, hash_buf2[0..] });
db.exec( db.exec(
\\UPDATE OR ROLLBACK image "UPDATE OR ROLLBACK image SET thumb = ? WHERE id = ?",
\\ SET thumb_id = last_insert_rowid() WHERE eid = ?
,
.{}, .{},
.{id}, .{
.thumb = resp.items,
.id = id,
},
) catch { ) catch {
sqliteErrorReport("Couldn't add image to DB.", db); sqliteErrorReport("Couldn't add thumb to DB", db);
return error.GO_ON;
};
hashit(resp.items) catch |err| {
log.err("Couldn't hash thumb for ID {d}: {s}", .{ id, err });
return error.GO_ON;
};
db.exec(
"UPDATE OR ROLLBACK image SET hash_thumb = ? WHERE id = ?",
.{},
.{ hash_buf2[0..], id },
) catch {
sqliteErrorReport("Couldn't add thumb hash", db);
return error.GO_ON; return error.GO_ON;
}; };
db.exec("COMMIT", .{}, .{}) catch { db.exec("COMMIT", .{}, .{}) catch {
@ -708,13 +712,13 @@ fn extractImage(db: *sqlite.Db, id: u64, alloc: *std.mem.Allocator) !void {
log.info("Extracting image for ID {d}.", .{id}); log.info("Extracting image for ID {d}.", .{id});
const foo = db.oneAlloc( const foo = db.oneAlloc(
struct { struct {
image: ?[]u8, image: ?[:0]u8,
extension: ?[]u8, extension: ?[:0]u8,
}, },
alloc, alloc,
"SELECT blob.data, image.extension FROM blob, image WHERE eid = ? AND image.image_id = blob.id;", "SELECT image, extension FROM image WHERE id = ?",
.{}, .{},
.{id}, .{ .id = id },
) catch { ) catch {
sqliteErrorReport("SQLite error while reading image", db); sqliteErrorReport("SQLite error while reading image", db);
return error.GO_ON; return error.GO_ON;

4
zig.mod
View File

@ -28,8 +28,6 @@ dev_dependencies:
- -DSQLITE_ENABLE_MATH_FUNCTIONS - -DSQLITE_ENABLE_MATH_FUNCTIONS
- -DSQLITE_ENABLE_BATCH_ATOMIC_WRITE=1 - -DSQLITE_ENABLE_BATCH_ATOMIC_WRITE=1
c_source_files: c_source_files:
- ../../../../../../sqlite-amalgamation-3360000/sqlite3.c - ../../../../../../sqlite/sqlite3.c
- src: git https://github.com/nektro/zig-json - src: git https://github.com/nektro/zig-json
- src: git https://github.com/truemedian/zfetch
- src: git https://github.com/alexnask/iguanaTLS
- src: git https://github.com/MasterQ32/zig-uri/ - src: git https://github.com/MasterQ32/zig-uri/

14
zigmod.lock
View File

@ -1,9 +1,7 @@
2 2
git https://github.com/Hejsil/zig-clap commit-c5fb22823a9a4a699acaefc1e9febfee0b8e506c git https://github.com/Hejsil/zig-clap commit-844c9370bcecf063daff697f296d6ae979190649
git https://github.com/vrischmann/zig-sqlite commit-4954c419d379ffbb637904b41d25ef910c6bb02b git https://github.com/vrischmann/zig-sqlite commit-3dc73fbe5bbe043e31b12637cb693b9bf6ceba95
git https://github.com/nektro/zig-json commit-72e555fbc0776f2600aee19b01e5ab1855ebec7a git https://github.com/nektro/zig-json commit-7f0e661371d41cfdc8b1649ed00e7a750c82e57a
git https://github.com/truemedian/zfetch commit-6ba2ba136ec7cfc887811039cd4a7d8a43ba725b git https://github.com/nektro/zig-extras commit-090ee5f1834af4ed7714101b24d7daec84572390
git https://github.com/truemedian/hzzp commit-492107d44caa2676c7b5aa4e934e1e937232d652 git https://github.com/nektro/zig-range commit-890ca308fe09b3d5c866d5cfb3b3d7a95dbf939f
git https://github.com/alexnask/iguanaTLS commit-0d39a361639ad5469f8e4dcdaea35446bbe54b48 git https://github.com/MasterQ32/zig-uri/ commit-52cdd2061bec0579519f0d30280597f3a1db8b75
git https://github.com/MasterQ32/zig-network commit-b9c91769d8ebd626c8e45b2abb05cbc28ccc50da
git https://github.com/MasterQ32/zig-uri commit-52cdd2061bec0579519f0d30280597f3a1db8b75