lonjil
/
derploader
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: lonjil:c1a646d38fc9046705181256f1b81dbb85241b2c
Branches Tags
lonjil:frunk
...
compare: lonjil:7fdd175c3e25ef84326c7fc3e5994562f4feb0b1
Branches Tags
lonjil:frunk

2 Commits
c1a646d38f ... 7fdd175c3e

Author SHA1 Message Date
lonjil 7fdd175c3e
Test this woodpecker thingy.
continuous-integration/woodpecker the build failed Details
2021-12-02 02:47:14 +01:00
lonjil 3038309758
Stuff. 2021-12-02 02:46:59 +01:00
5 changed files with 117 additions and 73 deletions
Show Stats Expand all files Collapse all files

28
.woodpecker.yml Normal file
View File

@ -0,0 +1,28 @@
platform: linux/amd64
pipeline:
build-x86_64:
image: alpine
environment:
- ZIG_BIN=https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.1815+20e19e75f.tar.xz
- ZIGMOD_BIN=https://github.com/nektro/zigmod/releases/download/v85/zigmod-x86_64-linux
commands:
- apk add --no-cache build-base curl-dev curl-static zlib-static \
openssl-libs-static nghttp2-static brotli-static xz
- wget $ZIG_BIN
- wget $ZIGMOD_BIN
- tar xf zig-linux-x86_64-0.9.0-dev.1815+20e19e75f.tar.xz
- zig=zig-linux-x86_64-0.9.0-dev.1815+20e19e75f/zig
- zigmod=zigmod-x86_64-linux
- zigmod ci
- zig build -Dstatic=true
- echo $CI_BUILD_NUMBER
upload:
image: techknowlogick/drone-b2
secrets: [ backblaze ]
bucket: lonjilcdn
account: 003b9ee84a86c3d0000000004
key: ${backblaze}
source: zig-out/bin/derploader
strip_prefix: zig-out/bin/
target: artifacts/${CI_REPO}/${CI_COMMIT_SHA}-${CI_BUILD_NUMBER}/

20
build.zig
View File

@ -1,6 +1,10 @@
const std = @import("std");
const deps = @import("./deps.zig");
fn blah(val: anytype, val2: anytype) bool {
return val != null and val.? == val2;
}
pub fn build(b: *std.build.Builder) void {
// Standard target options allows the person running `zig build` to choose
// what target to build for. Here we do not override the defaults, which
@ -29,9 +33,21 @@ pub fn build(b: *std.build.Builder) void {
exe.linkage = .static;
}
deps.addAllTo(exe);
exe.linkLibC();
//exe.linkLibC();
exe.single_threaded = true;
exe.linkSystemLibrary("libcurl");
if (blah(static, true)) {
exe.addIncludeDir("/usr/include/");
exe.addObjectFile("/usr/lib/libcurl.a");
exe.addObjectFile("/usr/lib/libssl.a");
exe.addObjectFile("/usr/lib/libcrypto.a");
exe.addObjectFile("/usr/lib/libnghttp2.a");
exe.addObjectFile("/usr/lib/libbrotlicommon.a");
exe.addObjectFile("/usr/lib/libbrotlidec.a");
exe.addObjectFile("/usr/lib/libbrotlienc.a");
exe.addObjectFile("/lib/libz.a");
} else {
exe.linkSystemLibrary("libcurl");
}
exe.setTarget(target);
exe.setBuildMode(mode);
exe.install();

124
src/main.zig
View File

@ -5,8 +5,6 @@ const curl = @cImport({
@cInclude("curl/curl.h");
});
const zfetch = @import("zfetch");
const tls = @import("iguanaTLS");
const uri = @import("uri");
const json = @import("json");
@ -106,8 +104,6 @@ pub fn main() anyerror!void {
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
const alloc = &gpa.allocator;
//const key = try std.process.getEnvVarOwned(alloc, "derpikey");
var diag = clap.Diagnostic{};
var args = clap.parse(
clap.Help,
@ -135,7 +131,7 @@ pub fn main() anyerror!void {
},
.threading_mode = .SingleThread,
});
db.exec(create, .{}, .{}) catch sqliteErrorReport("Couldn't create table", &db);
db.exec(create, .{}, .{ 0, 0, 0 }) catch sqliteErrorReport("Couldn't create table", &db);
if (args.flag("--migrate")) {
if (args.flag("--yolo")) try db.exec("PRAGMA synchronous=0;", .{}, .{});
@ -436,9 +432,9 @@ fn registerID(db: *sqlite.Db, id: u64) !void {
log.info("Registering ID {d}.", .{id});
const foo = db.one(
bool,
"SELECT true FROM image WHERE eid = ?;",
"SELECT true FROM image WHERE id = ?;",
.{},
.{ .eid = id },
.{ .id = id },
) catch {
sqliteErrorReport("SQLite error while checking if ID already present", db);
return error.GO_ON;
@ -451,7 +447,7 @@ fn registerID(db: *sqlite.Db, id: u64) !void {
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
db.exec(
\\INSERT OR ROLLBACK
\\ INTO image (eid)
\\ INTO image (id)
\\ VALUES (?);
, .{}, .{ .id = id }) catch {
sqliteErrorReport("Couldn't insert ID into database.", db);
@ -472,9 +468,9 @@ fn storeMetadata(
log.info("Storing metadata for ID {d}.", .{id});
const foobar = db.one(
bool,
"SELECT true FROM image WHERE eid = ? AND metadata IS NOT NULL;",
"SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;",
.{},
.{ .eid = id },
.{ .id = id },
) catch {
sqliteErrorReport("SQLite error while checking for metadata precence.", db);
return error.GO_ON;
@ -491,12 +487,12 @@ fn storeMetadata(
db.exec(
\\INSERT OR ROLLBACK
\\ INTO
\\ image (eid, metadata)
\\ image (id, metadata)
\\ VALUES (?, ?)
\\ ON CONFLICT (eid)
\\ ON CONFLICT (id)
\\ DO UPDATE
\\ SET metadata=excluded.metadata;
, .{}, .{ .eid = id, .metadata = metadata }) catch {
, .{}, .{ .id = id, .metadata = metadata }) catch {
sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db);
return error.GO_ON;
};
@ -505,7 +501,7 @@ fn storeMetadata(
return error.GO_ON;
};
db.exec(
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE eid = ?",
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
.{},
.{ hash_buf2[0..], id },
) catch {
@ -528,9 +524,9 @@ fn getMetadata(
log.info("Downloading metadata for ID {d}.", .{id});
const foobar = db.one(
bool,
"SELECT true FROM image WHERE eid = ? AND metadata IS NOT NULL;",
"SELECT true FROM image WHERE id = ? AND metadata IS NOT NULL;",
.{},
.{ .eid = id },
.{ .id = id },
) catch {
sqliteErrorReport("SQLite error while checking for metadata precence.", db);
return error.GO_ON;
@ -556,12 +552,12 @@ fn getMetadata(
db.exec(
\\INSERT OR ROLLBACK
\\ INTO
\\ image (eid, metadata)
\\ image (id, metadata)
\\ VALUES (?, ?)
\\ ON CONFLICT (eid)
\\ ON CONFLICT (id)
\\ DO UPDATE
\\ SET metadata=excluded.metadata;
, .{}, .{ .eid = id, .metadata = resp.items }) catch {
, .{}, .{ .id = id, .metadata = resp.items }) catch {
sqliteErrorReport("Couldn't add metadata for ID {d} to database.", db);
return error.GO_ON;
};
@ -570,7 +566,7 @@ fn getMetadata(
return error.GO_ON;
};
db.exec(
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE eid = ?",
"UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
.{},
.{ hash_buf2[0..], id },
) catch {
@ -601,9 +597,9 @@ fn getImage(
thumb_url: ?[:0]u8,
},
alloc,
"SELECT full_url, thumb_url FROM image WHERE eid = ?",
"SELECT full_url, thumb_url FROM image WHERE id = ?",
.{},
.{ .eid = id },
.{ .id = id },
) catch {
sqliteErrorReport("SQLite error while getting image URLs", db);
return error.GO_ON;
@ -613,7 +609,7 @@ fn getImage(
defer alloc.free(url);
const skipper = db.one(bool,
\\SELECT true FROM image
\\ WHERE eid = ? AND image_id IS NOT NULL;
\\ WHERE id = ? AND image IS NOT NULL;
, .{}, .{id}) catch {
sqliteErrorReport("SQLite error while checking if image is already downloaded", db);
return error.GO_ON;
@ -622,75 +618,83 @@ fn getImage(
log.info("Image for ID {d} already downloaded.", .{id});
break :blk;
}
defer {
resp.clearRetainingCapacity();
std.mem.set(u8, hash_buf[0..], 0);
std.mem.set(u8, hash_buf2[0..], 0);
}
easyFetch(handle, url, resp) catch {
log.info("Failed to download fullsize image for ID {d}", .{id});
return error.FATAL;
};
try db.exec("BEGIN IMMEDIATE;", .{}, .{});
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
db.exec(
"UPDATE OR ROLLBACK image SET image = ? WHERE id = ?",
.{},
.{
.image = resp.items,
.id = id,
},
) catch {
sqliteErrorReport("Couldn't add image to DB.", db);
return error.GO_ON;
};
hashit(resp.items) catch |err| {
log.err("Couldn't hash image for ID {d}: {s}", .{ id, err });
return error.GO_ON;
};
try db.exec("BEGIN IMMEDIATE;", .{}, .{});
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
try db.exec(
\\INSERT INTO blob (data, hash)
\\ VALUES (?, ?);
, .{}, .{ resp.items, hash_buf2[0..] });
db.exec(
\\UPDATE OR ROLLBACK image
\\ SET image_id = last_insert_rowid() WHERE eid = ?
,
"UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?",
.{},
.{id},
.{ hash_buf2[0..], id },
) catch {
sqliteErrorReport("Couldn't add image to DB.", db);
sqliteErrorReport("Couldn't set iamge hash", db);
return error.GO_ON;
};
db.exec("COMMIT", .{}, .{}) catch {
sqliteErrorReport("FATAL: couldn't commit database", db);
return error.FATAL;
};
resp.clearRetainingCapacity();
std.mem.set(u8, hash_buf[0..], 0);
std.mem.set(u8, hash_buf2[0..], 0);
}
if (res.thumb_url) |url| blk: {
defer alloc.free(url);
const skipper = db.one(bool,
\\SELECT true FROM image
\\ WHERE eid = ? AND thumb_id IS NOT NULL;
\\ WHERE id = ? AND thumb IS NOT NULL;
, .{}, .{id}) catch {
sqliteErrorReport("SQLite error while checking if image is already downloaded", db);
sqliteErrorReport("SQLite error while checking if thumb is already downloaded", db);
return error.GO_ON;
};
if (skipper) |_| {
log.info("Image for ID {d} already downloaded.", .{id});
log.info("Thumb for ID {d} already downloaded.", .{id});
break :blk;
}
easyFetch(handle, url, resp) catch {
log.info("Failed to download fullsize image for ID {d}", .{id});
return error.FATAL;
};
hashit(resp.items) catch |err| {
log.err("Couldn't hash image for ID {d}: {s}", .{ id, err });
log.info("Failed to download thumbnail image for ID {d}", .{id});
return error.GO_ON;
};
try db.exec("BEGIN IMMEDIATE;", .{}, .{});
errdefer db.exec("ROLLBACK;", .{}, .{}) catch {};
try db.exec(
\\INSERT INTO blob (data, hash)
\\ VALUES (?, ?);
, .{}, .{ resp.items, hash_buf2[0..] });
db.exec(
\\UPDATE OR ROLLBACK image
\\ SET thumb_id = last_insert_rowid() WHERE eid = ?
,
"UPDATE OR ROLLBACK image SET thumb = ? WHERE id = ?",
.{},
.{id},
.{
.thumb = resp.items,
.id = id,
},
) catch {
sqliteErrorReport("Couldn't add image to DB.", db);
sqliteErrorReport("Couldn't add thumb to DB", db);
return error.GO_ON;
};
hashit(resp.items) catch |err| {
log.err("Couldn't hash thumb for ID {d}: {s}", .{ id, err });
return error.GO_ON;
};
db.exec(
"UPDATE OR ROLLBACK image SET hash_thumb = ? WHERE id = ?",
.{},
.{ hash_buf2[0..], id },
) catch {
sqliteErrorReport("Couldn't add thumb hash", db);
return error.GO_ON;
};
db.exec("COMMIT", .{}, .{}) catch {
@ -708,13 +712,13 @@ fn extractImage(db: *sqlite.Db, id: u64, alloc: *std.mem.Allocator) !void {
log.info("Extracting image for ID {d}.", .{id});
const foo = db.oneAlloc(
struct {
image: ?[]u8,
extension: ?[]u8,
image: ?[:0]u8,
extension: ?[:0]u8,
},
alloc,
"SELECT blob.data, image.extension FROM blob, image WHERE eid = ? AND image.image_id = blob.id;",
"SELECT image, extension FROM image WHERE id = ?",
.{},
.{id},
.{ .id = id },
) catch {
sqliteErrorReport("SQLite error while reading image", db);
return error.GO_ON;

4
zig.mod
View File

@ -28,8 +28,6 @@ dev_dependencies:
- -DSQLITE_ENABLE_MATH_FUNCTIONS
- -DSQLITE_ENABLE_BATCH_ATOMIC_WRITE=1
c_source_files:
- ../../../../../../sqlite-amalgamation-3360000/sqlite3.c
- ../../../../../../sqlite/sqlite3.c
- src: git https://github.com/nektro/zig-json
- src: git https://github.com/truemedian/zfetch
- src: git https://github.com/alexnask/iguanaTLS
- src: git https://github.com/MasterQ32/zig-uri/

14
zigmod.lock
View File

@ -1,9 +1,7 @@
2
git https://github.com/Hejsil/zig-clap commit-c5fb22823a9a4a699acaefc1e9febfee0b8e506c
git https://github.com/vrischmann/zig-sqlite commit-4954c419d379ffbb637904b41d25ef910c6bb02b
git https://github.com/nektro/zig-json commit-72e555fbc0776f2600aee19b01e5ab1855ebec7a
git https://github.com/truemedian/zfetch commit-6ba2ba136ec7cfc887811039cd4a7d8a43ba725b
git https://github.com/truemedian/hzzp commit-492107d44caa2676c7b5aa4e934e1e937232d652
git https://github.com/alexnask/iguanaTLS commit-0d39a361639ad5469f8e4dcdaea35446bbe54b48
git https://github.com/MasterQ32/zig-network commit-b9c91769d8ebd626c8e45b2abb05cbc28ccc50da
git https://github.com/MasterQ32/zig-uri commit-52cdd2061bec0579519f0d30280597f3a1db8b75
git https://github.com/Hejsil/zig-clap commit-844c9370bcecf063daff697f296d6ae979190649
git https://github.com/vrischmann/zig-sqlite commit-3dc73fbe5bbe043e31b12637cb693b9bf6ceba95
git https://github.com/nektro/zig-json commit-7f0e661371d41cfdc8b1649ed00e7a750c82e57a
git https://github.com/nektro/zig-extras commit-090ee5f1834af4ed7714101b24d7daec84572390
git https://github.com/nektro/zig-range commit-890ca308fe09b3d5c866d5cfb3b3d7a95dbf939f
git https://github.com/MasterQ32/zig-uri/ commit-52cdd2061bec0579519f0d30280597f3a1db8b75