From 35adaf30098cde8f8f6b55e2d8ec42f957b5b826 Mon Sep 17 00:00:00 2001 From: Bob Farrell Date: Sun, 1 Dec 2024 23:33:14 +0000 Subject: [PATCH] Valkey backend for JetKV --- build.zig.zon | 8 +- demo/src/main.zig | 61 ++++++----- src/jetzig/App.zig | 15 +-- src/jetzig/config.zig | 12 +-- src/jetzig/http/Query.zig | 8 +- src/jetzig/http/Request.zig | 102 ++++++++++-------- src/jetzig/http/Server.zig | 12 +-- src/jetzig/jobs/Job.zig | 16 +-- src/jetzig/jobs/Pool.zig | 4 +- src/jetzig/jobs/Worker.zig | 4 +- src/jetzig/kv.zig | 35 ++++++- src/jetzig/kv/Store.zig | 203 ++++++++++++++++++++++++------------ src/jetzig/mail/Job.zig | 4 +- src/jetzig/testing/App.zig | 27 +++-- 14 files changed, 317 insertions(+), 194 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index fd0251e..7c68ef9 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -7,12 +7,12 @@ .hash = "1220d0e8734628fd910a73146e804d10a3269e3e7d065de6bb0e3e88d5ba234eb163", }, .zmpl = .{ - .url = "https://github.com/jetzig-framework/zmpl/archive/ef1930b08e1f174ddb02a3a0a01b35aa8a4af235.tar.gz", - .hash = "1220a7bacb828f12cd013b0906da61a17fac6819ab8cee81e00d9ae1aa0faa992720", + .url = "https://github.com/jetzig-framework/zmpl/archive/7b5e0309ee49c06b99c242fecd218d3f3d15cd40.tar.gz", + .hash = "12204d61eb58ee860f748e5817ef9300ad56c9d5efef84864ae590c87baf2e0380a1", }, .jetkv = .{ - .url = "https://github.com/jetzig-framework/jetkv/archive/2b1130a48979ea2871c8cf6ca89c38b1e7062839.tar.gz", - .hash = "12201d75d73aad5e1c996de4d5ae87a00e58479c8d469bc2eeb5fdeeac8857bc09af", + .url = "https://github.com/jetzig-framework/jetkv/archive/acaa30db281f1c331d20c48cfe6539186549ad45.tar.gz", + .hash = "1220b260b20cb65d801a00a39dc6506387f5faa1a225f85160e011bd2aabd2ce6e0b", }, .jetquery = .{ .url = "https://github.com/jetzig-framework/jetquery/archive/52e1cf900c94f3c103727ade6ba2dab3057c8663.tar.gz", diff --git a/demo/src/main.zig b/demo/src/main.zig index f051ee5..26f638a 100644 --- a/demo/src/main.zig +++ b/demo/src/main.zig @@ -94,43 +94,54 @@ pub const jetzig_options = struct { }, }; - /// Key-value store options. Set backend to `.file` to use a file-based store. - /// When using `.file` backend, you must also set `.file_options`. - /// The key-value store is exposed as `request.store` in views and is also available in as - /// `env.store` in all jobs/mailers. - pub const store: jetzig.kv.Store.KVOptions = .{ + /// Key-value store options. + /// Available backends: + /// * memory: Simple, in-memory hashmap-backed store. + /// * file: Rudimentary file-backed store. + /// * valkey: Valkey-backed store with connection pool. + /// + /// When using `.file` or `.valkey` backend, you must also set `.file_options` or + /// `.valkey_options` respectively. + /// + /// ## File backend: + // .backend = .file, + // .file_options = .{ + // .path = "/path/to/jetkv-store.db", + // .truncate = false, // Set to `true` to clear the store on each server launch. + // .address_space_size = jetzig.jetkv.JetKV.FileBackend.addressSpace(4096), + // }, + // + // ## Valkey backend + // .backend = .valkey, + // .valkey_options = .{ + // .host = "localhost", + // .port = 6379, + // .timeout = 1000, // in milliseconds, i.e. 1 second. + // .connect = .lazy, // Connect on first use, or `auto` to connect on server startup. + // .buffer_size = 8192, + // .pool_size = 8, + // }, + /// Available configuration options for `store`, `job_queue`, and `cache` are identical. + /// + /// For production deployment, the `valkey` backend is recommended for all use cases. + /// + /// The general-purpose key-value store is exposed as `request.store` in views and is also + /// available in as `env.store` in all jobs/mailers. + pub const store: jetzig.kv.Store.Options = .{ .backend = .memory, - // .backend = .file, - // .file_options = .{ - // .path = "/path/to/jetkv-store.db", - // .truncate = false, // Set to `true` to clear the store on each server launch. - // .address_space_size = jetzig.jetkv.JetKV.FileBackend.addressSpace(4096), - // }, }; /// Job queue options. Identical to `store` options, but allows using different /// backends (e.g. `.memory` for key-value store, `.file` for jobs queue. /// The job queue is managed internally by Jetzig. - pub const job_queue: jetzig.kv.Store.KVOptions = .{ + pub const job_queue: jetzig.kv.Store.Options = .{ .backend = .memory, - // .backend = .file, - // .file_options = .{ - // .path = "/path/to/jetkv-queue.db", - // .truncate = false, // Set to `true` to clear the store on each server launch. - // .address_space_size = jetzig.jetkv.JetKV.FileBackend.addressSpace(4096), - // }, }; /// Cache options. Identical to `store` options, but allows using different /// backends (e.g. `.memory` for key-value store, `.file` for cache. - pub const cache: jetzig.kv.Store.KVOptions = .{ + pub const cache: jetzig.kv.Store.Options = .{ .backend = .memory, - // .backend = .file, - // .file_options = .{ - // .path = "/path/to/jetkv-cache.db", - // .truncate = false, // Set to `true` to clear the store on each server launch. - // .address_space_size = jetzig.jetkv.JetKV.FileBackend.addressSpace(4096), - // }, }; /// SMTP configuration for Jetzig Mail. It is recommended to use a local SMTP relay, diff --git a/src/jetzig/App.zig b/src/jetzig/App.zig index 69f5b68..5d6b363 100644 --- a/src/jetzig/App.zig +++ b/src/jetzig/App.zig @@ -48,22 +48,13 @@ pub fn start(self: *const App, routes_module: type, options: AppOptions) !void { self.allocator.free(custom_route.template); }; - var store = try jetzig.kv.Store.init( - self.allocator, - jetzig.config.get(jetzig.kv.Store.KVOptions, "store"), - ); + var store = try jetzig.kv.Store.GeneralStore.init(self.allocator, self.env.logger, .general); defer store.deinit(); - var job_queue = try jetzig.kv.Store.init( - self.allocator, - jetzig.config.get(jetzig.kv.Store.KVOptions, "job_queue"), - ); + var job_queue = try jetzig.kv.Store.JobQueueStore.init(self.allocator, self.env.logger, .jobs); defer job_queue.deinit(); - var cache = try jetzig.kv.Store.init( - self.allocator, - jetzig.config.get(jetzig.kv.Store.KVOptions, "cache"), - ); + var cache = try jetzig.kv.Store.CacheStore.init(self.allocator, self.env.logger, .cache); defer cache.deinit(); var repo = try jetzig.database.repo(self.allocator, self); diff --git a/src/jetzig/config.zig b/src/jetzig/config.zig index 656f1e3..52663e9 100644 --- a/src/jetzig/config.zig +++ b/src/jetzig/config.zig @@ -107,38 +107,38 @@ pub const Schema: type = struct {}; /// When using `.file` backend, you must also set `.file_options`. /// The key-value store is exposed as `request.store` in views and is also available in as /// `env.store` in all jobs/mailers. -pub const store: kv.Store.KVOptions = .{ +pub const store: kv.Store.Options = .{ .backend = .memory, // .backend = .file, // .file_options = .{ // .path = "/path/to/jetkv-store.db", // .truncate = false, // Set to `true` to clear the store on each server launch. - // .address_space_size = jetzig.jetkv.JetKV.FileBackend.addressSpace(4096), + // .address_space_size = jetzig.jetkv.FileBackend.addressSpace(4096), // }, }; /// Job queue options. Identical to `store` options, but allows using different /// backends (e.g. `.memory` for key-value store, `.file` for jobs queue. /// The job queue is managed internally by Jetzig. -pub const job_queue: kv.Store.KVOptions = .{ +pub const job_queue: kv.Store.Options = .{ .backend = .memory, // .backend = .file, // .file_options = .{ // .path = "/path/to/jetkv-queue.db", // .truncate = false, // Set to `true` to clear the store on each server launch. - // .address_space_size = jetzig.jetkv.JetKV.FileBackend.addressSpace(4096), + // .address_space_size = jetzig.jetkv.JetKV.addressSpace(4096), // }, }; /// Cache. Identical to `store` options, but allows using different /// backends (e.g. `.memory` for key-value store, `.file` for cache. -pub const cache: kv.Store.KVOptions = .{ +pub const cache: kv.Store.Options = .{ .backend = .memory, // .backend = .file, // .file_options = .{ // .path = "/path/to/jetkv-cache.db", // .truncate = false, // Set to `true` to clear the store on each server launch. - // .address_space_size = jetzig.jetkv.JetKV.FileBackend.addressSpace(4096), + // .address_space_size = jetzig.jetkv.JetKV.addressSpace(4096), // }, }; diff --git a/src/jetzig/http/Query.zig b/src/jetzig/http/Query.zig index 9764cda..aa4cdb9 100644 --- a/src/jetzig/http/Query.zig +++ b/src/jetzig/http/Query.zig @@ -61,7 +61,7 @@ pub fn parse(self: *Query) !void { else => return error.JetzigQueryParseError, } } else { - var array = try jetzig.zmpl.Data.createArray(self.data.allocator()); + var array = try jetzig.zmpl.Data.createArray(self.data.allocator); try array.append(self.dataValue(item.value)); try params.put(key, array); } @@ -72,7 +72,7 @@ pub fn parse(self: *Query) !void { else => return error.JetzigQueryParseError, } } else { - var object = try jetzig.zmpl.Data.createObject(self.data.allocator()); + var object = try jetzig.zmpl.Data.createObject(self.data.allocator); try object.put(mapping.field, self.dataValue(item.value)); try params.put(mapping.key, object); } @@ -109,10 +109,10 @@ fn mappingParam(input: []const u8) ?struct { key: []const u8, field: []const u8 fn dataValue(self: Query, value: ?[]const u8) *jetzig.data.Data.Value { if (value) |item_value| { - const duped = self.data.allocator().dupe(u8, item_value) catch @panic("OOM"); + const duped = self.data.allocator.dupe(u8, item_value) catch @panic("OOM"); return self.data.string(uriDecode(duped)); } else { - return jetzig.zmpl.Data._null(self.data.allocator()); + return jetzig.zmpl.Data._null(self.data.allocator); } } diff --git a/src/jetzig/http/Request.zig b/src/jetzig/http/Request.zig index 843da55..0b47ed6 100644 --- a/src/jetzig/http/Request.zig +++ b/src/jetzig/http/Request.zig @@ -50,65 +50,75 @@ middleware_data: jetzig.http.middleware.MiddlewareData = undefined, rendered_multiple: bool = false, rendered_view: ?jetzig.views.View = null, start_time: i128, -store: RequestStore, -cache: RequestStore, +store: RequestStore(jetzig.kv.Store.GeneralStore), +cache: RequestStore(jetzig.kv.Store.CacheStore), repo: *jetzig.database.Repo, global: *jetzig.Global, /// Wrapper for KV store that uses the request's arena allocator for fetching values. -pub const RequestStore = struct { - allocator: std.mem.Allocator, - store: *jetzig.kv.Store, +pub fn RequestStore(T: type) type { + return struct { + allocator: std.mem.Allocator, + store: *T, - /// Put a String or into the key-value store. - pub fn get(self: RequestStore, key: []const u8) !?*jetzig.data.Value { - return try self.store.get(try self.data(), key); - } + const Self = @This(); - /// Get a String from the store. - pub fn put(self: RequestStore, key: []const u8, value: anytype) !void { - const alloc = (try self.data()).allocator(); - try self.store.put(key, try jetzig.Data.zmplValue(value, alloc)); - } + /// Get a Value from the store. + pub fn get(self: Self, key: []const u8) !?*jetzig.data.Value { + return try self.store.get(try self.data(), key); + } - /// Remove a String to from the key-value store and return it if found. - pub fn fetchRemove(self: RequestStore, key: []const u8) !?*jetzig.data.Value { - return try self.store.fetchRemove(try self.data(), key); - } + /// Store a Value in the key-value store. + pub fn put(self: Self, key: []const u8, value: anytype) !void { + const alloc = (try self.data()).allocator; + try self.store.put(key, try jetzig.Data.zmplValue(value, alloc)); + } - /// Remove a String to from the key-value store. - pub fn remove(self: RequestStore, key: []const u8) !void { - try self.store.remove(key); - } + /// Store a Value in the key-value store with an expiration time in seconds. + pub fn putExpire(self: Self, key: []const u8, value: anytype, expiration: i32) !void { + const alloc = (try self.data()).allocator; + try self.store.putExpire(key, try jetzig.Data.zmplValue(value, alloc), expiration); + } - /// Append a Value to the end of an Array in the key-value store. - pub fn append(self: RequestStore, key: []const u8, value: anytype) !void { - const alloc = (try self.data()).allocator(); - try self.store.append(key, try jetzig.Data.zmplValue(value, alloc)); - } + /// Remove a String to from the key-value store and return it if found. + pub fn fetchRemove(self: Self, key: []const u8) !?*jetzig.data.Value { + return try self.store.fetchRemove(try self.data(), key); + } - /// Prepend a Value to the start of an Array in the key-value store. - pub fn prepend(self: RequestStore, key: []const u8, value: anytype) !void { - const alloc = (try self.data()).allocator(); - try self.store.prepend(key, try jetzig.Data.zmplValue(value, alloc)); - } + /// Remove a String to from the key-value store. + pub fn remove(self: Self, key: []const u8) !void { + try self.store.remove(key); + } - /// Pop a String from an Array in the key-value store. - pub fn pop(self: RequestStore, key: []const u8) !?*jetzig.data.Value { - return try self.store.pop(try self.data(), key); - } + /// Append a Value to the end of an Array in the key-value store. + pub fn append(self: Self, key: []const u8, value: anytype) !void { + const alloc = (try self.data()).allocator; + try self.store.append(key, try jetzig.Data.zmplValue(value, alloc)); + } - /// Left-pop a String from an Array in the key-value store. - pub fn popFirst(self: RequestStore, key: []const u8) !?*jetzig.data.Value { - return try self.store.popFirst(try self.data(), key); - } + /// Prepend a Value to the start of an Array in the key-value store. + pub fn prepend(self: Self, key: []const u8, value: anytype) !void { + const alloc = (try self.data()).allocator; + try self.store.prepend(key, try jetzig.Data.zmplValue(value, alloc)); + } - fn data(self: RequestStore) !*jetzig.data.Data { - const arena_data = try self.allocator.create(jetzig.data.Data); - arena_data.* = jetzig.data.Data.init(self.allocator); - return arena_data; - } -}; + /// Pop a String from an Array in the key-value store. + pub fn pop(self: Self, key: []const u8) !?*jetzig.data.Value { + return try self.store.pop(try self.data(), key); + } + + /// Left-pop a String from an Array in the key-value store. + pub fn popFirst(self: Self, key: []const u8) !?*jetzig.data.Value { + return try self.store.popFirst(try self.data(), key); + } + + fn data(self: Self) !*jetzig.data.Data { + const arena_data = try self.allocator.create(jetzig.data.Data); + arena_data.* = jetzig.data.Data.init(self.allocator); + return arena_data; + } + }; +} pub fn init( allocator: std.mem.Allocator, diff --git a/src/jetzig/http/Server.zig b/src/jetzig/http/Server.zig index cdb9a85..4c4b2cf 100644 --- a/src/jetzig/http/Server.zig +++ b/src/jetzig/http/Server.zig @@ -15,9 +15,9 @@ job_definitions: []const jetzig.JobDefinition, mailer_definitions: []const jetzig.MailerDefinition, mime_map: *jetzig.http.mime.MimeMap, initialized: bool = false, -store: *jetzig.kv.Store, -job_queue: *jetzig.kv.Store, -cache: *jetzig.kv.Store, +store: *jetzig.kv.Store.GeneralStore, +job_queue: *jetzig.kv.Store.JobQueueStore, +cache: *jetzig.kv.Store.CacheStore, repo: *jetzig.database.Repo, global: *anyopaque, decoded_static_route_params: []*jetzig.data.Value = &.{}, @@ -33,9 +33,9 @@ pub fn init( job_definitions: []const jetzig.JobDefinition, mailer_definitions: []const jetzig.MailerDefinition, mime_map: *jetzig.http.mime.MimeMap, - store: *jetzig.kv.Store, - job_queue: *jetzig.kv.Store, - cache: *jetzig.kv.Store, + store: *jetzig.kv.Store.GeneralStore, + job_queue: *jetzig.kv.Store.JobQueueStore, + cache: *jetzig.kv.Store.CacheStore, repo: *jetzig.database.Repo, global: *anyopaque, ) Server { diff --git a/src/jetzig/jobs/Job.zig b/src/jetzig/jobs/Job.zig index 0ab87d6..5f7a1c6 100644 --- a/src/jetzig/jobs/Job.zig +++ b/src/jetzig/jobs/Job.zig @@ -22,9 +22,9 @@ pub const JobEnv = struct { /// All jobs detected by Jetzig on startup jobs: []const jetzig.JobDefinition, /// Global key-value store - store: *jetzig.kv.Store, + store: *jetzig.kv.Store.GeneralStore, /// Global cache - cache: *jetzig.kv.Store, + cache: *jetzig.kv.Store.CacheStore, /// Database repo repo: *jetzig.database.Repo, /// Global mutex - use with caution if it is necessary to guarantee thread safety/consistency @@ -33,9 +33,9 @@ pub const JobEnv = struct { }; allocator: std.mem.Allocator, -store: *jetzig.kv.Store, -job_queue: *jetzig.kv.Store, -cache: *jetzig.kv.Store, +store: *jetzig.kv.Store.GeneralStore, +job_queue: *jetzig.kv.Store.JobQueueStore, +cache: *jetzig.kv.Store.CacheStore, logger: jetzig.loggers.Logger, name: []const u8, definition: ?JobDefinition, @@ -47,9 +47,9 @@ const Job = @This(); /// Initialize a new Job pub fn init( allocator: std.mem.Allocator, - store: *jetzig.kv.Store, - job_queue: *jetzig.kv.Store, - cache: *jetzig.kv.Store, + store: *jetzig.kv.Store.GeneralStore, + job_queue: *jetzig.kv.Store.JobQueueStore, + cache: *jetzig.kv.Store.CacheStore, logger: jetzig.loggers.Logger, jobs: []const JobDefinition, name: []const u8, diff --git a/src/jetzig/jobs/Pool.zig b/src/jetzig/jobs/Pool.zig index caa9e85..6d95de1 100644 --- a/src/jetzig/jobs/Pool.zig +++ b/src/jetzig/jobs/Pool.zig @@ -5,7 +5,7 @@ const jetzig = @import("../../jetzig.zig"); const Pool = @This(); allocator: std.mem.Allocator, -job_queue: *jetzig.kv.Store, +job_queue: *jetzig.kv.Store.JobQueueStore, job_env: jetzig.jobs.JobEnv, pool: std.Thread.Pool = undefined, workers: std.ArrayList(*jetzig.jobs.Worker), @@ -13,7 +13,7 @@ workers: std.ArrayList(*jetzig.jobs.Worker), /// Initialize a new worker thread pool. pub fn init( allocator: std.mem.Allocator, - job_queue: *jetzig.kv.Store, + job_queue: *jetzig.kv.Store.JobQueueStore, job_env: jetzig.jobs.JobEnv, ) Pool { return .{ diff --git a/src/jetzig/jobs/Worker.zig b/src/jetzig/jobs/Worker.zig index 11bbe96..9bcdb3e 100644 --- a/src/jetzig/jobs/Worker.zig +++ b/src/jetzig/jobs/Worker.zig @@ -6,14 +6,14 @@ const Worker = @This(); allocator: std.mem.Allocator, job_env: jetzig.jobs.JobEnv, id: usize, -job_queue: *jetzig.kv.Store, +job_queue: *jetzig.kv.Store.JobQueueStore, interval: usize, pub fn init( allocator: std.mem.Allocator, job_env: jetzig.jobs.JobEnv, id: usize, - job_queue: *jetzig.kv.Store, + job_queue: *jetzig.kv.Store.JobQueueStore, interval: usize, ) Worker { return .{ diff --git a/src/jetzig/kv.zig b/src/jetzig/kv.zig index 96e3052..c10078e 100644 --- a/src/jetzig/kv.zig +++ b/src/jetzig/kv.zig @@ -1,3 +1,36 @@ const std = @import("std"); -pub const Store = @import("kv/Store.zig"); +const config = @import("config.zig"); + +pub const Store = struct { + /// Configuration for JetKV. Encompasses all backends: + /// * valkey + /// * memory + /// * file + /// + /// The Valkey backend is recommended for production deployment. `memory` and `file` can be + /// used in local development for convenience. All backends have a unified interface, i.e. + /// they can be swapped out without any code changes. + pub const Options = @import("kv/Store.zig").KVOptions; + + // For backward compatibility - `jetzig.kv.Options` is preferred. + pub const KVOptions = Options; + + /// General-purpose store. Use for storing data with no expiry. + pub const GeneralStore = @import("kv/Store.zig").Store(config.get(Store.Options, "store")); + + /// Store ephemeral data. + pub const CacheStore = @import("kv/Store.zig").Store(config.get(Store.Options, "cache")); + + /// Background job storage. + pub const JobQueueStore = @import("kv/Store.zig").Store(config.get(Store.Options, "job_queue")); + + /// Generic store type. Create a custom store by passing `Options`, e.g.: + /// ```zig + /// var store = Generic(.{ .backend = .memory }).init(allocator, logger, .custom); + /// ``` + pub const Generic = @import("kv/Store.zig").Store; + + /// Role a given store fills. Used in log outputs. + pub const Role = @import("kv/Store.zig").Role; +}; diff --git a/src/jetzig/kv/Store.zig b/src/jetzig/kv/Store.zig index cb5dccd..0f5db3f 100644 --- a/src/jetzig/kv/Store.zig +++ b/src/jetzig/kv/Store.zig @@ -1,87 +1,156 @@ const std = @import("std"); const jetzig = @import("../../jetzig.zig"); -const Store = @This(); - -store: jetzig.jetkv.JetKV, -options: KVOptions, - pub const KVOptions = struct { - backend: enum { memory, file } = .memory, + backend: enum { memory, file, valkey } = .memory, file_options: struct { path: ?[]const u8 = null, - address_space_size: u32 = jetzig.jetkv.JetKV.FileBackend.addressSpace(4096), + address_space_size: u32 = jetzig.jetkv.FileBackend.addressSpace(4096), truncate: bool = false, } = .{}, + valkey_options: struct { + host: []const u8 = "localhost", + port: u16 = 6379, + connect_timeout: u64 = 1000, // (ms) + read_timeout: u64 = 1000, // (ms) + connect: enum { auto, manual, lazy } = .lazy, + buffer_size: u32 = 8192, + pool_size: u16 = 8, + } = .{}, }; const ValueType = enum { string, array }; -/// Initialize a new memory or file store. -pub fn init(allocator: std.mem.Allocator, options: KVOptions) !Store { - const store = try jetzig.jetkv.JetKV.init( - allocator, - switch (options.backend) { - .file => .{ - .backend = .file, - .file_backend_options = .{ - .path = options.file_options.path, - .address_space_size = options.file_options.address_space_size, - .truncate = options.file_options.truncate, - }, +fn jetKVOptions(options: KVOptions) jetzig.jetkv.Options { + return switch (options.backend) { + .file => .{ + .backend = .file, + .file_backend_options = .{ + .path = options.file_options.path, + .address_space_size = options.file_options.address_space_size, + .truncate = options.file_options.truncate, }, - .memory => .{ - .backend = .memory, + }, + .memory => .{ + .backend = .memory, + }, + .valkey => .{ + .backend = .valkey, + .valkey_backend_options = .{ + .host = options.valkey_options.host, + .port = options.valkey_options.port, + .connect_timeout = options.valkey_options.connect_timeout * std.time.ms_per_s, + .read_timeout = options.valkey_options.read_timeout * std.time.ms_per_s, + .connect = std.enums.nameCast( + jetzig.jetkv.ValkeyBackendOptions.ConnectMode, + options.valkey_options.connect, + ), + .buffer_size = options.valkey_options.buffer_size, + .pool_size = options.valkey_options.pool_size, }, }, - ); - - return .{ .store = store, .options = options }; -} - -/// Free allocated resources/close database file. -pub fn deinit(self: *Store) void { - self.store.deinit(); -} - -/// Put a or into the key-value store. -pub fn put(self: *Store, key: []const u8, value: *jetzig.data.Value) !void { - try self.store.put(key, try value.toJson()); -} - -/// Get a Value from the store. -pub fn get(self: *Store, data: *jetzig.data.Data, key: []const u8) !?*jetzig.data.Value { - return try parseValue(data, try self.store.get(data.allocator(), key)); -} - -/// Remove a Value to from the key-value store and return it if found. -pub fn fetchRemove(self: *Store, data: *jetzig.data.Data, key: []const u8) !?*jetzig.data.Value { - return try parseValue(data, try self.store.fetchRemove(data.allocator(), key)); -} - -/// Remove a Value to from the key-value store. -pub fn remove(self: *Store, key: []const u8) !void { - try self.store.remove(key); -} - -/// Append a Value to the end of an Array in the key-value store. -pub fn append(self: *Store, key: []const u8, value: *const jetzig.data.Value) !void { - try self.store.append(key, try value.toJson()); -} - -/// Prepend a Value to the start of an Array in the key-value store. -pub fn prepend(self: *Store, key: []const u8, value: *const jetzig.data.Value) !void { - try self.store.prepend(key, try value.toJson()); -} - -/// Pop a Value from an Array in the key-value store. -pub fn pop(self: *Store, data: *jetzig.data.Data, key: []const u8) !?*jetzig.data.Value { - return try parseValue(data, try self.store.pop(data.allocator(), key)); + }; } -/// Left-pop a Value from an Array in the key-value store. -pub fn popFirst(self: *Store, data: *jetzig.data.Data, key: []const u8) !?*jetzig.data.Value { - return try parseValue(data, try self.store.popFirst(data.allocator(), key)); +/// Role a given store fills. Used in log outputs. +pub const Role = enum { jobs, cache, general, custom }; + +pub fn Store(comptime options: KVOptions) type { + return struct { + const Self = @This(); + + store: jetzig.jetkv.JetKV(jetKVOptions(options)), + logger: jetzig.loggers.Logger, + options: KVOptions, + role: Role, + + /// Initialize a new memory or file store. + pub fn init(allocator: std.mem.Allocator, logger: jetzig.loggers.Logger, role: Role) !Self { + const store = try jetzig.jetkv.JetKV(jetKVOptions(options)).init(allocator); + + return .{ .store = store, .role = role, .logger = logger, .options = options }; + } + + /// Free allocated resources/close database file. + pub fn deinit(self: *Self) void { + self.store.deinit(); + } + + /// Put a or into the key-value store. + pub fn put(self: *Self, key: []const u8, value: *jetzig.data.Value) !void { + try self.store.put(key, try value.toJson()); + if (self.role == .cache) { + try self.logger.DEBUG( + "[cache:{s}:store] {s}", + .{ @tagName(self.store.backend), key }, + ); + } + } + + /// Put a or into the key-value store with an expiration in seconds. + pub fn putExpire(self: *Self, key: []const u8, value: *jetzig.data.Value, expiration: i32) !void { + try self.store.putExpire(key, try value.toJson(), expiration); + if (self.role == .cache) { + try self.logger.DEBUG( + "[cache:{s}:store:expire:{d}s] {s}", + .{ @tagName(self.store.backend), expiration, key }, + ); + } + } + + /// Get a Value from the store. + pub fn get(self: *Self, data: *jetzig.data.Data, key: []const u8) !?*jetzig.data.Value { + const start = std.time.nanoTimestamp(); + const json = try self.store.get(data.allocator, key); + const value = try parseValue(data, json); + const end = std.time.nanoTimestamp(); + if (self.role == .cache) { + if (value == null) { + try self.logger.DEBUG("[cache:miss] {s}", .{key}); + } else { + try self.logger.DEBUG( + "[cache:{s}:hit:{}] {s}", + .{ + @tagName(self.store.backend), + std.fmt.fmtDuration(@intCast(end - start)), + key, + }, + ); + } + } + return value; + } + + /// Remove a Value to from the key-value store and return it if found. + pub fn fetchRemove(self: *Self, data: *jetzig.data.Data, key: []const u8) !?*jetzig.data.Value { + return try parseValue(data, try self.store.fetchRemove(data.allocator, key)); + } + + /// Remove a Value to from the key-value store. + pub fn remove(self: *Self, key: []const u8) !void { + try self.store.remove(key); + } + + /// Append a Value to the end of an Array in the key-value store. + pub fn append(self: *Self, key: []const u8, value: *const jetzig.data.Value) !void { + try self.store.append(key, try value.toJson()); + } + + /// Prepend a Value to the start of an Array in the key-value store. + pub fn prepend(self: *Self, key: []const u8, value: *const jetzig.data.Value) !void { + try self.store.prepend(key, try value.toJson()); + } + + /// Pop a Value from an Array in the key-value store. + pub fn pop(self: *Self, data: *jetzig.data.Data, key: []const u8) !?*jetzig.data.Value { + return try parseValue(data, try self.store.pop(data.allocator, key)); + } + + /// Left-pop a Value from an Array in the key-value store. + pub fn popFirst(self: *Self, data: *jetzig.data.Data, key: []const u8) !?*jetzig.data.Value { + return try parseValue(data, try self.store.popFirst(data.allocator, key)); + } + }; } fn parseValue(data: *jetzig.data.Data, maybe_json: ?[]const u8) !?*jetzig.data.Value { diff --git a/src/jetzig/mail/Job.zig b/src/jetzig/mail/Job.zig index e4b7c23..b29bb99 100644 --- a/src/jetzig/mail/Job.zig +++ b/src/jetzig/mail/Job.zig @@ -130,7 +130,7 @@ fn defaultHtml( data.value = if (params.get("params")) |capture| capture else - try jetzig.zmpl.Data.createObject(data.allocator()); + try jetzig.zmpl.Data.createObject(data.allocator); try data.addConst("jetzig_view", data.string("")); try data.addConst("jetzig_action", data.string("")); return if (jetzig.zmpl.findPrefixed("mailers", mailer.html_template)) |template| @@ -148,7 +148,7 @@ fn defaultText( data.value = if (params.get("params")) |capture| capture else - try jetzig.zmpl.Data.createObject(data.allocator()); + try jetzig.zmpl.Data.createObject(data.allocator); try data.addConst("jetzig_view", data.string("")); try data.addConst("jetzig_action", data.string("")); return if (jetzig.zmpl.findPrefixed("mailers", mailer.text_template)) |template| diff --git a/src/jetzig/testing/App.zig b/src/jetzig/testing/App.zig index ef4d2b7..09f8c46 100644 --- a/src/jetzig/testing/App.zig +++ b/src/jetzig/testing/App.zig @@ -4,13 +4,14 @@ const jetzig = @import("../../jetzig.zig"); const httpz = @import("httpz"); const App = @This(); +const MemoryStore = jetzig.kv.Store.Generic(.{ .backend = .memory }); allocator: std.mem.Allocator, routes: []const jetzig.views.Route, arena: *std.heap.ArenaAllocator, -store: *jetzig.kv.Store, -cache: *jetzig.kv.Store, -job_queue: *jetzig.kv.Store, +store: *MemoryStore, +cache: *MemoryStore, +job_queue: *MemoryStore, multipart_boundary: ?[]const u8 = null, logger: jetzig.loggers.Logger, server: Server, @@ -60,9 +61,9 @@ pub fn init(allocator: std.mem.Allocator, routes_module: type) !App { .arena = arena, .allocator = allocator, .routes = &routes_module.routes, - .store = try createStore(arena.allocator()), - .cache = try createStore(arena.allocator()), - .job_queue = try createStore(arena.allocator()), + .store = try createStore(arena.allocator(), logger, .general), + .cache = try createStore(arena.allocator(), logger, .cache), + .job_queue = try createStore(arena.allocator(), logger, .jobs), .logger = logger, .server = .{ .logger = logger }, .repo = repo, @@ -391,9 +392,17 @@ fn multiFormKeyValue(allocator: std.mem.Allocator, max: usize) !*httpz.key_value return key_value; } -fn createStore(allocator: std.mem.Allocator) !*jetzig.kv.Store { - const store = try allocator.create(jetzig.kv.Store); - store.* = try jetzig.kv.Store.init(allocator, .{}); +fn createStore( + allocator: std.mem.Allocator, + logger: jetzig.loggers.Logger, + role: jetzig.kv.Store.Role, +) !*MemoryStore { + const store = try allocator.create(MemoryStore); + store.* = try MemoryStore.init( + allocator, + logger, + role, + ); return store; }