Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

environments readable / writable #8

Merged
merged 2 commits into from
Jan 29, 2024
Merged

environments readable / writable #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e51d62e
ref 38 distinct readable and writable assemblies and environments
CodiePP Jan 15, 2024
568fb3c
cleanup
CodiePP Jan 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions bin/lxr_assembly.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ let main () = Arg.parse argspec anon_args_fun "lxr_assembly: vaf";
let%lwt () = Lwt_io.printlf "rnd256 %d" (Conversion.n2i @@ Utilities.rnd @@ Conversion.i2n 0) in
let%lwt () = Lwt_io.printlf "sha256 of /bin/sh %s" (Fsutils.fchksum "/bin/sh") in
let c : Configuration.configuration = { config_nchunks = (Nchunks.from_int 16); path_chunks = "./chunks"; path_db = "/tmp/db"; my_id = "16"} in
let e0 = Environment.initial_environment c in
let e0 = Environment.EnvironmentWritable.initial_environment c in
let a = Environment.cur_assembly e0 in
let b = Environment.cur_buffer e0 in
let%lwt () = Lwt_io.printlf "assembly %s %d %d" a.aid (Conversion.p2i a.nchunks) (Conversion.n2i a.apos) in
Expand All @@ -59,8 +59,8 @@ let main () = Arg.parse argspec anon_args_fun "lxr_assembly: vaf";
(* let msg = "testing some longer message." in *)
let content = BufferPlain.buffer_create (Conversion.i2n 1024) in (* (fun i -> if i < 28 then String.get msg i else '0') in *)
let (a', bi) = Elykseer__Lxr.Assembly.backup a b (* "test1M" *) (Conversion.i2n 0) content in
let e1 = Environment.env_add_file_block "test1M" e0 bi in
let e2 = Environment.env_add_aid_key (aid a') e1 {pkey="abc97391af";ivec="323453";localnchunks=Nchunks.to_positive c.config_nchunks;localid=c.my_id} in
let e1 = Environment.EnvironmentWritable.env_add_file_block "test1M" e0 bi in
let e2 = Environment.EnvironmentWritable.env_add_aid_key (aid a') e1 {pkey="abc97391af";ivec="323453";localnchunks=Nchunks.to_positive c.config_nchunks;localid=c.my_id} in
let relkeys = Environment.keys e2 in
let relkey = List.assoc (aid a') relkeys in
let (a'', b') = Elykseer__Lxr.Assembly.finish a' b in
Expand Down
8 changes: 4 additions & 4 deletions bin/lxr_backup.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ let execute_backup_fileblock e _anum (fname,blocks) =
Cstdio.File.fread buf sz fptr |> function
| Ok _nread ->
let bplain = Buffer.BufferPlain.from_buffer buf in
Environment.backup e0 fname fpos bplain
EnvironmentWritable.backup e0 fname fpos bplain
| Error (errno,errstr) ->
Printf.printf "read error no:%d err:%s\n" errno errstr; e0
end
Expand All @@ -101,7 +101,7 @@ let rec run_backup_for_assembly e0 pid anum acount (bp : backup_plan) =
let anum_p = Conversion.i2p anum in
let fileblocks' = extract_fileblocks anum_p bp in
let e1 = List.fold_left (fun env fb -> execute_backup_fileblock env anum fb) e0 fileblocks' in
let e2 = Environment.finalise_and_recreate_assembly e1 in
let e2 = Environment.EnvironmentWritable.finalise_and_recreate_assembly e1 in
run_backup_for_assembly e2 pid (anum + 1) acount bp
end

Expand Down Expand Up @@ -178,7 +178,7 @@ let main () = Arg.parse argspec anon_args_fun "lxr_backup: vyxdnji";
path_chunks = !arg_chunkpath;
path_db = !arg_dbpath;
my_id = myid } in
let e0 = Environment.initial_environment conf in
let e0 = Environment.EnvironmentWritable.initial_environment conf in
let%lwt backup_plan = plan_backup e0 !arg_files in
let fcount = List.map (fun bpf -> bpf.fhash) backup_plan.bp |> List.sort_uniq (compare) |> List.length
and acount = List.map (fun bpf -> bpf.curbs) backup_plan.bp |> List.map (List.map BackupPlanner.fbanum) |>
Expand All @@ -189,7 +189,7 @@ let main () = Arg.parse argspec anon_args_fun "lxr_backup: vyxdnji";
List.flatten |> List.sort_uniq (compare) |> List.length in
Lwt_io.printlf "reference contains %d %s" rcount (Utils.pluralise2 "assembly" "assemblies" rcount) in
let e1 = run_distributed_backup e0 !arg_nproc acount backup_plan in
let e2 = Environment.finalise_assembly e1 in
let e2 = Environment.EnvironmentWritable.finalise_assembly e1 in
let%lwt () = output_relations e2 backup_plan in
let%lwt () = Lwt_io.printl "done." in
Lwt.return ()
Expand Down
11 changes: 5 additions & 6 deletions bin/lxr_restore.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ let ensure_assembly e relk aid =
match%lwt Relkeys.find aid relk with
| None -> Lwt.return @@ Error "no key found"
| Some ki ->
let e1 = Environment.env_add_aid_key aid e ki in
match Environment.restore_assembly e1 aid with
let e1 = Environment.EnvironmentReadable.env_add_aid_key aid e ki in
match Environment.EnvironmentReadable.restore_assembly e1 aid with
| None -> Lwt.return @@ Error "failed to restore assembly"
| Some e2 -> Lwt.return @@ Ok (e2.cur_assembly, e2.cur_buffer)

Expand All @@ -53,10 +53,9 @@ let restore_file_blocks e0 relk fptr (fb : Assembly.blockinformation) =
| Error errstr -> let%lwt () = Lwt_io.printlf " failed to recall assembly %s with '%s'" fb.blockaid errstr in
Lwt.return (0,e0)
| Ok (a,b) ->
let (_a'', b'') = Assembly.finish a b in
let sz = Conversion.n2i fb.blocksize in
let fbuf = Cstdio.File.Buffer.create sz in
let abuf = Buffer.BufferPlain.to_buffer @@ Assembly.id_buffer_t_from_full b'' in
let abuf = Buffer.BufferPlain.to_buffer @@ Assembly.id_buffer_t_from_full b in
let nread = Elykseer_base.Assembly.get_content ~src:abuf ~sz:sz ~pos:(Conversion.n2i fb.blockapos) ~tgt:fbuf in
Cstdio.File.fseek fptr (Conversion.n2i fb.filepos) |> function
| Error _ -> let%lwt () = Lwt_io.printlf " failed to 'fseek'\n" in Lwt.return (0,e0)
Expand Down Expand Up @@ -94,7 +93,7 @@ let restore_file e0 relf relk fname =
in
Lwt.return (cnt,e1)

let ensure_all_available (e : Environment.environment) fns =
let ensure_all_available (e : Environment.EnvironmentReadable.coq_E) fns =
let%lwt rel = Relfiles.new_map e.config in
let%lwt ls = Lwt_list.map_s (fun fname -> let fhash = sha256 fname in
match%lwt Relfiles.find fhash rel with None -> Lwt.return 0 | Some _ -> Lwt.return 1) fns in
Expand Down Expand Up @@ -133,7 +132,7 @@ let main () = Arg.parse argspec anon_args_fun "lxr_restore: vxodnji";
path_chunks = !arg_chunkpath;
path_db = !arg_dbpath;
my_id = myid } in
let e0 = Environment.initial_environment conf in
let e0 = Environment.EnvironmentReadable.initial_environment conf in
let%lwt relf = Relfiles.new_map conf in
let%lwt relk = Relkeys.new_map conf in
restore_files e0 relf relk !arg_files
Expand Down
8 changes: 4 additions & 4 deletions elykseer-utils/actrl.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@ open Elykseer__Lxr

type t = {
myid : string;
env : Environment.environment
env : Environment.EnvironmentWritable.coq_E
}

let create (config : Configuration.configuration) = Lwt.return {
myid = config.my_id;
env = Environment.initial_environment config
env = Environment.EnvironmentWritable.initial_environment config
}

let stop actrl =
let env = Environment.finalise_assembly actrl.env in
let env = Environment.EnvironmentWritable.finalise_assembly actrl.env in
let%lwt relk = Relkeys.new_map env.config in
let%lwt () = Lwt_list.iter_s (fun (aid, ki) ->
let%lwt _ = Relkeys.add aid ki relk in Lwt.return ()) env.keys in
Expand All @@ -22,6 +22,6 @@ let addblock actrl fn (fb : Assembly.blockinformation) buf =
(* let%lwt () = Lwt_io.printlf "adding block (%d@%d=%d) with buffer (%d)" (Conversion.p2i fb.blockid) (Conversion.n2i fb.filepos) (Conversion.n2i fb.blocksize)
(Cstdio.File.Buffer.size buf) in *)
let bplain = Buffer.BufferPlain.from_buffer buf in
let env' = Environment.backup actrl.env fn fb.filepos bplain in
let env' = Environment.EnvironmentWritable.backup actrl.env fn fb.filepos bplain in
let (_fname, fb') = List.hd @@ Environment.fblocks env' in
Lwt.return ({actrl with env = env'}, {fb' with blockid = fb.blockid})
Loading
Loading