2019-08-01 02:29:38 Testing bvt's Blockwise Read vpatch
In the logs recently, I read that bvt created a blog post, along with a vpatch, and signature for resolving the problem of (v)diffing large files. I wanted to give this a test, just to ensure that it works. Especially in light of recent conversations regarding using large genesis files.To begin with, I went to the base page for 'vtools' so that I could see the most recent vtree. By clicking on each node in the vtree, I was able to download all of the vpatches and corresponding signatures.
I then pressed the vtree to my local environment, and built the source. After a successful compilation, I wanted to run a five tests. The tests begin with diffing files 25 Mb each, then doubling each pass. Below, I'm posting my log file so others can review and comment if they wish.
mod6@cuntoo-test1 ~/large-vdiff $ mkdir .wot
mod6@cuntoo-test1 ~/large-vdiff $ mkdir patches
mod6@cuntoo-test1 ~/large-vdiff $ mkdir .seals# Add patches, seals, and keys here...mod6@cuntoo-test1 ~/large-vdiff $ vk.pl w
bvt:6CF3EFF892A7F23E7E798E5EBA6B8C054B962B68:bvt aka BT
phf:BDDE12104FE81BE7F83B698F5356DE4752432A9E:Philip Fominykh
mod6@cuntoo-test1 ~/large-vdiff $ vk.pl f
vtools_genesis.vpatch (phf)
vdiff_fixes_newline_gcc.vpatch (phf)
keccak.vpatch (phf)
vdiff_keccak.vpatch (phf)
vtools_fixes_bitrate_char_array.vpatch (phf)
vtools_vpatch.vpatch (phf)
vtools_fixes_static_tohex.vpatch (phf)
vtools_vpatch_newline.vpatch (phf)
vtools_ksum.vpatch (phf)
vtools_tempfile_standalone_notmp.vpatch (phf)
vdiff_blockwise_read-2.vpatch (bvt)
mod6@cuntoo-test1 ~/large-vdiff $ vk.pl pp vdiff_blockwise_read-2.vpatch
vtools_genesis.vpatch (phf)
vdiff_fixes_newline_gcc.vpatch (phf)
keccak.vpatch (phf)
vdiff_keccak.vpatch (phf)
vtools_fixes_bitrate_char_array.vpatch (phf)
vtools_vpatch.vpatch (phf)
vtools_fixes_static_tohex.vpatch (phf)
vtools_vpatch_newline.vpatch (phf)
vtools_ksum.vpatch (phf)
vtools_tempfile_standalone_notmp.vpatch (phf)
vdiff_blockwise_read-2.vpatch (bvt)
mod6@cuntoo-test1 ~/large-vdiff $
mod6@cuntoo-test1 ~/large-vdiff $ vk.pl p v test vdiff_blockwise_read-2.vpatch
vtools_genesis.vpatch
creating vtools/Makefile
creating vtools/lib/cmpbuf.c
creating vtools/lib/cmpbuf.h
creating vtools/lib/diffseq.h
creating vtools/lib/dirname.c
creating vtools/lib/dirname.h
creating vtools/lib/error.c
creating vtools/lib/error.h
creating vtools/lib/filenamecat.c
creating vtools/lib/filenamecat.h
creating vtools/lib/filetype.c
creating vtools/lib/filetype.h
creating vtools/lib/hash.c
creating vtools/lib/hash.h
creating vtools/lib/progname.c
creating vtools/lib/progname.h
creating vtools/lib/xalloc.c
creating vtools/lib/xalloc.h
creating vtools/manifest
creating vtools/src/analyze.c
creating vtools/src/context.c
creating vtools/src/diff.c
creating vtools/src/diff.h
creating vtools/src/dir.c
creating vtools/src/io.c
creating vtools/src/system.h
creating vtools/src/util.c
vdiff_fixes_newline_gcc.vpatch
patching vtools/Makefile
patching vtools/lib/xalloc.h
patching vtools/manifest
patching vtools/src/analyze.c
patching vtools/src/io.c
keccak.vpatch
patching vtools/manifest
creating vtools/src/smg_keccak.adb
creating vtools/src/smg_keccak.ads
vdiff_keccak.vpatch
patching vtools/Makefile
patching vtools/manifest
creating vtools/obj/readme
patching vtools/src/analyze.c
patching vtools/src/context.c
patching vtools/src/diff.c
patching vtools/src/diff.h
patching vtools/src/io.c
creating vtools/src/keccak.h
creating vtools/src/keccak_c.adb
creating vtools/src/keccak_c.ads
patching vtools/src/smg_keccak.adb
patching vtools/src/smg_keccak.ads
creating vtools/vdiff.gpr
vtools_fixes_bitrate_char_array.vpatch
patching vtools/manifest
patching vtools/src/keccak_c.adb
patching vtools/src/keccak_c.ads
patching vtools/src/smg_keccak.adb
vtools_vpatch.vpatch
patching vtools/Makefile
patching vtools/manifest
creating vtools/src/bits.adb
creating vtools/src/bits.ads
patching vtools/src/keccak_c.adb
patching vtools/src/keccak_c.ads
patching vtools/src/smg_keccak.ads
creating vtools/src/vpatch.adb
patching vtools/vdiff.gpr
creating vtools/vpatch.gpr
vtools_fixes_static_tohex.vpatch
patching vtools/lib/xalloc.h
patching vtools/manifest
patching vtools/src/bits.adb
vtools_vpatch_newline.vpatch
patching vtools/manifest
creating vtools/src/character_io.adb
creating vtools/src/character_io.ads
patching vtools/src/vpatch.adb
vtools_ksum.vpatch
creating vtools/ksum.gpr
patching vtools/manifest
creating vtools/src/ksum.adb
vtools_tempfile_standalone_notmp.vpatch
patching vtools/manifest
creating vtools/src/temporary_file.adb
creating vtools/src/temporary_file.ads
patching vtools/src/vpatch.adb
vdiff_blockwise_read-2.vpatch
patching vtools/manifest
patching vtools/src/keccak_c.adb
mod6@cuntoo-test1 ~/large-vdiff $ cd test
mod6@cuntoo-test1 ~/large-vdiff/test $ ls
vtools
mod6@cuntoo-test1 ~/large-vdiff/test $ cd vtools/
mod6@cuntoo-test1 ~/large-vdiff/test/vtools $ ls
Makefile ksum.gpr lib manifest obj src vdiff.gpr vpatch.gpr
mod6@cuntoo-test1 ~/large-vdiff/test/vtools $ make
gprbuild -Pvdiff.gpr
Compile
[C] diff.c
[C] filetype.c
[C] error.c
[C] cmpbuf.c
[C] progname.c
[C] dirname.c
[C] hash.c
[C] filenamecat.c
[C] xalloc.c
[C] dir.c
[C] io.c
[C] util.c
[C] context.c
[C] analyze.c
[Ada] bits.adb
[Ada] ksum.adb
[Ada] temporary_file.adb
[Ada] character_io.adb
[Ada] keccak_c.adb
[Ada] smg_keccak.adb
Bind
[gprbind] diff.bexch
[Ada] bits.ali -n
Link
[archive] libvdiff.a
[index] libvdiff.a
[link] diff.c
mod6@cuntoo-test1 ~/large-vdiff/test/vtools $
SIZE=25;
time for i in `seq 1 5`; do
mkdir -p test$i/{a,b};
echo "$SIZE" | perl -e '@a=; my $mb = $a[0] * 1024000; print "A" x $mb;' > test$i/a/test$i.txt;
echo "$SIZE" | perl -e '@a=; my $mb = $a[0] * 1024000; print "B" x $mb;' > test$i/b/test$i.txt;
./vdiff test$i/a/test$i.txt test$i/b/test$i.txt > test$i/test$i.vpatch;
SIZE=$((SIZE * 2));
donereal 5m49.038s
user 5m44.641s
sys 0m4.297smod6@cuntoo-test1 ~/large-vdiff/test/vtools $
for i in `seq 1 5`; do
cd test$i;
pwd;
ls -al a/test$i.txt b/test$i.txt test$i.vpatch;
KS_A=$(ksum a/test$i.txt);
grep -q $KS_A test$i.vpatch;
if [[ $? != 0 ]]; then
echo "eggog! ksum mismatch in test$i.vpatch :: $KS_A";
fi;
KS_B=$(ksum b/test$i.txt);
grep -q $KS_B test$i.vpatch;
if [[ $? != 0 ]]; then
echo "eggog! ksum mismatch in test$i.vpatch :: $KS_B";
fi;
cd ..;
done
/home/mod6/large-vdiff/test/vtools/test1
-rw-r--r-- 1 mod6 mod6 25600000 Aug 1 01:32 a/test1.txt
-rw-r--r-- 1 mod6 mod6 25600000 Aug 1 01:32 b/test1.txt
-rw-r--r-- 1 mod6 mod6 51200374 Aug 1 01:32 test1.vpatch
/home/mod6/large-vdiff/test/vtools/test2
-rw-r--r-- 1 mod6 mod6 51200000 Aug 1 01:32 a/test2.txt
-rw-r--r-- 1 mod6 mod6 51200000 Aug 1 01:32 b/test2.txt
-rw-r--r-- 1 mod6 mod6 102400374 Aug 1 01:32 test2.vpatch
/home/mod6/large-vdiff/test/vtools/test3
-rw-r--r-- 1 mod6 mod6 102400000 Aug 1 01:32 a/test3.txt
-rw-r--r-- 1 mod6 mod6 102400000 Aug 1 01:32 b/test3.txt
-rw-r--r-- 1 mod6 mod6 204800374 Aug 1 01:33 test3.vpatch
/home/mod6/large-vdiff/test/vtools/test4
-rw-r--r-- 1 mod6 mod6 204800000 Aug 1 01:33 a/test4.txt
-rw-r--r-- 1 mod6 mod6 204800000 Aug 1 01:33 b/test4.txt
-rw-r--r-- 1 mod6 mod6 409600374 Aug 1 01:34 test4.vpatch
/home/mod6/large-vdiff/test/vtools/test5
-rw-r--r-- 1 mod6 mod6 409600000 Aug 1 01:34 a/test5.txt
-rw-r--r-- 1 mod6 mod6 409600000 Aug 1 01:34 b/test5.txt
-rw-r--r-- 1 mod6 mod6 819200374 Aug 1 01:37 test5.vpatch
mod6@cuntoo-test1 ~/large-vdiff/test/vtools $