diff -r 88ea8de5bdf7 sys/include/venti.h
--- a/sys/include/venti.h Sat May 26 00:00:00 2012 +0200
+++ b/sys/include/venti.h Tue May 29 00:00:00 2012 +0200
@@ -83,7 +83,6 @@
{
VtScoreSize = 20,
VtMaxStringSize = 1024,
- VtMaxLumpSize = 56*1024,
VtPointerDepth = 7
};
#define VtMaxFileSize ((1ULL<<48)-1)
@@ -134,7 +133,9 @@
_VtEntryDir = 1<<1, /* a directory */
_VtEntryDepthShift = 2, /* shift for pointer depth */
_VtEntryDepthMask = 7<<2, /* mask for pointer depth */
- VtEntryLocal = 1<<5 /* for local storage only */
+ VtEntryLocal = 1<<5, /* for local storage only */
+ _VtEntryBig = 1<<6,
+ VtEntryNoArchive = 1<<7, /* for local storage only */
};
enum
{
@@ -143,8 +144,8 @@
struct VtEntry
{
ulong gen; /* generation number */
- ushort psize; /* pointer block size */
- ushort dsize; /* data block size */
+ ulong psize; /* pointer block size */
+ ulong dsize; /* data block size */
uchar type;
uchar flags;
uvlong size;
@@ -159,14 +160,15 @@
char name[128];
char type[128];
uchar score[VtScoreSize]; /* to a Dir block */
- ushort blocksize; /* maximum block size */
+ ulong blocksize; /* maximum block size */
uchar prev[VtScoreSize]; /* last root block */
};
enum
{
VtRootSize = 300,
- VtRootVersion = 2
+ VtRootVersion = 2,
+ _VtRootVersionBig = 1<<15,
};
void vtrootpack(VtRoot*, uchar*);
@@ -293,7 +295,7 @@
uint nauth; /* TauthX, RauthX */
uchar score[VtScoreSize]; /* Tread, Rwrite */
uchar blocktype; /* Tread, Twrite */
- ushort count; /* Tread */
+ uint count; /* Tread */
Packet *data; /* Rread, Twrite */
};
@@ -334,7 +336,9 @@
};
VtConn* vtconn(int infd, int outfd);
+int vtreconn(VtConn*, int, int);
VtConn* vtdial(char*);
+int vtredial(VtConn*, char *);
void vtfreeconn(VtConn*);
int vtsend(VtConn*, Packet*);
Packet* vtrecv(VtConn*);
@@ -378,6 +382,10 @@
int vtsync(VtConn*);
int vtping(VtConn*);
+/* sha1 */
+void vtsha1(uchar score[VtScoreSize], uchar*, int);
+int vtsha1check(uchar score[VtScoreSize], uchar*, int);
+
/*
* Data blocks and block cache.
*/
@@ -397,7 +405,8 @@
uchar *data;
uchar score[VtScoreSize];
- uchar type; /* BtXXX */
+ uchar type; /* VtXXX */
+ ulong size;
/* internal to cache */
int nlock;
@@ -415,15 +424,14 @@
u32int vtglobaltolocal(uchar[VtScoreSize]);
void vtlocaltoglobal(u32int, uchar[VtScoreSize]);
-VtCache*vtcachealloc(VtConn*, int blocksize, ulong nblocks);
+VtCache*vtcachealloc(VtConn*, ulong maxmem);
void vtcachefree(VtCache*);
VtBlock*vtcachelocal(VtCache*, u32int addr, int type);
-VtBlock*vtcacheglobal(VtCache*, uchar[VtScoreSize], int type);
-VtBlock*vtcacheallocblock(VtCache*, int type);
+VtBlock*vtcacheglobal(VtCache*, uchar[VtScoreSize], int type, ulong size);
+VtBlock*vtcacheallocblock(VtCache*, int type, ulong size);
void vtcachesetwrite(VtCache*,
int(*)(VtConn*, uchar[VtScoreSize], uint, uchar*, int));
void vtblockput(VtBlock*);
-u32int vtcacheblocksize(VtCache*);
int vtblockwrite(VtBlock*);
VtBlock*vtblockcopy(VtBlock*);
void vtblockduplock(VtBlock*);
@@ -442,6 +450,7 @@
int local;
VtBlock *b; /* block containing this file */
uchar score[VtScoreSize]; /* score of block containing this file */
+ int bsize; /* size of block */
/* immutable */
VtCache *c;
diff -r 88ea8de5bdf7 sys/man/1/vac
--- a/sys/man/1/vac Sat May 26 00:00:00 2012 +0200
+++ b/sys/man/1/vac Tue May 29 00:00:00 2012 +0200
@@ -6,6 +6,9 @@
[
.B -mqsv
] [
+.B -a
+.I vacfile
+] [
.B -b
.I blocksize
] [
@@ -23,12 +26,15 @@
] [
.B -h
.I host
+] [
+.B -x
+.I excludefile
]
.I file ...
.PP
.B unvac
[
-.B -Tctv
+.B -Tcdtv
] [
.B -h
.I host
@@ -64,11 +70,27 @@
vac:64daefaecc4df4b5cb48a368b361ef56012a4f46
.EE
.PP
-The options to
-.I vac
-are:
-.TF "-d\fI oldvacfile"
-.PD
+The options are:
+.TP
+.BI -a " vacfile
+Specifies that vac should create or update a backup archive, inserting
+the files under an extra two levels of directory hierarchy named
+.I yyyy/mmdd
+(year, month, day)
+in the style of the dump file system
+(see Plan 9's \fIfs\fR(4)).
+If
+.I vacfile
+already exists, an additional backup day is added to the
+existing hierarchy, behaving as though the
+.B -d
+flag was specified giving the most recent backup tree in the archive.
+Typically, this option
+is used as part of a nightly backup script.
+This option cannot be used with
+.B -d
+or
+.BR -f .
.TP
.BI -b " blocksize
Specifies the block size that data will be broken into.
@@ -90,6 +112,12 @@
Do not include the file or directory specified by
.IR exclude .
This option may be repeated multiple times.
+.I Exclude
+can be a shell pattern as accepted by
+.IR rc (1),
+with one extension:
+.B \&...
+matches any sequence of characters including slashes.
.TP
.BI -f " vacfile
The results of
@@ -127,8 +155,10 @@
.TP
.B -q
Increase the performance of the
+.B -a
+or
.B -d
-option by detecting unchanged files based on a match of the files name and other meta data,
+options by detecting unchanged files based on a match of the files name and other meta data,
rather than examining the contents of the files.
.TP
.B -s
@@ -137,6 +167,27 @@
.B -v
Produce more verbose output on standard error, including the name of the files added to the archive
and the vac archives that are expanded and merged.
+.TP
+.BI -x " excfile
+Read exclude patterns from the file
+.IR excfile .
+Blank lines and lines beginning with
+.B #
+are ignored.
+All other lines should be of the form
+.B include
+.I pattern
+or
+.B exclude
+.I pattern .
+When considering whether to include a directory or file
+in the vac archive,
+the earliest matching pattern in the file
+applies.
+The patterns are the same syntax accepted by the
+.B -e
+option.
+This option may be repeated multiple times.
.PP
.I Unvac
lists or extracts files stored in the vac archive
@@ -156,9 +207,12 @@
.B -c
Write extracted files to standard output instead of creating a file.
.TP
-.B -h
-as per
-.IR vac .
+.B -d
+Reduce the number of blocks read from Venti by
+comparing the files to be stored with their counterparts
+in the file system.
+This option cannot be used with
+.BR -c .
.TP
.B -t
Print a list of the files to standard output rather than extracting them.
diff -r 88ea8de5bdf7 sys/man/1/venti
--- a/sys/man/1/venti Sat May 26 00:00:00 2012 +0200
+++ b/sys/man/1/venti Tue May 29 00:00:00 2012 +0200
@@ -28,7 +28,7 @@
.br
.B venti/copy
[
-.B -fir
+.B -fimrVv
]
[
.B -t
@@ -37,9 +37,6 @@
.I srchost
.I dsthost
.I score
-[
-.I type
-]
.SH DESCRIPTION
Venti is a SHA1-addressed block storage server.
See
@@ -102,14 +99,35 @@
to the server
.IR dsthost .
.PP
+Venti's blocks are arranged in a directed acyclic graph (see venti(6));
+there may be multiple paths from a root score to an
+interior block (for example, if the same file contents are stored
+under multiple names in an archive).
+.I Copy
+runs more efficiently if it does not copy blocks
+(and all their children) multiple times.
The
.B -f
option causes
.I copy
-to run in `fast' mode,
-assuming that if a block already exists on the
-destination Venti server, all its children also
-exist and need not be checked.
+to assume that if a block already exists on the destination
+Venti server, all its children also exist and need not be considered.
+The
+.B -m
+option causes
+.I copy
+to maintain an in-memory list of blocks it has copied
+and avoid considering the same block multiple times.
+The
+.B -f
+option is only useful if the destination Venti server is
+known not to have lost any blocks due to disk corruption
+or other failures.
+The
+.B -m
+option is only useful if enough memory is available to
+hold the block list, which typically requires about 1%
+of the total number of bytes being copied.
.PP
The
.B -i
@@ -138,6 +156,14 @@
replaces pointers to unreadable blocks with
pointers to the zero block.
It writes the new root score to standard output.
+The
+.B -v
+option prints scores as it copies them, total writes, and other
+debugging information.
+The
+.B -V
+option prints debugging information about the Venti protocol
+messages send/received.
.SH SOURCE
.B /sys/src/cmd/venti
.SH SEE ALSO
diff -r 88ea8de5bdf7 sys/man/2/venti
--- a/sys/man/2/venti Sat May 26 00:00:00 2012 +0200
+++ b/sys/man/2/venti Tue May 29 00:00:00 2012 +0200
@@ -2,6 +2,7 @@
.SH NAME
venti \- archival storage server
.SH SYNOPSIS
+.PP
.ft L
#include <u.h>
.br
@@ -10,6 +11,8 @@
#include <venti.h>
.SH DESCRIPTION
The Venti library provides support for writing Venti servers and clients.
+This manual page describes general utility functions.
+.PP
Other manual pages describe the library functions in detail.
.PP
.IR Venti-cache (2)
diff -r 88ea8de5bdf7 sys/man/2/venti-cache
--- a/sys/man/2/venti-cache Sat May 26 00:00:00 2012 +0200
+++ b/sys/man/2/venti-cache Tue May 29 00:00:00 2012 +0200
@@ -7,7 +7,6 @@
vtblockwrite,
vtcachealloc,
vtcacheallocblock,
-vtcacheblocksize,
vtcachefree,
vtcacheglobal,
vtcachelocal,
@@ -36,28 +35,25 @@
.ta +\w'\fLVtBlock* 'u +\w'\fLxxxxxxxx'u
.PP
.B
-VtCache* vtcachealloc(VtConn *z, int blocksize, ulong nblocks);
+VtCache* vtcachealloc(VtConn *z, ulong maxmem);
.PP
.B
void vtcachefree(VtCache *c);
.PP
.B
-u32int vtcacheblocksize(VtCache *c);
-.PP
-.B
u32int vtglobaltolocal(uchar score[VtScoreSize])
.br
.B
void vtlocaltoglobal(u32int local, uchar score[VtScoreSize])
.PP
.B
-VtBlock* vtcacheallocblock(VtCache *c, int type);
+VtBlock* vtcacheallocblock(VtCache *c, int type, ulong size);
.PP
.B
VtBlock* vtcachelocal(VtCache *c, u32int addr, int type);
.PP
.B
-VtBlock* vtcacheglobal(VtCache *c, uchar[VtScoreSize], int type);
+VtBlock* vtcacheglobal(VtCache *c, uchar[VtScoreSize], int type, ulong size);
.PP
.B
void vtblockput(VtBlock *b);
@@ -119,17 +115,13 @@
.IR venti-conn (2)
and
.IR venti-client (2)),
-with room for
-.I nblocks
-of maximum block size
-.I blocksize .
+with
+.I maxmem
+bytes of memory.
.PP
.I Vtcachefree
frees a cache and all the associated blocks.
.PP
-.I Vtcacheblocksize
-returns the cache's maximum block size.
-.PP
.I Vtglobaltolocal
returns the local address corresponding to the given
local
@@ -147,7 +139,9 @@
.PP
.I Vtcacheallocblock
allocates a new local block with the given
-.IR type .
+.I type
+and
+.IR size .
.PP
.I Vtcachelocal
retrieves the local block at address
@@ -160,9 +154,10 @@
.PP
.I Vtcacheglobal
retrieves the block with the given
-.I score
+.IR score ,
+.I dtype
and
-.I dtype
+.I size
from the cache, consulting the Venti server
if necessary.
If passed a local score,
diff -r 88ea8de5bdf7 sys/man/2/venti-conn
--- a/sys/man/2/venti-conn Sat May 26 00:00:00 2012 +0200
+++ b/sys/man/2/venti-conn Tue May 29 00:00:00 2012 +0200
@@ -28,9 +28,15 @@
VtConn* vtconn(int infd, int outfd)
.PP
.B
+int vtreconn(VtConn *z, int infd, int outfd)
+.PP
+.B
VtConn* vtdial(char *addr)
.PP
.B
+int vtredial(VtConn *z, char *addr)
+.PP
+.B
int vtversion(VtConn *z)
.PP
.B
diff -r 88ea8de5bdf7 sys/man/4/vacfs
--- a/sys/man/4/vacfs Sat May 26 00:00:00 2012 +0200
+++ b/sys/man/4/vacfs Tue May 29 00:00:00 2012 +0200
@@ -7,10 +7,6 @@
.B -dips
]
[
-.B -c
-.I cachesize
-]
-[
.B -h
.I host
]
@@ -22,6 +18,10 @@
.B -S
.I srvname
]
+[
+.B -M
+.I mem
+]
.I vacfile
.SH DESCRIPTION
.I Vacfs
@@ -41,11 +41,6 @@
Options to
.I vacfs
are:
-.TF "-c\fI cachesize"
-.PD
-.TP
-.BI -c " cachesize
-The number of file system blocks to cache in memory. The default is 1000 blocks.
.TP
.B -d
Print debugging information to standard error.
@@ -83,6 +78,10 @@
rather than
mounting it on
.IR mtpt .
+.TP
+.BI -M " mem
+The amount of memory, in bytes, allocated to the block cache. The default is 16M.
+.PD
.SH SOURCE
.B /sys/src/cmd/vac
.SH "SEE ALSO"
diff -r 88ea8de5bdf7 sys/man/6/venti
--- a/sys/man/6/venti Sat May 26 00:00:00 2012 +0200
+++ b/sys/man/6/venti Tue May 29 00:00:00 2012 +0200
@@ -434,6 +434,21 @@
upon receiving the
.BR VtTgoodbye
message, the server terminates up the connection.
+.PP
+Version
+.B 04
+of the Venti protocol is similar to version
+.B 02
+(described above)
+but has two changes to accomodates larger payloads.
+First, it replaces the leading 2-byte packet size with
+a 4-byte size.
+Second, the
+.I count
+in the
+.B VtTread
+packet may be either 2 or 4 bytes;
+the total packet length distinguishes the two cases.
.SH SEE ALSO
.IR venti (1),
.IR venti (2),
diff -r 88ea8de5bdf7 sys/man/8/venti-fmt
--- a/sys/man/8/venti-fmt Sat May 26 00:00:00 2012 +0200
+++ b/sys/man/8/venti-fmt Tue May 29 00:00:00 2012 +0200
@@ -13,7 +13,7 @@
.PP
.B venti/fmtarenas
[
-.B -Z
+.B -4Z
]
[
.B -a
diff -r 88ea8de5bdf7 sys/src/cmd/vac/file.c
--- a/sys/src/cmd/vac/file.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/vac/file.c Tue May 29 00:00:00 2012 +0200
@@ -1735,7 +1735,7 @@
static char EBadVacFormat[] = "bad format for vac file";
static VacFs *
-vacfsalloc(VtConn *z, int bsize, int ncache, int mode)
+vacfsalloc(VtConn *z, int bsize, ulong cachemem, int mode)
{
VacFs *fs;
@@ -1743,7 +1743,7 @@
fs->z = z;
fs->bsize = bsize;
fs->mode = mode;
- fs->cache = vtcachealloc(z, bsize, ncache);
+ fs->cache = vtcachealloc(z, cachemem);
return fs;
}
@@ -1772,7 +1772,7 @@
}
VacFs*
-vacfsopen(VtConn *z, char *file, int mode, int ncache)
+vacfsopen(VtConn *z, char *file, int mode, ulong cachemem)
{
int fd;
uchar score[VtScoreSize];
@@ -1793,11 +1793,12 @@
}
close(fd);
}
- return vacfsopenscore(z, score, mode, ncache);
+if(debug) fprint(2, "vacfsopen %V\n", score);
+ return vacfsopenscore(z, score, mode, cachemem);
}
VacFs*
-vacfsopenscore(VtConn *z, u8int *score, int mode, int ncache)
+vacfsopenscore(VtConn *z, u8int *score, int mode, ulong cachemem)
{
VacFs *fs;
int n;
@@ -1808,29 +1809,42 @@
VtEntry e;
n = vtread(z, score, VtRootType, buf, VtRootSize);
- if(n < 0)
+ if(n < 0) {
+if(debug) fprint(2, "read %r\n");
return nil;
+ }
if(n != VtRootSize){
werrstr("vtread on root too short");
+if(debug) fprint(2, "size %d\n", n);
return nil;
}
- if(vtrootunpack(&rt, buf) < 0)
+ if(vtrootunpack(&rt, buf) < 0) {
+if(debug) fprint(2, "unpack: %r\n");
return nil;
+ }
if(strcmp(rt.type, "vac") != 0) {
+if(debug) fprint(2, "bad type %s\n", rt.type);
werrstr("not a vac root");
return nil;
}
- fs = vacfsalloc(z, rt.blocksize, ncache, mode);
+ fs = vacfsalloc(z, rt.blocksize, cachemem, mode);
memmove(fs->score, score, VtScoreSize);
fs->mode = mode;
memmove(e.score, rt.score, VtScoreSize);
e.gen = 0;
- e.psize = rt.blocksize;
+
+ // Don't waste cache memory on pointer blocks
+ // when rt.blocksize is large.
+ e.psize = (rt.blocksize/VtEntrySize)*VtEntrySize;
+ if(e.psize > 60000)
+ e.psize = (60000/VtEntrySize)*VtEntrySize;
+
e.dsize = rt.blocksize;
+if(debug) fprint(2, "openscore %d psize %d dsize %d\n", (int)rt.blocksize, (int)e.psize, (int)e.dsize);
e.type = VtDirType;
e.flags = VtEntryActive;
e.size = 3*VtEntrySize;
@@ -1930,7 +1944,7 @@
* Create a fresh vac fs.
*/
VacFs *
-vacfscreate(VtConn *z, int bsize, int ncache)
+vacfscreate(VtConn *z, int bsize, ulong cachemem)
{
VacFs *fs;
VtFile *f;
@@ -1941,26 +1955,26 @@
VacDir vd;
MetaEntry me;
int psize;
- int mbsize;
- if((fs = vacfsalloc(z, bsize, ncache, VtORDWR)) == nil)
+ if((fs = vacfsalloc(z, bsize, cachemem, VtORDWR)) == nil)
return nil;
-
+
/*
* Fake up an empty vac fs.
*/
- psize = bsize;
+ psize = bsize/VtScoreSize*VtScoreSize;
+ if(psize > 60000)
+ psize = 60000/VtScoreSize*VtScoreSize;
+if(debug) fprint(2, "create bsize %d psize %d\n", bsize, psize);
+
f = vtfilecreateroot(fs->cache, psize, bsize, VtDirType);
+ if(f == nil)
+ sysfatal("vtfilecreateroot: %r");
vtfilelock(f, VtORDWR);
-
- /* Metablocks can't be too big -- they have 16-bit offsets in them. */
- mbsize = bsize;
- if(mbsize >= 56*1024)
- mbsize = 56*1024;
/* Write metablock containing root directory VacDir. */
- b = vtcacheallocblock(fs->cache, VtDataType);
- mbinit(&mb, b->data, mbsize, mbsize/BytesPerEntry);
+ b = vtcacheallocblock(fs->cache, VtDataType, bsize);
+ mbinit(&mb, b->data, bsize, bsize/BytesPerEntry);
memset(&vd, 0, sizeof vd);
vd.elem = "/";
vd.mode = 0777|ModeDir;
@@ -1988,13 +2002,12 @@
/* Second entry: empty metadata stream. */
e.type = VtDataType;
- e.dsize = mbsize;
vtentrypack(&e, buf, 0);
vtfilewrite(f, buf, VtEntrySize, VtEntrySize);
/* Third entry: metadata stream with root directory. */
memmove(e.score, metascore, VtScoreSize);
- e.size = mbsize;
+ e.size = bsize;
vtentrypack(&e, buf, 0);
vtfilewrite(f, buf, VtEntrySize, VtEntrySize*2);
diff -r 88ea8de5bdf7 sys/src/cmd/vac/pack.c
--- a/sys/src/cmd/vac/pack.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/vac/pack.c Tue May 29 00:00:00 2012 +0200
@@ -77,7 +77,7 @@
magic = U32GET(p);
if(magic != MetaMagic && magic != MetaMagic+1) {
- werrstr("bad meta block magic");
+ werrstr("bad meta block magic %#08ux", magic);
return -1;
}
mb->size = U16GET(p+4);
diff -r 88ea8de5bdf7 sys/src/cmd/vac/unvac.c
--- a/sys/src/cmd/vac/unvac.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/vac/unvac.c Tue May 29 00:00:00 2012 +0200
@@ -92,7 +92,7 @@
if(vtconnect(conn) < 0)
sysfatal("vtconnect: %r");
- fs = vacfsopen(conn, argv[0], VtOREAD, 128);
+ fs = vacfsopen(conn, argv[0], VtOREAD, 4<<20);
if(fs == nil)
sysfatal("vacfsopen: %r");
@@ -225,7 +225,7 @@
if(!table && !tostdout && vdir){
// create directory
if((dp = dirstat(name)) == nil){
- if((fd = create(name, OREAD, DMDIR|(mode&0777))) < 0){
+ if((fd = create(name, OREAD, DMDIR|0700|(mode&0777))) < 0){
fprint(2, "mkdir %s: %r\n", name);
vdeclose(vde);
}
diff -r 88ea8de5bdf7 sys/src/cmd/vac/vac.c
--- a/sys/src/cmd/vac/vac.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/vac/vac.c Tue May 29 00:00:00 2012 +0200
@@ -15,6 +15,7 @@
enum
{
BlockSize = 8*1024,
+ CacheSize = 4<<20,
};
struct
@@ -81,8 +82,6 @@
u = unittoull(EARGF(usage()));
if(u < 512)
u = 512;
- if(u > VtMaxLumpSize)
- u = VtMaxLumpSize;
blocksize = u;
break;
case 'd':
@@ -151,10 +150,10 @@
if((outfd = create(archivefile, OWRITE, 0666)) < 0)
sysfatal("create %s: %r", archivefile);
atexit(removevacfile); // because it is new
- if((fs = vacfscreate(z, blocksize, 512)) == nil)
+ if((fs = vacfscreate(z, blocksize, CacheSize)) == nil)
sysfatal("vacfscreate: %r");
}else{
- if((fs = vacfsopen(z, archivefile, VtORDWR, 512)) == nil)
+ if((fs = vacfsopen(z, archivefile, VtORDWR, CacheSize)) == nil)
sysfatal("vacfsopen %s: %r", archivefile);
if((fdiff = recentarchive(fs, oldpath)) != nil){
if(verbose)
@@ -194,13 +193,13 @@
else if((outfd = create(vacfile, OWRITE, 0666)) < 0)
sysfatal("create %s: %r", vacfile);
atexit(removevacfile);
- if((fs = vacfscreate(z, blocksize, 512)) == nil)
+ if((fs = vacfscreate(z, blocksize, CacheSize)) == nil)
sysfatal("vacfscreate: %r");
f = vacfsgetroot(fs);
fdiff = nil;
if(diffvac){
- if((fsdiff = vacfsopen(z, diffvac, VtOREAD, 128)) == nil)
+ if((fsdiff = vacfsopen(z, diffvac, VtOREAD, CacheSize)) == nil)
warn("vacfsopen %s: %r", diffvac);
else
fdiff = vacfsgetroot(fsdiff);
@@ -414,7 +413,7 @@
vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
{
char *elem, *s;
- static char buf[65536];
+ static char *buf;
int fd, i, n, bsize;
vlong off;
Dir *dk; // kids
@@ -461,7 +460,11 @@
if(vacfilesetdir(f, &vd) < 0)
warn("vacfilesetdir %s: %r", name);
-
+
+ bsize = fs->bsize;
+ if(buf == nil)
+ buf = vtmallocz(bsize);
+
if(d->mode&DMDIR){
while((n = dirread(fd, &dk)) > 0){
for(i=0; i<n; i++){
@@ -476,7 +479,6 @@
}
}else{
off = 0;
- bsize = fs->bsize;
if(fdiff){
/*
* Copy fdiff's contents into f by moving the score.
@@ -644,7 +646,7 @@
if(strlen(name) < 4 || strcmp(name+strlen(name)-4, ".vac") != 0)
return -1;
- if((mfs = vacfsopen(z, name, VtOREAD, 100)) == nil)
+ if((mfs = vacfsopen(z, name, VtOREAD, CacheSize)) == nil)
return -1;
if(verbose)
fprint(2, "merging %s\n", name);
diff -r 88ea8de5bdf7 sys/src/cmd/vac/vac.h
--- a/sys/src/cmd/vac/vac.h Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/vac/vac.h Tue May 29 00:00:00 2012 +0200
@@ -95,9 +95,9 @@
VtCache *cache;
};
-VacFs *vacfsopen(VtConn *z, char *file, int mode, int ncache);
-VacFs *vacfsopenscore(VtConn *z, u8int *score, int mode, int ncache);
-VacFs *vacfscreate(VtConn *z, int bsize, int ncache);
+VacFs *vacfsopen(VtConn *z, char *file, int mode, ulong cachemem);
+VacFs *vacfsopenscore(VtConn *z, u8int *score, int mode, ulong cachemem);
+VacFs *vacfscreate(VtConn *z, int bsize, ulong cachemem);
void vacfsclose(VacFs *fs);
int vacfssync(VacFs *fs);
int vacfssnapshot(VacFs *fs, char *src, char *dst);
diff -r 88ea8de5bdf7 sys/src/cmd/vac/vacfs.c
--- a/sys/src/cmd/vac/vacfs.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/vac/vacfs.c Tue May 29 00:00:00 2012 +0200
@@ -2,16 +2,10 @@
#include <fcall.h>
#include "vac.h"
-#define convM2Su(a, b, c, d) convM2S(a, b, c)
-#define convS2Mu(a, b, c, d) convS2M(a, b, c)
-#define convM2Du(a, b, c, d) convM2D(a, b, c)
-#define convD2Mu(a, b, c, d) convD2M(a, b, c)
-
typedef struct Fid Fid;
enum
{
- Stacksize = 320 * 1024, /* was 32K */
OPERM = 0x3 /* mask of all permission types in open mode */
};
@@ -49,7 +43,6 @@
VacFs *fs;
VtConn *conn;
int noperm;
-int dotu;
char *defmnt;
Fid * newfid(int);
@@ -117,6 +110,31 @@
noted(NDFLT);
}
+#define TWID64 ~(u64int)0
+static u64int
+unittoull(char *s)
+{
+ char *es;
+ u64int n;
+
+ if(s == nil)
+ return TWID64;
+ n = strtoul(s, &es, 0);
+ if(*es == 'k' || *es == 'K'){
+ n *= 1024;
+ es++;
+ }else if(*es == 'm' || *es == 'M'){
+ n *= 1024*1024;
+ es++;
+ }else if(*es == 'g' || *es == 'G'){
+ n *= 1024*1024*1024;
+ es++;
+ }
+ if(*es != '\0')
+ return TWID64;
+ return n;
+}
+
void
threadmain(int argc, char *argv[])
{
@@ -124,10 +142,10 @@
int p[2], fd;
int stdio;
char *host = nil;
- long ncache;
+ ulong mem;
+ mem = 16<<20;
stdio = 0;
- ncache = 256;
fmtinstall('H', encodefmt);
fmtinstall('V', vtscorefmt);
fmtinstall('F', vtfcallfmt);
@@ -139,9 +157,6 @@
fmtinstall('F', fcallfmt);
dflag = 1;
break;
- case 'c':
- ncache = atoi(EARGF(usage()));
- break;
case 'i':
defmnt = nil;
stdio = 1;
@@ -157,6 +172,9 @@
case 's':
defsrv = "vacfs";
break;
+ case 'M':
+ mem = unittoull(EARGF(usage()));
+ break;
case 'm':
defmnt = EARGF(usage());
break;
@@ -190,7 +208,7 @@
if(vtconnect(conn) < 0)
sysfatal("vtconnect: %r");
- fs = vacfsopen(conn, argv[0], VtOREAD, ncache);
+ fs = vacfsopen(conn, argv[0], VtOREAD, mem);
if(fs == nil)
sysfatal("vacfsopen: %r");
@@ -202,7 +220,7 @@
srvfd = p[1];
}
- procrfork(srv, 0, Stacksize, RFFDG|RFNAMEG|RFNOTEG);
+ procrfork(srv, 0, 32 * 1024, RFFDG|RFNAMEG|RFNOTEG);
if(!stdio){
close(p[0]);
@@ -234,7 +252,7 @@
void
usage(void)
{
- fprint(2, "usage: %s [-sd] [-h host] [-c ncache] [-m mountpoint] vacfile\n", argv0);
+ fprint(2, "usage: %s [-sd] [-h host] [-m mountpoint] [-M mem] vacfile\n", argv0);
threadexitsall("usage");
}
@@ -258,10 +276,6 @@
if(strncmp(rhdr.version, "9P2000", 6) != 0)
return vtstrdup("unrecognized 9P version");
thdr.version = "9P2000";
- if(strncmp(rhdr.version, "9P2000.u", 8) == 0){
- dotu = 1;
- thdr.version = "9P2000.u";
- }
return nil;
}
@@ -634,7 +648,7 @@
dir.gid = vd->gid;
dir.muid = vd->mid;
- ret = convD2Mu(&dir, p, np, dotu);
+ ret = convD2M(&dir, p, np);
return ret;
}
@@ -708,7 +722,7 @@
n = read9pmsg(mfd[0], mdata, sizeof mdata);
if(n <= 0)
break;
- if(convM2Su(mdata, n, &rhdr, dotu) != n)
+ if(convM2S(mdata, n, &rhdr) != n)
sysfatal("convM2S conversion error");
if(dflag)
@@ -729,9 +743,9 @@
thdr.tag = rhdr.tag;
if(dflag)
fprint(2, "vacfs:->%F\n", &thdr);
- n = convS2Mu(&thdr, mdata, messagesize, dotu);
+ n = convS2M(&thdr, mdata, messagesize);
if(n <= BIT16SZ)
- sysfatal("convS2Mu conversion error");
+ sysfatal("convS2M conversion error");
if(err)
vtfree(err);
diff -r 88ea8de5bdf7 sys/src/cmd/venti/copy.c
--- a/sys/src/cmd/venti/copy.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/copy.c Tue May 29 00:00:00 2012 +0200
@@ -2,9 +2,16 @@
#include <libc.h>
#include <venti.h>
#include <libsec.h>
+#include <thread.h>
#include <avl.h>
#include <bin.h>
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65535,
+};
+
int changes;
int rewrite;
int ignoreerrors;
@@ -71,12 +78,12 @@
void
usage(void)
{
- fprint(2, "usage: %s [-fimrv] [-t type] srchost dsthost score\n", argv0);
- exits("usage");
+ fprint(2, "usage: %s [-fimrVv] [-t type] srchost dsthost score\n", argv0);
+ threadexitsall("usage");
}
void
-walk(uchar score[VtScoreSize], uint type, int base)
+walk(uchar score[VtScoreSize], uint type, int base, int depth)
{
int i, n;
uchar *buf;
@@ -84,6 +91,12 @@
VtEntry e;
VtRoot root;
+ if(verbose){
+ for(i = 0; i < depth; i++)
+ fprint(2, " ");
+ fprint(2, "-> %d %d %d %V\n", depth, type, base, score);
+ }
+
if(memcmp(score, vtzeroscore, VtScoreSize) == 0 || memcmp(score, zeroscore, VtScoreSize) == 0)
return;
@@ -116,8 +129,8 @@
fprint(2, "warning: could not unpack root in %V %d\n", score, type);
break;
}
- walk(root.prev, VtRootType, 0);
- walk(root.score, VtDirType, 0);
+ walk(root.prev, VtRootType, 0, depth+1);
+ walk(root.score, VtDirType, 0, depth+1);
if(rewrite)
vtrootpack(&root, buf); /* walk might have changed score */
break;
@@ -130,7 +143,7 @@
}
if(!(e.flags & VtEntryActive))
continue;
- walk(e.score, e.type, e.type&VtTypeBaseMask);
+ walk(e.score, e.type, e.type&VtTypeBaseMask, depth+1);
/*
* Don't repack unless we're rewriting -- some old
* vac files have psize==0 and dsize==0, and these
@@ -149,7 +162,7 @@
default: /* pointers */
for(i=0; i<n; i+=VtScoreSize)
if(memcmp(buf+i, vtzeroscore, VtScoreSize) != 0)
- walk(buf+i, type-1, base);
+ walk(buf+i, type-1, base, depth+1);
break;
}
@@ -161,10 +174,16 @@
sha1(buf, n, score, nil);
sysfatal("writing block %V (type %d): %r", score, type);
}
- if(!rewrite && memcmp(score, nscore, VtScoreSize) != 0){
- fprint(2, "not rewriting: wrote %V got %V\n", score, nscore);
- abort();
- sysfatal("not rewriting: wrote %V got %V", score, nscore);
+ if(!rewrite && memcmp(score, nscore, VtScoreSize) != 0)
+ sysfatal("not rewriting: wrote %V got %V", score, nscore);
+
+ if((type !=0 || base !=0) && verbose){
+ n = vtzerotruncate(type, buf, n);
+ sha1(buf, n, score, nil);
+
+ for(i = 0; i < depth; i++)
+ fprint(2, " ");
+ fprint(2, "<- %V\n", score);
}
markvisited(score, type);
@@ -172,7 +191,7 @@
}
void
-main(int argc, char *argv[])
+threadmain(int argc, char *argv[])
{
int type, n;
uchar score[VtScoreSize];
@@ -248,7 +267,7 @@
sysfatal("could not find block %V of any type", score);
}
- walk(score, type, VtDirType);
+ walk(score, type, VtDirType, 0);
if(changes)
print("%s:%V (%d pointers rewritten)\n", prefix, score, changes);
@@ -258,5 +277,5 @@
if(vtsync(zdst) < 0)
sysfatal("could not sync dst server: %r");
- exits(0);
+ threadexitsall(0);
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/dump.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/src/cmd/venti/dump.c Tue May 29 00:00:00 2012 +0200
@@ -0,0 +1,139 @@
+#include <u.h>
+#include <libc.h>
+#include <bio.h>
+#include <venti.h>
+#include <libsec.h>
+#include <thread.h>
+
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65535,
+};
+
+VtConn *z;
+char *host;
+
+void
+usage(void)
+{
+ fprint(2, "usage: venti/dump [-h host] score\n");
+ threadexitsall("usage");
+}
+
+Biobuf bout;
+char spaces[256];
+
+void
+dump(int indent, uchar *score, int type)
+{
+ int i, n;
+ uchar *buf;
+ VtEntry e;
+ VtRoot root;
+
+ if(spaces[0] == 0)
+ memset(spaces, ' ', sizeof spaces-1);
+
+ buf = vtmallocz(VtMaxLumpSize);
+ if(memcmp(score, vtzeroscore, VtScoreSize) == 0)
+ n = 0;
+ else
+ n = vtread(z, score, type, buf, VtMaxLumpSize);
+ if(n < 0){
+ Bprint(&bout, "%.*serror reading %V: %r\n", indent*4, spaces, score);
+ goto out;
+ }
+ switch(type){
+ case VtRootType:
+ if(vtrootunpack(&root, buf) < 0){
+ Bprint(&bout, "%.*serror unpacking root %V: %r\n", indent*4, spaces, score);
+ goto out;
+ }
+ Bprint(&bout, "%.*s%V root name=%s type=%s prev=%V bsize=%ld\n",
+ indent*4, spaces, score, root.name, root.type, root.prev, root.blocksize);
+ dump(indent+1, root.score, VtDirType);
+ break;
+
+ case VtDirType:
+ Bprint(&bout, "%.*s%V dir n=%d\n", indent*4, spaces, score, n);
+ for(i=0; i*VtEntrySize<n; i++){
+ if(vtentryunpack(&e, buf, i) < 0){
+ Bprint(&bout, "%.*s%d: cannot unpack\n", indent+1, spaces, i);
+ continue;
+ }
+ Bprint(&bout, "%.*s%d: gen=%#lux psize=%ld dsize=%ld type=%d flags=%#x size=%llud score=%V\n",
+ (indent+1)*4, spaces, i, e.gen, e.psize, e.dsize, e.type, e.flags, e.size, e.score);
+ dump(indent+2, e.score, e.type);
+ }
+ break;
+
+ case VtDataType:
+ Bprint(&bout, "%.*s%V data n=%d", indent*4, spaces, score, n);
+ for(i=0; i<n; i++){
+ if(i%16 == 0)
+ Bprint(&bout, "\n%.*s", (indent+1)*4, spaces);
+ Bprint(&bout, " %02x", buf[i]);
+ }
+ Bprint(&bout, "\n");
+ break;
+
+ default:
+ if(type >= VtDirType)
+ Bprint(&bout, "%.*s%V dir+%d\n", indent*4, spaces, score, type-VtDirType);
+ else
+ Bprint(&bout, "%.*s%V data+%d\n", indent*4, spaces, score, type-VtDirType);
+ for(i=0; i<n; i+=VtScoreSize)
+ dump(indent+1, buf+i, type-1);
+ break;
+ }
+out:
+ free(buf);
+}
+
+
+void
+threadmain(int argc, char *argv[])
+{
+ int type, n;
+ uchar score[VtScoreSize];
+ uchar *buf;
+ char *prefix;
+
+ fmtinstall('F', vtfcallfmt);
+ fmtinstall('V', vtscorefmt);
+
+ ARGBEGIN{
+ case 'h':
+ host = EARGF(usage());
+ break;
+ default:
+ usage();
+ }ARGEND
+
+ if(argc != 1)
+ usage();
+
+ if(vtparsescore(argv[0], &prefix, score) < 0)
+ sysfatal("could not parse score: %r");
+
+ buf = vtmallocz(VtMaxLumpSize);
+ z = vtdial(host);
+ if(z == nil)
+ sysfatal("dialing venti: %r");
+ if(vtconnect(z) < 0)
+ sysfatal("vtconnect src: %r");
+
+ for(type=0; type<VtMaxType; type++){
+ n = vtread(z, score, type, buf, VtMaxLumpSize);
+ if(n >= 0)
+ goto havetype;
+ }
+ sysfatal("cannot find block %V", score);
+
+havetype:
+ Binit(&bout, 1, OWRITE);
+ dump(0, score, type);
+ Bflush(&bout);
+ threadexitsall(nil);
+}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/mkfile
--- a/sys/src/cmd/venti/mkfile Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/mkfile Tue May 29 00:00:00 2012 +0200
@@ -6,6 +6,7 @@
ro\
sync\
write\
+ dump\
BIN=/$objtype/bin/venti
@@ -14,7 +15,7 @@
CFLAGS=$CFLAGS -I.
-extra:V: $O.devnull $O.mkroot $O.randtest $O.readlist $O.root
+extra:V: $O.devnull $O.mkroot $O.randtest $O.readlist $O.ro $O.root
all:V: srv.all.dir
install:V: srv.install.dir
diff -r 88ea8de5bdf7 sys/src/cmd/venti/randtest.c
--- a/sys/src/cmd/venti/randtest.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/randtest.c Tue May 29 00:00:00 2012 +0200
@@ -96,8 +96,9 @@
buf = vtmalloc(blocksize);
cur = 0;
packets = totalbytes/blocksize;
- if(maxpackets == 0)
- maxpackets = packets;
+ if(maxpackets > 0 && maxpackets < packets)
+ packets = maxpackets;
+ totalbytes = (vlong)packets * blocksize;
order = vtmalloc(packets*sizeof order[0]);
for(i=0; i<packets; i++)
order[i] = i;
@@ -109,7 +110,7 @@
order[j] = t;
}
}
- for(i=0; i<packets && i<maxpackets; i++){
+ for(i=0; i<packets; i++){
memmove(buf, template, blocksize);
*(uint*)buf = order[i];
if(c){
diff -r 88ea8de5bdf7 sys/src/cmd/venti/read.c
--- a/sys/src/cmd/venti/read.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/read.c Tue May 29 00:00:00 2012 +0200
@@ -4,6 +4,12 @@
#include <libsec.h>
#include <thread.h>
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65535,
+};
+
void
usage(void)
{
diff -r 88ea8de5bdf7 sys/src/cmd/venti/readfile.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/src/cmd/venti/readfile.c Tue May 29 00:00:00 2012 +0200
@@ -0,0 +1,114 @@
+#include <u.h>
+#include <libc.h>
+#include <venti.h>
+#include <libsec.h>
+#include <thread.h>
+
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65535,
+};
+
+int chatty;
+
+void
+usage(void)
+{
+ fprint(2, "usage: readfile [-v] [-h host] score\n");
+ threadexitsall("usage");
+}
+
+void
+threadmain(int argc, char *argv[])
+{
+ int n;
+ uchar score[VtScoreSize];
+ uchar *buf;
+ char *host, *type;
+ vlong off;
+ VtEntry e;
+ VtRoot root;
+ VtCache *c;
+ VtConn *z;
+ VtFile *f;
+
+ quotefmtinstall();
+ fmtinstall('F', vtfcallfmt);
+ fmtinstall('V', vtscorefmt);
+
+ host = nil;
+ ARGBEGIN{
+ case 'V':
+ chattyventi++;
+ break;
+ case 'h':
+ host = EARGF(usage());
+ break;
+ case 'v':
+ chatty++;
+ break;
+ default:
+ usage();
+ break;
+ }ARGEND
+
+ if(argc != 1)
+ usage();
+
+ type = nil;
+ if(vtparsescore(argv[0], &type, score) < 0)
+ sysfatal("could not parse score '%s': %r", argv[0]);
+ if(type == nil || strcmp(type, "file") != 0)
+ sysfatal("bad score - not file:...");
+
+ buf = vtmallocz(VtMaxLumpSize);
+
+ z = vtdial(host);
+ if(z == nil)
+ sysfatal("could not connect to server: %r");
+
+ if(vtconnect(z) < 0)
+ sysfatal("vtconnect: %r");
+
+ // root block ...
+ n = vtread(z, score, VtRootType, buf, VtMaxLumpSize);
+ if(n < 0)
+ sysfatal("could not read root %V: %r", score);
+ if(n != VtRootSize)
+ sysfatal("root block %V is wrong size %d != %d", score, n, VtRootSize);
+ if(vtrootunpack(&root, buf) < 0)
+ sysfatal("unpacking root block %V: %r", score);
+ if(strcmp(root.type, "file") != 0)
+ sysfatal("bad root type %q (not 'file')", root.type);
+ if(chatty)
+ fprint(2, "%V: %q %q %V %d %V\n",
+ score, root.name, root.type,
+ root.score, root.blocksize, root.prev);
+
+ // ... points at entry block
+ n = vtread(z, root.score, VtDirType, buf, VtMaxLumpSize);
+ if(n < 0)
+ sysfatal("could not read entry %V: %r", root.score);
+ if(n != VtEntrySize)
+ sysfatal("dir block %V is wrong size %d != %d", root.score, n, VtEntrySize);
+ if(vtentryunpack(&e, buf, 0) < 0)
+ sysfatal("unpacking dir block %V: %r", root.score);
+ if((e.type&VtTypeBaseMask) != VtDataType)
+ sysfatal("not a single file");
+
+ // open and read file
+ c = vtcachealloc(z, root.blocksize*32);
+ if(c == nil)
+ sysfatal("vtcachealloc: %r");
+ f = vtfileopenroot(c, &e);
+ if(f == nil)
+ sysfatal("vtfileopenroot: %r");
+ off = 0;
+ vtfilelock(f, VtOREAD);
+ while((n = vtfileread(f, buf, VtMaxLumpSize, off)) > 0){
+ write(1, buf, n);
+ off += n;
+ }
+ threadexitsall(0);
+}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/readlist.c
--- a/sys/src/cmd/venti/readlist.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/readlist.c Tue May 29 00:00:00 2012 +0200
@@ -4,6 +4,12 @@
#include <venti.h>
#include <bio.h>
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65535,
+};
+
char *host;
Biobuf b;
VtConn *z;
diff -r 88ea8de5bdf7 sys/src/cmd/venti/root.c
--- a/sys/src/cmd/venti/root.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/root.c Tue May 29 00:00:00 2012 +0200
@@ -4,6 +4,12 @@
#include <libsec.h>
#include <thread.h>
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65535,
+};
+
void
usage(void)
{
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/arena.c
--- a/sys/src/cmd/venti/srv/arena.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/arena.c Tue May 29 00:00:00 2012 +0200
@@ -31,7 +31,9 @@
{
needzeroscore(); /* OS X */
+ qlock(&sumlock);
sumwait.l = &sumlock;
+ qunlock(&sumlock);
if(vtproc(sumproc, nil) < 0){
seterr(EOk, "can't start arena checksum slave: %r");
@@ -478,9 +480,6 @@
{
ASum *as;
- if(sumwait.l == nil)
- return;
-
as = MK(ASum);
if(as == nil)
return;
@@ -492,7 +491,12 @@
else
sumq = as;
sumqtail = as;
- rwakeup(&sumwait);
+ /*
+ * Might get here while initializing arenas,
+ * before initarenasum has been called.
+ */
+ if(sumwait.l)
+ rwakeup(&sumwait);
qunlock(&sumlock);
}
@@ -513,7 +517,6 @@
qunlock(&sumlock);
arena = as->arena;
free(as);
-
sumarena(arena);
}
}
@@ -683,9 +686,8 @@
logerr(ECorrupt, "arena tail name %s head %s",
arena->name, head.name);
else if(arena->clumpmagic != head.clumpmagic)
- logerr(ECorrupt, "arena %d tail clumpmagic 0x%lux head 0x%lux",
- debugarena, (ulong)arena->clumpmagic,
- (ulong)head.clumpmagic);
+ logerr(ECorrupt, "arena tail clumpmagic 0x%lux head 0x%lux",
+ (ulong)arena->clumpmagic, (ulong)head.clumpmagic);
else if(arena->version != head.version)
logerr(ECorrupt, "arena tail version %d head version %d",
arena->version, head.version);
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/bloom.c
--- a/sys/src/cmd/venti/srv/bloom.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/bloom.c Tue May 29 00:00:00 2012 +0200
@@ -229,6 +229,22 @@
runlock(&b->lk);
}
+void
+markbloomfiltern(Bloom *b, u8int score[][20], int n)
+{
+ int i;
+
+ if(b == nil || b->data == nil)
+ return;
+
+ rlock(&b->lk);
+ qlock(&b->mod);
+ for(i=0; i<n; i++)
+ _markbloomfilter(b, score[i]);
+ qunlock(&b->mod);
+ runlock(&b->lk);
+}
+
static void
bloomwriteproc(void *v)
{
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/buildindex.c
--- a/sys/src/cmd/venti/srv/buildindex.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/buildindex.c Tue May 29 00:00:00 2012 +0200
@@ -11,6 +11,20 @@
MaxBufSize = 4*1024*1024,
};
+typedef struct IEntryBuf IEntryBuf;
+struct IEntryBuf
+{
+ IEntry ie[100];
+ int nie;
+};
+
+typedef struct ScoreBuf ScoreBuf;
+struct ScoreBuf
+{
+ uchar score[100][VtScoreSize];
+ int nscore;
+};
+
int dumb;
int errors;
char **isect;
@@ -36,7 +50,7 @@
void
usage(void)
{
- fprint(2, "usage: buildindex [-b] [-i isect]... [-M imem] venti.conf\n");
+ fprint(2, "usage: buildindex [-bd] [-i isect]... [-M imem] venti.conf\n");
threadexitsall("usage");
}
@@ -121,10 +135,10 @@
/* start index procs */
fprint(2, "%T read index\n");
- isectdonechan = chancreate(sizeof(void*), 0);
+ isectdonechan = chancreate(sizeof(void*), 1);
for(i=0; i<ix->nsects; i++){
if(shouldprocess(ix->sects[i])){
- ix->sects[i]->writechan = chancreate(sizeof(IEntry), 0);
+ ix->sects[i]->writechan = chancreate(sizeof(IEntryBuf), 1);
vtproc(isectproc, ix->sects[i]);
}
}
@@ -216,12 +230,17 @@
ClumpInfo *ci, *cis;
IEntry ie;
Part *p;
+ IEntryBuf *buf, *b;
+ uchar *score;
+ ScoreBuf sb;
p = v;
threadsetname("arenaproc %s", p->name);
+ buf = MKNZ(IEntryBuf, ix->nsects);
nskip = 0;
tot = 0;
+ sb.nscore = 0;
cis = MKN(ClumpInfo, ClumpChunks);
for(i=0; i<ix->narenas; i++){
a = ix->arenas[i];
@@ -260,10 +279,23 @@
tot++;
x = indexsect(ix, ie.score);
assert(0 <= x && x < ix->nsects);
- if(ix->sects[x]->writechan)
- send(ix->sects[x]->writechan, &ie);
- if(ix->bloom)
- markbloomfilter(ix->bloom, ie.score);
+ if(ix->sects[x]->writechan) {
+ b = &buf[x];
+ b->ie[b->nie] = ie;
+ b->nie++;
+ if(b->nie == nelem(b->ie)) {
+ send(ix->sects[x]->writechan, b);
+ b->nie = 0;
+ }
+ }
+ if(ix->bloom) {
+ score = sb.score[sb.nscore++];
+ scorecp(score, ie.score);
+ if(sb.nscore == nelem(sb.score)) {
+ markbloomfiltern(ix->bloom, sb.score, sb.nscore);
+ sb.nscore = 0;
+ }
+ }
}
}
}
@@ -272,6 +304,14 @@
}
add(&arenaentries, tot);
add(&skipentries, nskip);
+
+ for(i=0; i<ix->nsects; i++)
+ if(ix->sects[i]->writechan && buf[i].nie > 0)
+ send(ix->sects[i]->writechan, &buf[i]);
+ free(buf);
+ free(cis);
+ if(ix->bloom && sb.nscore > 0)
+ markbloomfiltern(ix->bloom, sb.score, sb.nscore);
sendp(arenadonechan, p);
}
@@ -751,6 +791,7 @@
uchar *data, *p;
Buf *buf;
IEntry ie;
+ IEntryBuf ieb;
IPool *ipool;
ISect *is;
Minibuf *mbuf, *mb;
@@ -813,7 +854,7 @@
}
if (nbuf == 0) {
fprint(2, "%s: brand-new index, no work to do\n", argv0);
- threadexitsall(0);
+ threadexitsall(nil);
}
/* size buffer to use extra memory */
@@ -850,14 +891,17 @@
assert(p == data+nbuf*bufsize);
n = 0;
- while(recv(is->writechan, &ie) == 1){
- if(ie.ia.addr == 0)
+ while(recv(is->writechan, &ieb) == 1){
+ if(ieb.nie == 0)
break;
- buck = score2bucket(is, ie.score);
- i = buck/bufbuckets;
- assert(i < nbuf);
- bwrite(&buf[i], &ie);
- n++;
+ for(j=0; j<ieb.nie; j++){
+ ie = ieb.ie[j];
+ buck = score2bucket(is, ie.score);
+ i = buck/bufbuckets;
+ assert(i < nbuf);
+ bwrite(&buf[i], &ie);
+ n++;
+ }
}
add(&indexentries, n);
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/config.c
--- a/sys/src/cmd/venti/srv/config.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/config.c Tue May 29 00:00:00 2012 +0200
@@ -86,7 +86,9 @@
}
line = estrdup(s);
i = getfields(s, flds, MaxArgs + 1, 1, " \t\r");
- if(i == 2 && strcmp(flds[0], "isect") == 0){
+ if(i > 0 && strcmp(flds[0], "mgr") == 0){
+ /* do nothing */
+ }else if(i == 2 && strcmp(flds[0], "isect") == 0){
sv = MKN(ISect*, config->nsects + 1);
for(i = 0; i < config->nsects; i++)
sv[i] = config->sects[i];
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/dat.h
--- a/sys/src/cmd/venti/srv/dat.h Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/dat.h Tue May 29 00:00:00 2012 +0200
@@ -37,6 +37,12 @@
enum
{
+ /*
+ * formerly fundamental constant,
+ * now a server-imposed limitation.
+ */
+ VtMaxLumpSize = 56*1024,
+
ABlockLog = 9, /* log2(512), the quantum for reading arenas */
ANameSize = 64,
MaxDiskBlock = 64*1024, /* max. allowed size for a disk block */
@@ -734,6 +740,7 @@
extern u8int zeroscore[VtScoreSize];
extern int compressblocks;
extern int writestodevnull; /* dangerous - for performance debugging */
+extern int bootstrap; /* writes but does not index - cannot read */
extern int collectstats;
extern QLock memdrawlock;
extern int icachesleeptime;
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/fmtarenas.c
--- a/sys/src/cmd/venti/srv/fmtarenas.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/fmtarenas.c Tue May 29 00:00:00 2012 +0200
@@ -5,7 +5,7 @@
void
usage(void)
{
- fprint(2, "usage: fmtarenas [-Z] [-b blocksize] [-a arenasize] name file\n");
+ fprint(2, "usage: fmtarenas [-4Z] [-a arenasize] [-b blocksize] name file\n");
threadexitsall(0);
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/fmtbloom.c
--- a/sys/src/cmd/venti/srv/fmtbloom.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/fmtbloom.c Tue May 29 00:00:00 2012 +0200
@@ -7,7 +7,7 @@
void
usage(void)
{
- fprint(2, "usage: fmtbloom [-s size] [-n nblocks | -N nhash] file\n");
+ fprint(2, "usage: fmtbloom [-n nblocks | -N nhash] [-s size] file\n");
threadexitsall(0);
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/fmtindex.c
--- a/sys/src/cmd/venti/srv/fmtindex.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/fmtindex.c Tue May 29 00:00:00 2012 +0200
@@ -5,7 +5,7 @@
void
usage(void)
{
- fprint(2, "usage: fmtindex [-a] config\n");
+ fprint(2, "usage: fmtindex [-a] venti.conf\n");
threadexitsall(0);
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/fmtisect.c
--- a/sys/src/cmd/venti/srv/fmtisect.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/fmtisect.c Tue May 29 00:00:00 2012 +0200
@@ -5,7 +5,7 @@
void
usage(void)
{
- fprint(2, "usage: fmtisect [-Z] [-b blocksize] name file\n");
+ fprint(2, "usage: fmtisect [-1Z] [-b blocksize] name file\n");
threadexitsall(0);
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/fns.h
--- a/sys/src/cmd/venti/srv/fns.h Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/fns.h Tue May 29 00:00:00 2012 +0200
@@ -105,6 +105,7 @@
int lookupscore(u8int *score, int type, IAddr *ia);
int maparenas(AMap *am, Arena **arenas, int n, char *what);
void markbloomfilter(Bloom*, u8int*);
+void markbloomfiltern(Bloom*, u8int[][20], int);
uint msec(void);
int namecmp(char *s, char *t);
void namecp(char *dst, char *src);
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/httpd.c
--- a/sys/src/cmd/venti/srv/httpd.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/httpd.c Tue May 29 00:00:00 2012 +0200
@@ -396,6 +396,7 @@
"ignorebloom", &ignorebloom,
"syncwrites", &syncwrites,
"icacheprefetch", &icacheprefetch,
+ "bootstrap", &bootstrap,
0
};
@@ -562,6 +563,8 @@
hprint(hout, " mem=sealed");
if(arena->diskstats.sealed)
hprint(hout, " disk=sealed");
+ if(arena->inqueue)
+ hprint(hout, " inqueue");
hprint(hout, "\n");
if(scorecmp(zeroscore, arena->score) != 0)
hprint(hout, "\tscore=%V\n", arena->score);
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/icache.c
--- a/sys/src/cmd/venti/srv/icache.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/icache.c Tue May 29 00:00:00 2012 +0200
@@ -375,6 +375,9 @@
{
IEntry *ie;
+ if(bootstrap)
+ return -1;
+
qlock(&icache.lock);
addstat(StatIcacheLookup, 1);
if((ie = ihashlookup(icache.hash, score, type)) != nil){
@@ -405,6 +408,9 @@
{
ISum *toload;
+ if(bootstrap)
+ return -1;
+
qlock(&icache.lock);
icacheinsert(score, ia, state);
if(state == IEClean)
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/index.c
--- a/sys/src/cmd/venti/srv/index.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/index.c Tue May 29 00:00:00 2012 +0200
@@ -75,18 +75,44 @@
blocksize = ix->blocksize;
for(i = 0; i < ix->nsects; i++){
is = sects[i];
- if(namecmp(ix->name, is->index) != 0
- || is->blocksize != blocksize
- || is->tabsize != tabsize
- || namecmp(is->name, ix->smap[i].name) != 0
- || is->start != ix->smap[i].start
- || is->stop != ix->smap[i].stop
- || last != is->start
- || is->start > is->stop){
- seterr(ECorrupt, "inconsistent index sections in %s", ix->name);
+ if(namecmp(is->index, ix->name) != 0) {
+ seterr(ECorrupt, "%s: index name is %s, not %s",
+ sects[i]->part->name, is->index, ix->name);
+ bad:
freeindex(ix);
return nil;
}
+ if(is->blocksize != blocksize) {
+ seterr(ECorrupt, "%s: blocksize is %d, not %d",
+ sects[i]->part->name, (int)is->blocksize, (int)blocksize);
+ goto bad;
+ }
+ if(is->tabsize != tabsize) {
+ seterr(ECorrupt, "%s: tabsize is %d, not %d",
+ sects[i]->part->name, (int)is->tabsize, (int)tabsize);
+ goto bad;
+ }
+ if(namecmp(is->name, ix->smap[i].name) != 0) {
+ seterr(ECorrupt, "%s: name is %s, not %s",
+ sects[i]->part->name, is->name, ix->smap[i].name);
+ goto bad;
+ }
+ if(is->start != ix->smap[i].start || is->stop != ix->smap[i].stop) {
+ seterr(ECorrupt, "%s: range is %lld,%lld, not %lld,%lld",
+ sects[i]->part->name, is->start, is->stop,
+ ix->smap[i].start, ix->smap[i].stop);
+ goto bad;
+ }
+ if(is->start > is->stop) {
+ seterr(ECorrupt, "%s: invalid range %lld,%lld",
+ sects[i]->part->name, is->start, is->stop);
+ goto bad;
+ }
+ if(is->start != last || is->start > is->stop) {
+ seterr(ECorrupt, "%s: range %lld-%lld, but last section ended at %lld",
+ sects[i]->part->name, is->start, is->stop, last);
+ goto bad;
+ }
last = is->stop;
}
ix->tabsize = tabsize;
@@ -272,11 +298,15 @@
return nil;
}
if(blocksize != sects[i]->blocksize){
- seterr(EOk, "mismatched block sizes in index sections");
+ seterr(EOk, "%s has block size %d, but %s has %d",
+ sects[0]->part->name, (int)blocksize,
+ sects[i]->part->name, (int)sects[i]->blocksize);
return nil;
}
if(tabsize != sects[i]->tabsize){
- seterr(EOk, "mismatched config table sizes in index sections");
+ seterr(EOk, "%s has table size %d, but %s has %d",
+ sects[0]->part->name, (int)tabsize,
+ sects[i]->part->name, (int)sects[i]->tabsize);
return nil;
}
nb += sects[i]->blocks;
@@ -288,7 +318,10 @@
for(i = 0; i < n; i++){
for(j = i + 1; j < n; j++){
if(namecmp(sects[i]->name, sects[j]->name) == 0){
- seterr(EOk, "duplicate section name %s for index %s", sects[i]->name, name);
+ seterr(EOk, "%s and %s both have section name %s",
+ sects[i]->part->name,
+ sects[j]->part->name,
+ sects[i]->name);
return nil;
}
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/lump.c
--- a/sys/src/cmd/venti/srv/lump.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/lump.c Tue May 29 00:00:00 2012 +0200
@@ -2,6 +2,7 @@
#include "dat.h"
#include "fns.h"
+int bootstrap = 0;
int syncwrites = 0;
int queuewrites = 0;
int writestodevnull = 0;
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/mgr.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/src/cmd/venti/srv/mgr.c Tue May 29 00:00:00 2012 +0200
@@ -0,0 +1,1022 @@
+/*
+ * mirror manager.
+ * a work in progress.
+ * use at your own risk.
+ */
+
+#include "stdinc.h"
+#include <regexp.h>
+#include <bio.h>
+#include "dat.h"
+#include "fns.h"
+
+#ifdef PLAN9PORT
+#define sp s.sp
+#define ep e.ep
+#endif
+
+void sendmail(char *content, char *subject, char *msg);
+#define TIME "[0-9]+/[0-9]+ [0-9]+:[0-9]+:[0-9]+"
+
+char *mirrorregexp =
+ "^" TIME " ("
+ "([^ ]+ \\([0-9,]+-[0-9,]+\\))"
+ "|( copy [0-9,]+-[0-9,]+ (data|hole|directory|tail))"
+ "|( sha1 [0-9,]+-[0-9,]+)"
+ "|([^ ]+: [0-9,]+ used mirrored)"
+ "|([^ \\-]+-[^ \\-]+( mirrored| sealed| empty)+)"
+ ")$";
+Reprog *mirrorprog;
+
+char *verifyregexp =
+ "^" TIME " ("
+ "([^ ]+: unsealed [0-9,]+ bytes)"
+ ")$";
+Reprog *verifyprog;
+
+#undef pipe
+enum
+{
+ LogSize = 4*1024*1024 // TODO: make smaller
+};
+
+VtLog *errlog;
+
+typedef struct Mirror Mirror;
+struct Mirror
+{
+ char *src;
+ char *dst;
+};
+
+typedef struct Conf Conf;
+struct Conf
+{
+ Mirror *mirror;
+ int nmirror;
+ char **verify;
+ int nverify;
+ char *httpaddr;
+ char *webroot;
+ char *smtp;
+ char *mailfrom;
+ char *mailto;
+ int mirrorfreq;
+ int verifyfreq;
+};
+
+typedef struct Job Job;
+struct Job
+{
+ char *name;
+ QLock lk;
+ char *argv[10];
+ int oldok;
+ int newok;
+ VtLog *oldlog;
+ VtLog *newlog;
+ int pid;
+ int pipe;
+ int nrun;
+ vlong freq;
+ vlong runstart;
+ vlong runend;
+ double offset;
+ int (*ok)(char*);
+};
+
+Job *job;
+int njob;
+char *bin;
+
+vlong time0;
+Conf conf;
+
+void
+usage(void)
+{
+ fprint(2, "usage: mgr [-s] [-b bin/venti/] venti.conf\n");
+ threadexitsall(0);
+}
+
+int
+rdconf(char *file, Conf *conf)
+{
+ char *s, *line, *flds[10];
+ int i, ok;
+ IFile f;
+
+ if(readifile(&f, file) < 0)
+ return -1;
+ memset(conf, 0, sizeof *conf);
+ ok = -1;
+ line = nil;
+ for(;;){
+ s = ifileline(&f);
+ if(s == nil){
+ ok = 0;
+ break;
+ }
+ line = estrdup(s);
+ i = getfields(s, flds, nelem(flds), 1, " \t\r");
+ if(i <= 0 || strcmp(flds[0], "mgr") != 0) {
+ /* do nothing */
+ }else if(i == 4 && strcmp(flds[1], "mirror") == 0) {
+ if(conf->nmirror%64 == 0)
+ conf->mirror = vtrealloc(conf->mirror, (conf->nmirror+64)*sizeof(conf->mirror[0]));
+ conf->mirror[conf->nmirror].src = vtstrdup(flds[2]);
+ conf->mirror[conf->nmirror].dst = vtstrdup(flds[3]);
+ conf->nmirror++;
+ }else if(i == 3 && strcmp(flds[1], "mirrorfreq") == 0) {
+ conf->mirrorfreq = atoi(flds[2]);
+ }else if(i == 3 && strcmp(flds[1], "verify") == 0) {
+ if(conf->nverify%64 == 0)
+ conf->verify = vtrealloc(conf->verify, (conf->nverify+64)*sizeof(conf->verify[0]));
+ conf->verify[conf->nverify++] = vtstrdup(flds[2]);
+ }else if(i == 3 && strcmp(flds[1], "verifyfreq") == 0) {
+ conf->verifyfreq = atoi(flds[2]);
+ }else if(i == 3 && strcmp(flds[1], "httpaddr") == 0){
+ if(conf->httpaddr){
+ seterr(EAdmin, "duplicate httpaddr lines in configuration file %s", file);
+ break;
+ }
+ conf->httpaddr = estrdup(flds[2]);
+ }else if(i == 3 && strcmp(flds[1], "webroot") == 0){
+ if(conf->webroot){
+ seterr(EAdmin, "duplicate webroot lines in configuration file %s", file);
+ break;
+ }
+ conf->webroot = estrdup(flds[2]);
+ }else if(i == 3 && strcmp(flds[1], "smtp") == 0) {
+ if(conf->smtp){
+ seterr(EAdmin, "duplicate smtp lines in configuration file %s", file);
+ break;
+ }
+ conf->smtp = estrdup(flds[2]);
+ }else if(i == 3 && strcmp(flds[1], "mailfrom") == 0) {
+ if(conf->mailfrom){
+ seterr(EAdmin, "duplicate mailfrom lines in configuration file %s", file);
+ break;
+ }
+ conf->mailfrom = estrdup(flds[2]);
+ }else if(i == 3 && strcmp(flds[1], "mailto") == 0) {
+ if(conf->mailto){
+ seterr(EAdmin, "duplicate mailto lines in configuration file %s", file);
+ break;
+ }
+ conf->mailto = estrdup(flds[2]);
+ }else{
+ seterr(EAdmin, "illegal line '%s' in configuration file %s", line, file);
+ break;
+ }
+ free(line);
+ line = nil;
+ }
+ free(line);
+ freeifile(&f);
+ return ok;
+}
+
+static QLock loglk;
+static char *logbuf;
+
+char*
+logtext(VtLog *l)
+{
+ int i;
+ char *p;
+ VtLogChunk *c;
+
+ p = logbuf;
+ c = l->w;
+ for(i=0; i<l->nchunk; i++) {
+ if(++c == l->chunk+l->nchunk)
+ c = l->chunk;
+ memmove(p, c->p, c->wp - c->p);
+ p += c->wp - c->p;
+ }
+ *p = 0;
+ return logbuf;
+}
+
+
+typedef struct HttpObj HttpObj;
+
+static int fromwebdir(HConnect*);
+
+enum
+{
+ ObjNameSize = 64,
+ MaxObjs = 64
+};
+
+struct HttpObj
+{
+ char name[ObjNameSize];
+ int (*f)(HConnect*);
+};
+
+static HttpObj objs[MaxObjs];
+static void httpproc(void*);
+
+static HConnect*
+mkconnect(void)
+{
+ HConnect *c;
+
+ c = mallocz(sizeof(HConnect), 1);
+ if(c == nil)
+ sysfatal("out of memory");
+ c->replog = nil;
+ c->hpos = c->header;
+ c->hstop = c->header;
+ return c;
+}
+
+static int
+preq(HConnect *c)
+{
+ if(hparseheaders(c, 0) < 0)
+ return -1;
+ if(strcmp(c->req.meth, "GET") != 0
+ && strcmp(c->req.meth, "HEAD") != 0)
+ return hunallowed(c, "GET, HEAD");
+ if(c->head.expectother || c->head.expectcont)
+ return hfail(c, HExpectFail, nil);
+ return 0;
+}
+
+int
+hsettype(HConnect *c, char *type)
+{
+ Hio *hout;
+ int r;
+
+ r = preq(c);
+ if(r < 0)
+ return r;
+
+ hout = &c->hout;
+ if(c->req.vermaj){
+ hokheaders(c);
+ hprint(hout, "Content-type: %s\r\n", type);
+ if(http11(c))
+ hprint(hout, "Transfer-Encoding: chunked\r\n");
+ hprint(hout, "\r\n");
+ }
+
+ if(http11(c))
+ hxferenc(hout, 1);
+ else
+ c->head.closeit = 1;
+ return 0;
+}
+
+int
+hsethtml(HConnect *c)
+{
+ return hsettype(c, "text/html; charset=utf-8");
+}
+
+int
+hsettext(HConnect *c)
+{
+ return hsettype(c, "text/plain; charset=utf-8");
+}
+
+int
+hnotfound(HConnect *c)
+{
+ int r;
+
+ r = preq(c);
+ if(r < 0)
+ return r;
+ return hfail(c, HNotFound, c->req.uri);
+}
+
+static int
+xloglist(HConnect *c)
+{
+ if(hsettype(c, "text/html") < 0)
+ return -1;
+ vtloghlist(&c->hout);
+ hflush(&c->hout);
+ return 0;
+}
+
+static int
+strpcmp(const void *va, const void *vb)
+{
+ return strcmp(*(char**)va, *(char**)vb);
+}
+
+void
+vtloghlist(Hio *h)
+{
+ char **p;
+ int i, n;
+
+ hprint(h, "<html><head>\n");
+ hprint(h, "<title>Venti Server Logs</title>\n");
+ hprint(h, "</head><body>\n");
+ hprint(h, "<b>Venti Server Logs</b>\n<p>\n");
+
+ p = vtlognames(&n);
+ qsort(p, n, sizeof(p[0]), strpcmp);
+ for(i=0; i<n; i++)
+ hprint(h, "<a href=\"/log?log=%s\">%s</a><br>\n", p[i], p[i]);
+ vtfree(p);
+ hprint(h, "</body></html>\n");
+}
+
+void
+vtloghdump(Hio *h, VtLog *l)
+{
+ int i;
+ VtLogChunk *c;
+ char *name;
+
+ name = l ? l->name : "<nil>";
+
+ hprint(h, "<html><head>\n");
+ hprint(h, "<title>Venti Server Log: %s</title>\n", name);
+ hprint(h, "</head><body>\n");
+ hprint(h, "<b>Venti Server Log: %s</b>\n<p>\n", name);
+
+ if(l){
+ c = l->w;
+ for(i=0; i<l->nchunk; i++){
+ if(++c == l->chunk+l->nchunk)
+ c = l->chunk;
+ hwrite(h, c->p, c->wp-c->p);
+ }
+ }
+ hprint(h, "</body></html>\n");
+}
+
+
+char*
+hargstr(HConnect *c, char *name, char *def)
+{
+ HSPairs *p;
+
+ for(p=c->req.searchpairs; p; p=p->next)
+ if(strcmp(p->s, name) == 0)
+ return p->t;
+ return def;
+}
+
+static int
+xlog(HConnect *c)
+{
+ char *name;
+ VtLog *l;
+
+ name = hargstr(c, "log", "");
+ if(!name[0])
+ return xloglist(c);
+ l = vtlogopen(name, 0);
+ if(l == nil)
+ return hnotfound(c);
+ if(hsettype(c, "text/html") < 0){
+ vtlogclose(l);
+ return -1;
+ }
+ vtloghdump(&c->hout, l);
+ vtlogclose(l);
+ hflush(&c->hout);
+ return 0;
+}
+
+static void
+httpdproc(void *vaddress)
+{
+ HConnect *c;
+ char *address, ndir[NETPATHLEN], dir[NETPATHLEN];
+ int ctl, nctl, data;
+
+ address = vaddress;
+ ctl = announce(address, dir);
+ if(ctl < 0){
+ sysfatal("announce %s: %r", address);
+ return;
+ }
+
+ if(0) print("announce ctl %d dir %s\n", ctl, dir);
+ for(;;){
+ /*
+ * wait for a call (or an error)
+ */
+ nctl = listen(dir, ndir);
+ if(0) print("httpd listen %d %s...\n", nctl, ndir);
+ if(nctl < 0){
+ fprint(2, "mgr: httpd can't listen on %s: %r\n", address);
+ return;
+ }
+
+ data = accept(ctl, ndir);
+ if(0) print("httpd accept %d...\n", data);
+ if(data < 0){
+ fprint(2, "mgr: httpd accept: %r\n");
+ close(nctl);
+ continue;
+ }
+ if(0) print("httpd close nctl %d\n", nctl);
+ close(nctl);
+ c = mkconnect();
+ hinit(&c->hin, data, Hread);
+ hinit(&c->hout, data, Hwrite);
+ vtproc(httpproc, c);
+ }
+}
+
+static void
+httpproc(void *v)
+{
+ HConnect *c;
+ int ok, i, n;
+
+ c = v;
+
+ for(;;){
+ /*
+ * No timeout because the signal appears to hit every
+ * proc, not just us.
+ */
+ if(hparsereq(c, 0) < 0)
+ break;
+
+ for(i = 0; i < MaxObjs && objs[i].name[0]; i++){
+ n = strlen(objs[i].name);
+ if((objs[i].name[n-1] == '/' && strncmp(c->req.uri, objs[i].name, n) == 0)
+ || (objs[i].name[n-1] != '/' && strcmp(c->req.uri, objs[i].name) == 0)){
+ ok = (*objs[i].f)(c);
+ goto found;
+ }
+ }
+ ok = fromwebdir(c);
+ found:
+ hflush(&c->hout);
+ if(c->head.closeit)
+ ok = -1;
+ hreqcleanup(c);
+
+ if(ok < 0)
+ break;
+ }
+ hreqcleanup(c);
+ close(c->hin.fd);
+ free(c);
+}
+
+static int
+httpdobj(char *name, int (*f)(HConnect*))
+{
+ int i;
+
+ if(name == nil || strlen(name) >= ObjNameSize)
+ return -1;
+ for(i = 0; i < MaxObjs; i++){
+ if(objs[i].name[0] == '\0'){
+ strcpy(objs[i].name, name);
+ objs[i].f = f;
+ return 0;
+ }
+ if(strcmp(objs[i].name, name) == 0)
+ return -1;
+ }
+ return -1;
+}
+
+
+struct {
+ char *ext;
+ char *type;
+} exttab[] = {
+ ".html", "text/html",
+ ".txt", "text/plain",
+ ".xml", "text/xml",
+ ".png", "image/png",
+ ".gif", "image/gif",
+ 0
+};
+
+static int
+fromwebdir(HConnect *c)
+{
+ char buf[4096], *p, *ext, *type;
+ int i, fd, n, defaulted;
+ Dir *d;
+
+ if(conf.webroot == nil || strstr(c->req.uri, ".."))
+ return hnotfound(c);
+ snprint(buf, sizeof buf-20, "%s/%s", conf.webroot, c->req.uri+1);
+ defaulted = 0;
+reopen:
+ if((fd = open(buf, OREAD)) < 0)
+ return hnotfound(c);
+ d = dirfstat(fd);
+ if(d == nil){
+ close(fd);
+ return hnotfound(c);
+ }
+ if(d->mode&DMDIR){
+ if(!defaulted){
+ defaulted = 1;
+ strcat(buf, "/index.html");
+ free(d);
+ close(fd);
+ goto reopen;
+ }
+ free(d);
+ return hnotfound(c);
+ }
+ free(d);
+ p = buf+strlen(buf);
+ type = "application/octet-stream";
+ for(i=0; exttab[i].ext; i++){
+ ext = exttab[i].ext;
+ if(p-strlen(ext) >= buf && strcmp(p-strlen(ext), ext) == 0){
+ type = exttab[i].type;
+ break;
+ }
+ }
+ if(hsettype(c, type) < 0){
+ close(fd);
+ return 0;
+ }
+ while((n = read(fd, buf, sizeof buf)) > 0)
+ if(hwrite(&c->hout, buf, n) < 0)
+ break;
+ close(fd);
+ hflush(&c->hout);
+ return 0;
+}
+
+static int
+hmanager(HConnect *c)
+{
+ Hio *hout;
+ int r;
+ int i, k;
+ Job *j;
+ VtLog *l;
+ VtLogChunk *ch;
+
+ r = hsethtml(c);
+ if(r < 0)
+ return r;
+
+ hout = &c->hout;
+ hprint(hout, "<html><head><title>venti mgr status</title></head>\n");
+ hprint(hout, "<body><h2>venti mgr status</h2>\n");
+
+ for(i=0; i<njob; i++) {
+ j = &job[i];
+ hprint(hout, "<b>");
+ if(j->nrun == 0)
+ hprint(hout, "----/--/-- --:--:--");
+ else
+ hprint(hout, "%+T", (long)(j->runstart + time0));
+ hprint(hout, " %s", j->name);
+ if(j->nrun > 0) {
+ if(j->newok == -1) {
+ hprint(hout, " (running)");
+ } else if(!j->newok) {
+ hprint(hout, " <font color=\"#cc0000\">(FAILED)</font>");
+ }
+ }
+ hprint(hout, "</b>\n");
+ hprint(hout, "<font size=-1><pre>\n");
+ l = j->newlog;
+ ch = l->w;
+ for(k=0; k<l->nchunk; k++){
+ if(++ch == l->chunk+l->nchunk)
+ ch = l->chunk;
+ hwrite(hout, ch->p, ch->wp-ch->p);
+ }
+ hprint(hout, "</pre></font>\n");
+ hprint(hout, "\n");
+ }
+ hprint(hout, "</body></html>\n");
+ hflush(hout);
+ return 0;
+}
+
+void
+piper(void *v)
+{
+ Job *j;
+ char buf[512];
+ VtLog *l;
+ int n;
+ int fd;
+ char *p;
+ int ok;
+
+ j = v;
+ fd = j->pipe;
+ l = j->newlog;
+ while((n = read(fd, buf, 512-1)) > 0) {
+ buf[n] = 0;
+ if(l != nil)
+ vtlogprint(l, "%s", buf);
+ }
+ qlock(&loglk);
+ p = logtext(l);
+ ok = j->ok(p);
+ qunlock(&loglk);
+ j->newok = ok;
+ close(fd);
+}
+
+void
+kickjob(Job *j)
+{
+ int i;
+ int fd[3];
+ int p[2];
+ VtLog *l;
+
+ if((fd[0] = open("/dev/null", ORDWR)) < 0) {
+ vtlogprint(errlog, "%T open /dev/null: %r\n");
+ return;
+ }
+ if(pipe(p) < 0) {
+ vtlogprint(errlog, "%T pipe: %r\n");
+ close(fd[0]);
+ return;
+ }
+ qlock(&j->lk);
+ l = j->oldlog;
+ j->oldlog = j->newlog;
+ j->newlog = l;
+ qlock(&l->lk);
+ for(i=0; i<l->nchunk; i++)
+ l->chunk[i].wp = l->chunk[i].p;
+ qunlock(&l->lk);
+ j->oldok = j->newok;
+ j->newok = -1;
+ qunlock(&j->lk);
+
+ fd[1] = p[1];
+ fd[2] = p[1];
+ j->pid = threadspawn(fd, j->argv[0], j->argv);
+ if(j->pid < 0) {
+ vtlogprint(errlog, "%T exec %s: %r\n", j->argv[0]);
+ close(fd[0]);
+ close(fd[1]);
+ close(p[0]);
+ }
+ // fd[0], fd[1], fd[2] are closed now
+ j->pipe = p[0];
+ j->nrun++;
+ vtproc(piper, j);
+}
+
+int
+getline(Resub *text, Resub *line)
+{
+ char *p;
+
+ if(text->sp >= text->ep)
+ return -1;
+ line->sp = text->sp;
+ p = memchr(text->sp, '\n', text->ep - text->sp);
+ if(p == nil) {
+ line->ep = text->ep;
+ text->sp = text->ep;
+ } else {
+ line->ep = p;
+ text->sp = p+1;
+ }
+ return 0;
+}
+
+int
+verifyok(char *output)
+{
+ Resub text, line, m;
+
+ text.sp = output;
+ text.ep = output+strlen(output);
+ while(getline(&text, &line) >= 0) {
+ *line.ep = 0;
+ memset(&m, 0, sizeof m);
+ if(!regexec(verifyprog, line.sp, nil, 0))
+ return 0;
+ *line.ep = '\n';
+ }
+ return 1;
+}
+
+int
+mirrorok(char *output)
+{
+ Resub text, line, m;
+
+ text.sp = output;
+ text.ep = output+strlen(output);
+ while(getline(&text, &line) >= 0) {
+ *line.ep = 0;
+ memset(&m, 0, sizeof m);
+ if(!regexec(mirrorprog, line.sp, nil, 0))
+ return 0;
+ *line.ep = '\n';
+ }
+ return 1;
+}
+
+void
+mkjob(Job *j, ...)
+{
+ int i;
+ char *p;
+ va_list arg;
+
+ memset(j, 0, sizeof *j);
+ i = 0;
+ va_start(arg, j);
+ while((p = va_arg(arg, char*)) != nil) {
+ j->argv[i++] = p;
+ if(i >= nelem(j->argv))
+ sysfatal("job argv size too small");
+ }
+ j->argv[i] = nil;
+ j->oldlog = vtlogopen(smprint("log%ld.0", j-job), LogSize);
+ j->newlog = vtlogopen(smprint("log%ld.1", j-job), LogSize);
+ va_end(arg);
+}
+
+void
+manager(void *v)
+{
+ int i;
+ Job *j;
+ vlong now;
+
+ USED(v);
+ for(;; sleep(1000)) {
+ for(i=0; i<njob; i++) {
+ now = time(0) - time0;
+ j = &job[i];
+ if(j->pid > 0 || j->newok == -1) {
+ // still running
+ if(now - j->runstart > 2*j->freq) {
+ //TODO: log slow running j
+ }
+ continue;
+ }
+ if((j->nrun > 0 && now - j->runend > j->freq)
+ || (j->nrun == 0 && now > (vlong)(j->offset*j->freq))) {
+ j->runstart = now;
+ j->runend = 0;
+ kickjob(j);
+ }
+ }
+ }
+}
+
+void
+waitproc(void *v)
+{
+ Channel *c;
+ Waitmsg *w;
+ int i;
+ Job *j;
+
+ c = v;
+ for(;;) {
+ w = recvp(c);
+ for(i=0; i<njob; i++) {
+ j = &job[i];
+ if(j->pid == w->pid) {
+ j->pid = 0;
+ j->runend = time(0) - time0;
+ break;
+ }
+ }
+ free(w);
+ }
+}
+
+void
+threadmain(int argc, char **argv)
+{
+ int i;
+ int nofork;
+ char *prog;
+ Job *j;
+
+ ventilogging = 1;
+ ventifmtinstall();
+#ifdef PLAN9PORT
+ bin = unsharp("#9/bin/venti");
+#else
+ bin = "/bin/venti";
+#endif
+ nofork = 0;
+ ARGBEGIN{
+ case 'b':
+ bin = EARGF(usage());
+ break;
+ case 's':
+ nofork = 1;
+ break;
+ default:
+ usage();
+ }ARGEND
+
+ if(argc != 1)
+ usage();
+ if(rdconf(argv[0], &conf) < 0)
+ sysfatal("reading config: %r");
+ if(conf.httpaddr == nil)
+ sysfatal("config has no httpaddr");
+ if(conf.smtp != nil && conf.mailfrom == nil)
+ sysfatal("config has smtp but no mailfrom");
+ if(conf.smtp != nil && conf.mailto == nil)
+ sysfatal("config has smtp but no mailto");
+ if((mirrorprog = regcomp(mirrorregexp)) == nil)
+ sysfatal("mirrorregexp did not complete");
+ if((verifyprog = regcomp(verifyregexp)) == nil)
+ sysfatal("verifyregexp did not complete");
+ if(conf.nverify > 0 && conf.verifyfreq == 0)
+ sysfatal("config has no verifyfreq");
+ if(conf.nmirror > 0 && conf.mirrorfreq == 0)
+ sysfatal("config has no mirrorfreq");
+
+ time0 = time(0);
+// sendmail("startup", "mgr is starting\n");
+
+ logbuf = vtmalloc(LogSize+1); // +1 for NUL
+
+ errlog = vtlogopen("errors", LogSize);
+ job = vtmalloc((conf.nmirror+conf.nverify)*sizeof job[0]);
+ prog = smprint("%s/mirrorarenas", bin);
+ for(i=0; i<conf.nmirror; i++) {
+ // job: /bin/venti/mirrorarenas -v src dst
+ // filter output
+ j = &job[njob++];
+ mkjob(j, prog, "-v", conf.mirror[i].src, conf.mirror[i].dst, nil);
+ j->name = smprint("mirror %s %s", conf.mirror[i].src, conf.mirror[i].dst);
+ j->ok = mirrorok;
+ j->freq = conf.mirrorfreq; // 4 hours // TODO: put in config
+ j->offset = (double)i/conf.nmirror;
+ }
+
+ prog = smprint("%s/verifyarena", bin);
+ for(i=0; i<conf.nverify; i++) {
+ // job: /bin/venti/verifyarena -b 64M -s 1000 -v arena
+ // filter output
+ j = &job[njob++];
+ mkjob(j, prog, "-b64M", "-s1000", conf.verify[i], nil);
+ j->name = smprint("verify %s", conf.verify[i]);
+ j->ok = verifyok;
+ j->freq = conf.verifyfreq;
+ j->offset = (double)i/conf.nverify;
+ }
+
+ httpdobj("/mgr", hmanager);
+ httpdobj("/log", xlog);
+ vtproc(httpdproc, conf.httpaddr);
+ vtproc(waitproc, threadwaitchan());
+ if(nofork)
+ manager(nil);
+ else
+ vtproc(manager, nil);
+}
+
+
+void
+qp(Biobuf *b, char *p)
+{
+ int n, nspace;
+
+ nspace = 0;
+ n = 0;
+ for(; *p; p++) {
+ if(*p == '\n') {
+ if(nspace > 0) {
+ nspace = 0;
+ Bprint(b, "=\n");
+ }
+ Bputc(b, '\n');
+ n = 0;
+ continue;
+ }
+ if(n > 70) {
+ Bprint(b, "=\n");
+ nspace = 0;
+ continue;
+ }
+ if(33 <= *p && *p <= 126 && *p != '=') {
+ Bputc(b, *p);
+ n++;
+ nspace = 0;
+ continue;
+ }
+ if(*p == ' ' || *p == '\t') {
+ Bputc(b, *p);
+ n++;
+ nspace++;
+ continue;
+ }
+ Bprint(b, "=%02X", (uchar)*p);
+ n += 3;
+ nspace = 0;
+ }
+}
+
+int
+smtpread(Biobuf *b, int code)
+{
+ char *p, *q;
+ int n;
+
+ while((p = Brdstr(b, '\n', 1)) != nil) {
+ n = strtol(p, &q, 10);
+ if(n == 0 || q != p+3) {
+ error:
+ vtlogprint(errlog, "sending mail: %s\n", p);
+ free(p);
+ return -1;
+ }
+ if(*q == ' ') {
+ if(n == code) {
+ free(p);
+ return 0;
+ }
+ goto error;
+ }
+ if(*q != '-') {
+ goto error;
+ }
+ }
+ return -1;
+}
+
+
+void
+sendmail(char *content, char *subject, char *msg)
+{
+ int fd;
+ Biobuf *bin, *bout;
+
+ if((fd = dial(conf.smtp, 0, 0, 0)) < 0) {
+ vtlogprint(errlog, "dial %s: %r\n", conf.smtp);
+ return;
+ }
+ bin = vtmalloc(sizeof *bin);
+ bout = vtmalloc(sizeof *bout);
+ Binit(bin, fd, OREAD);
+ Binit(bout, fd, OWRITE);
+ if(smtpread(bin, 220) < 0){
+ error:
+ close(fd);
+ Bterm(bin);
+ Bterm(bout);
+ return;
+ }
+
+ Bprint(bout, "HELO venti-mgr\n");
+ Bflush(bout);
+ if(smtpread(bin, 250) < 0)
+ goto error;
+
+ Bprint(bout, "MAIL FROM:<%s>\n", conf.mailfrom);
+ Bflush(bout);
+ if(smtpread(bin, 250) < 0)
+ goto error;
+
+ Bprint(bout, "RCPT TO:<%s>\n", conf.mailfrom);
+ Bflush(bout);
+ if(smtpread(bin, 250) < 0)
+ goto error;
+
+ Bprint(bout, "DATA\n");
+ Bflush(bout);
+ if(smtpread(bin, 354) < 0)
+ goto error;
+
+ Bprint(bout, "From: \"venti mgr\" <%s>\n", conf.mailfrom);
+ Bprint(bout, "To: <%s>\n", conf.mailto);
+ Bprint(bout, "Subject: %s\n", subject);
+ Bprint(bout, "MIME-Version: 1.0\n");
+ Bprint(bout, "Content-Type: %s; charset=\"UTF-8\"\n", content);
+ Bprint(bout, "Content-Transfer-Encoding: quoted-printable\n");
+ Bprint(bout, "Message-ID: %08lux%08lux@venti.swtch.com\n", fastrand(), fastrand());
+ Bprint(bout, "\n");
+ qp(bout, msg);
+ Bprint(bout, ".\n");
+ Bflush(bout);
+ if(smtpread(bin, 250) < 0)
+ goto error;
+
+ Bprint(bout, "QUIT\n");
+ Bflush(bout);
+ Bterm(bin);
+ Bterm(bout);
+ close(fd);
+}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/mirror-log.awk
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/src/cmd/venti/srv/mirror-log.awk Tue May 29 00:00:00 2012 +0200
@@ -0,0 +1,105 @@
+# possible cron job:
+#
+# cd /cfg/backup
+# { for(i in 'E0 E1' 'E2 E3' 'E4 E5' 'E6 E7' 'F0 F1' 'F2 F3' 'F4 F5'){
+# x=`{echo $i}
+# venti/mirrorarenas -v /dev/sd$x(1)^/arenas /dev/sd$x(2)^/arenas
+# } } >www/mirror1.txt >[2=1]
+# mv www/mirror1.txt www/mirror.txt
+# awk -f mirror-log.awk www/mirror.txt >www/mirror.html
+
+BEGIN {
+ print "<html><body><h1>mirror status</h1>"
+ print "details in <a href=mirror.txt>mirror.txt</a><br><br>"
+ print "<hr><table cellpadding=5 cellspacing=0 border=0>"
+ laststatus = ""
+ firstarena = ""
+ lastarena = ""
+ status = ""
+ arena = ""
+
+}
+
+function fmt( color) {
+ nfmt++
+ if(nfmt%2 == 0)
+ color = "#cccccc"
+ else
+ color = "#ffffff"
+ return "<tr bgcolor=" color "><td valign=top>%s</td><td valign=top>%s</td><td>%s</td><td>%s</td><td>"
+}
+
+
+function finish() {
+ if(!arena && !status)
+ return
+ if(info == "" && laststatus == status){
+ lastarena = arena
+ return
+ }
+ if(firstarena != ""){
+ if(firstarena == lastarena)
+ printf(fmt(), time, firstarena, "", "");
+ else
+ printf(fmt(), time, firstarena, "-", lastarena);
+ print laststatus "</td></tr>"
+ firstarena = ""
+ lastarena = ""
+ laststatus = ""
+ }
+ if(info == ""){
+ firstarena = arena
+ laststatus = status
+ lastarena = arena
+ return
+ }
+ printf(fmt(), time, arena, "", "");
+ print status
+ if(info != ""){
+ print "<pre>"
+ printf("%s", info)
+ print "</pre>"
+ }
+ print "</td>"
+}
+
+$3 !~ /:$/ && $4 ~ /^\(.*-.*\)$/ {
+ finish();
+ arena = $3
+ range = $4
+ status = ""
+ info = ""
+ size = 0
+ time = $1 " " $2
+ next
+}
+
+$3 ~ /:$/ && $0 ~ /^....\/.... ..:..:.. [^ ]/ {
+ if($4 == "0" && $5 == "used" && $6 == "mirrored"){
+ status = "empty"
+ next
+ }
+ if($4 ~ /^[0-9,]+$/ && $5 == "used" && $6 == "mirrored"){
+ size = $4
+ status = "partial " size ", mirrored"
+ next
+ }
+ if($4 ~ /^[0-9a-f]+$/ && length($4) == 40 && $5 == "sealed" && $6 == "mirrored"){
+ status = "sealed, mirrored";
+ next
+ }
+}
+
+{
+ info = info $0 "\n"
+}
+
+END{
+ finish();
+ status = "done"
+ arena = ""
+ info = ""
+ finish();
+ print "</table><hr>"
+ print "</body></html>"
+}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/mirrorarenas.c
--- a/sys/src/cmd/venti/srv/mirrorarenas.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/mirrorarenas.c Tue May 29 00:00:00 2012 +0200
@@ -34,9 +34,11 @@
}
char *tagged;
+char *tagname;
+int tagindx;
void
-tag(char *fmt, ...)
+tag(int indx, char *name, char *fmt, ...)
{
va_list arg;
@@ -44,16 +46,72 @@
free(tagged);
tagged = nil;
}
+ tagindx = indx;
+ tagname = name;
va_start(arg, fmt);
tagged = vsmprint(fmt, arg);
va_end(arg);
}
+enum
+{
+ Sealed = 1,
+ Mirrored = 2,
+ Empty = 4,
+};
+
+void
+setstatus(int bits)
+{
+ static int startindx = -1, endindx;
+ static char *startname, *endname;
+ static int lastbits;
+ char buf[100];
+
+ if(bits != lastbits) {
+ if(startindx >= 0) {
+ switch(lastbits) {
+ case Sealed:
+ snprint(buf, sizeof buf, "sealed");
+ break;
+ case Mirrored:
+ snprint(buf, sizeof buf, "mirrored");
+ break;
+ case Sealed+Mirrored:
+ snprint(buf, sizeof buf, "mirrored sealed");
+ break;
+ case Empty:
+ snprint(buf, sizeof buf, "empty");
+ break;
+ default:
+ snprint(buf, sizeof buf, "%d", bits);
+ break;
+ }
+ print("%T %s-%s %s\n", startname, endname, buf);
+ }
+ lastbits = bits;
+ startindx = tagindx;
+ endindx = tagindx;
+ startname = tagname;
+ endname = tagname;
+ } else {
+ endindx = tagindx;
+ endname = tagname;
+ }
+ if(bits < 0) {
+ startindx = -1;
+ endindx = -1;
+ return;
+ }
+}
+
void
chat(char *fmt, ...)
{
va_list arg;
+ setstatus(-1);
+
if(tagged){
write(1, tagged, strlen(tagged));
free(tagged);
@@ -64,7 +122,7 @@
va_end(arg);
}
-#pragma varargck argpos tag 1
+#pragma varargck argpos tag 3
#pragma varargck argpos chat 1
@@ -113,13 +171,25 @@
{
int i, n;
uvlong o;
- static uchar tmp[2][1024*1024];
+ enum {
+ Chunk = 1024*1024
+ };
+ static uchar tmpbuf[2*Chunk+MaxIo];
+ static uchar *tmp[2];
+ uchar *p;
Write w[2];
assert(start <= end);
assert(astart <= start && start < aend);
assert(astart <= end && end <= aend);
+ // align the buffers so readpart/writepart can do big transfers
+ p = tmpbuf;
+ if((uintptr)p%MaxIo)
+ p += MaxIo - (uintptr)p%MaxIo;
+ tmp[0] = p;
+ tmp[1] = p + Chunk;
+
if(verbose && start != end)
chat("%T copy %,llud-%,llud %s\n", start, end, what);
@@ -128,7 +198,7 @@
for(o=start; o<end; o+=n){
if(w[i].error)
goto error;
- n = sizeof tmp[i];
+ n = Chunk;
if(o+n > end)
n = end - o;
if(ereadpart(src, o, tmp[i], n) < 0)
@@ -235,7 +305,7 @@
}
void
-mirror(Arena *sa, Arena *da)
+mirror(int indx, Arena *sa, Arena *da)
{
vlong v, si, di, end;
int clumpmax, blocksize, sealed;
@@ -251,7 +321,7 @@
astart = base - blocksize;
aend = end + blocksize;
- tag("%T %s (%,llud-%,llud)\n", sa->name, astart, aend);
+ tag(indx, sa->name, "%T %s (%,llud-%,llud)\n", sa->name, astart, aend);
if(force){
copy(astart, aend, "all", nil);
@@ -260,7 +330,8 @@
if(sa->diskstats.sealed && da->diskstats.sealed && scorecmp(da->score, zeroscore) != 0){
if(scorecmp(sa->score, da->score) == 0){
- if(verbose)
+ setstatus(Sealed+Mirrored);
+ if(verbose > 1)
chat("%T %s: %V sealed mirrored\n", sa->name, sa->score);
return;
}
@@ -378,7 +449,8 @@
memset(buf, 0, VtScoreSize);
sha1(buf, VtScoreSize, da->score, ds);
if(scorecmp(sa->score, da->score) == 0){
- if(verbose)
+ setstatus(Sealed+Mirrored);
+ if(verbose > 1)
chat("%T %s: %V sealed mirrored\n", sa->name, sa->score);
if(ewritepart(dst, end+blocksize-VtScoreSize, da->score, VtScoreSize) < 0)
return;
@@ -391,14 +463,21 @@
status = "errors";
}
}else{
- if(verbose)
+ setstatus(Mirrored);
+ if(verbose > 1)
chat("%T %s: %V mirrored\n", sa->name, sa->score);
if(ewritepart(dst, end+blocksize-VtScoreSize, sa->score, VtScoreSize) < 0)
return;
}
}else{
- chat("%T %s: %,lld used mirrored\n",
- sa->name, sa->diskstats.used);
+ if(sa->diskstats.used > 0 || verbose > 1) {
+ chat("%T %s: %,lld used mirrored\n",
+ sa->name, sa->diskstats.used);
+ }
+ if(sa->diskstats.used > 0)
+ setstatus(Mirrored);
+ else
+ setstatus(Empty);
}
}
@@ -413,8 +492,9 @@
for(i=0; i<sp->narenas; i++){
sa = sp->arenas[i];
da = dp->arenas[i];
- mirror(sa, da);
+ mirror(i, sa, da);
}
+ setstatus(-1);
return;
}
if(strcmp(range, "none") == 0)
@@ -445,8 +525,9 @@
for(i=lo; i<=hi; i++){
sa = sp->arenas[i];
da = dp->arenas[i];
- mirror(sa, da);
+ mirror(i, sa, da);
}
+ setstatus(-1);
}
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/printarenapart.c
--- a/sys/src/cmd/venti/srv/printarenapart.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/printarenapart.c Tue May 29 00:00:00 2012 +0200
@@ -11,7 +11,8 @@
threadexitsall("usage");
}
-static void
+/* unused */
+void
rdarena(Arena *arena, u64int offset)
{
u64int a, aa, e;
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/rdarena.c
--- a/sys/src/cmd/venti/srv/rdarena.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/rdarena.c Tue May 29 00:00:00 2012 +0200
@@ -7,7 +7,7 @@
void
usage(void)
{
- fprint(2, "usage: rdarena [-v] arenapart arena\n");
+ fprint(2, "usage: rdarena [-qv] arenapart arena\n");
threadexitsall(0);
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/venti.c
--- a/sys/src/cmd/venti/srv/venti.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/venti.c Tue May 29 00:00:00 2012 +0200
@@ -149,9 +149,8 @@
void
usage(void)
{
- fprint(2, "usage: venti [-Ldrsw] [-a ventiaddr] [-c config] "
-"[-h httpaddr] [-m %%mem] [-B blockcachesize] [-C cachesize] [-I icachesize] "
-"[-W webroot]\n");
+ fprint(2, "usage: venti [-Ldrs] [-a address] [-B blockcachesize] [-c config] "
+"[-C lumpcachesize] [-h httpaddress] [-I indexcachesize] [-m %%mem] [-W webroot]\n");
threadexitsall("usage");
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/verifyarena.c
--- a/sys/src/cmd/venti/srv/verifyarena.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/verifyarena.c Tue May 29 00:00:00 2012 +0200
@@ -60,7 +60,8 @@
u32int bs;
u8int score[VtScoreSize];
- fprint(2, "%T verify %s\n", name);
+ if(verbose)
+ fprint(2, "%T verify %s\n", name);
memset(&arena, 0, sizeof arena);
memset(&s, 0, sizeof s);
@@ -140,16 +141,19 @@
/*
* check for no checksum or the same
*/
- if(scorecmp(score, arena.score) == 0)
- fprint(2, "%T %s: verified score\n", name);
- else if(scorecmp(zeroscore, arena.score) == 0)
- fprint(2, "%T %s: unsealed\n", name);
- else{
+ if(scorecmp(score, arena.score) == 0) {
+ if(verbose)
+ fprint(2, "%T %s: verified score\n", name);
+ } else if(scorecmp(zeroscore, arena.score) == 0) {
+ if(verbose || arena.diskstats.used > 0)
+ fprint(2, "%T %s: unsealed %,lld bytes\n", name, arena.diskstats.used);
+ } else{
fprint(2, "%T %s: mismatch checksum - found=%V calculated=%V\n",
name, arena.score, score);
return;
}
- printarena(2, &arena);
+ if(verbose > 1)
+ printarena(2, &arena);
}
static int
@@ -196,7 +200,10 @@
break;
}ARGEND
- data = vtmalloc(blocksize);
+ data = vtmalloc(MaxIo + blocksize);
+ if((uintptr)data % MaxIo)
+ data += MaxIo - (uintptr)data%MaxIo;
+
if(argc == 0){
fd = 0;
verifyarena("<stdin>", 0);
@@ -212,8 +219,9 @@
sysfatal("read arena part header: %r");
if(unpackarenapart(&ap, data) < 0)
sysfatal("corrupted arena part header: %r");
- fprint(2, "%T # arena part version=%d blocksize=%d arenabase=%d\n",
- ap.version, ap.blocksize, ap.arenabase);
+ if(verbose)
+ fprint(2, "%T # arena part version=%d blocksize=%d arenabase=%d\n",
+ ap.version, ap.blocksize, ap.arenabase);
ap.tabbase = (PartBlank+HeadSize+ap.blocksize-1)&~(ap.blocksize-1);
ap.tabsize = ap.arenabase - ap.tabbase;
table = malloc(ap.tabsize+1);
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/wrarena.c
--- a/sys/src/cmd/venti/srv/wrarena.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/wrarena.c Tue May 29 00:00:00 2012 +0200
@@ -24,7 +24,7 @@
void
usage(void)
{
- fprint(2, "usage: wrarena [-h host] arenafile [offset]\n");
+ fprint(2, "usage: wrarena [-o fileoffset] [-h host] arenafile [clumpoffset]\n");
threadexitsall("usage");
}
@@ -131,13 +131,13 @@
int i;
char *file;
Arena *arena;
+ ArenaPart *ap;
u64int offset, aoffset;
Part *part;
uchar buf[8192];
ArenaHead head;
ZClump zerocl;
- ventifmtinstall();
qlock(&godot);
aoffset = 0;
ARGBEGIN{
@@ -174,13 +174,21 @@
file = argv[0];
}
- fmtinstall('V', vtscorefmt);
+ ventifmtinstall();
statsinit();
part = initpart(file, OREAD);
if(part == nil)
sysfatal("can't open file %s: %r", file);
+ initdcache(8 * MaxDiskBlock);
+
+ // Try as arena partition.
+ arena = nil;
+ ap = initarenapart(part);
+ if(ap == nil)
+ goto loaded;
+
if(readpart(part, aoffset, buf, sizeof buf) < 0)
sysfatal("can't read file %s: %r", file);
@@ -192,12 +200,12 @@
head.size, part->size);
partblocksize(part, head.blocksize);
- initdcache(8 * MaxDiskBlock);
arena = initarena(part, aoffset, head.size, head.blocksize);
if(arena == nil)
sysfatal("initarena: %r");
+loaded:
z = nil;
if(host==nil || strcmp(host, "/dev/null") != 0){
z = vtdial(host);
@@ -207,19 +215,26 @@
sysfatal("vtconnect: %r");
}
+ print("%T starting to send data\n");
c = chancreate(sizeof(ZClump), 0);
for(i=0; i<12; i++)
vtproc(vtsendthread, nil);
- rdarena(arena, offset);
- if(vtsync(z) < 0)
- sysfatal("executing sync: %r");
+ if(ap != nil) {
+ for(i=0; i<ap->narenas; i++)
+ rdarena(ap->arenas[i], 0);
+ } else
+ rdarena(arena, offset);
memset(&zerocl, 0, sizeof zerocl);
for(i=0; i<12; i++)
send(c, &zerocl);
+ if(vtsync(z) < 0)
+ sysfatal("executing sync: %r");
if(z){
vthangup(z);
}
+ print("%T sent all data\n");
+
threadexitsall(0);
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/www/debug.html
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/src/cmd/venti/srv/www/debug.html Tue May 29 00:00:00 2012 +0200
@@ -0,0 +1,41 @@
+<h1>venti debugging</h1>
+
+<a href="/debug?op=amap">amap</a><br>
+<a href="/debug?op=mem">mem</a><br>
+<a href="/disk">disk</a><br>
+<form method=get action="/disk">
+arena part:
+<input type=hidden name=type value=a></input>
+<input type=text name=disk size=50></input>
+<input type=submit value="Go">
+</form><br>
+<form method=get action="/disk">
+bloom filter:
+<input type=hidden name=type value=b></input>
+<input type=text name=disk size=50></input>
+<input type=submit value="Go">
+</form><br>
+<form method=get action="/disk">
+index section:
+<input type=hidden name=type value=i></input>
+<input type=text name=disk size=50></input>
+<input type=submit value="Go">
+</form><br>
+<br>
+<form method=get action="/debug">
+<input type=hidden name=op value=read></input>
+read score:
+<input type=text name=score size=50></input>
+<input type=submit value="Go">
+</form><br>
+<br>
+<form method=get action="/disk">
+read score:
+<input type=hidden name=type value=a></input>
+<input type=text name=score size=50></input>
+in arena
+<input type=text name=arena size=20></input>
+in arena partition
+<input type=text name=disk size=20></input>
+<input type=submit value="Go">
+</form><br>
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/www/stats.html
--- a/sys/src/cmd/venti/srv/www/stats.html Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/www/stats.html Tue May 29 00:00:00 2012 +0200
@@ -6,6 +6,7 @@
<script language="javascript" src="status.js"></script>
</head>
<body bgcolor=#ffffff>
+ <iframe name="hidden" frameborder="0" height="0"></iframe>
<center>
<b>venti.your-domain.com – venti server statistics</b>
diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/www/stats.js
--- a/sys/src/cmd/venti/srv/www/stats.js Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/srv/www/stats.js Tue May 29 00:00:00 2012 +0200
@@ -15,11 +15,11 @@
"arena write bytes/second",
"arg=bloomfalsemiss&graph=pctdiff&arg2=bloomlookup&max=100",
- "bloom false hit %",
+ "bloom false miss %",
"arg=bloomhit&graph=pctdiff&arg2=bloomlookup&max=100",
+ "bloom hit %",
+ "arg=bloommiss&graph=pctdiff&arg2=bloomlookup&max=100",
"bloom miss %",
- "arg=bloomlookuptime&graph=divdiff&arg2=bloomlookup",
- "bloom lookup time",
"arg=bloomones&graph=pct&arg2=bloombits&max=100",
"bloom usage %",
@@ -38,10 +38,16 @@
"icache dirty %",
"arg=icachehit&graph=pctdiff&arg2=icachelookup&max=100",
"icache hit %",
+ "arg=scachehit&graph=pctdiff&arg2=icachelookup&max=100",
+ "scache hit %",
+ "arg=icachemiss&graph=pctdiff&arg2=icachelookup&max=100",
+ "icache miss %",
"arg=icachelookuptime&graph=divdiff&arg2=icachelookup",
"icache lookup time",
"arg=icacheprefetch&graph=diff",
"icache prefetches/second",
+ "arg=scacheprefetch&graph=diff",
+ "scache prefetches/second",
"arg=icachewrite&graph=diff",
"icache writes/second",
@@ -75,6 +81,8 @@
"fresh write RPC time",
"arg=rpcwriteoldtime&graph=divdiff&arg2=rpcwriteold",
"dup write RPC time",
+ "arg=cigloadtime&graph=divdiff&arg2=cigload",
+ "cig load time",
"arg=sumreadbyte&graph=diff",
"checksum bytes/second",
@@ -107,6 +115,7 @@
"!bloom filter",
"arg=bloomhit&graph=pctdiff&arg2=bloomlookup&max=100",
+ "arg=bloommiss&graph=pctdiff&arg2=bloomlookup&max=100",
"arg=bloomfalsemiss&graph=pctdiff&arg2=bloomlookup&max=100",
"arg=bloomones&graph=pct&arg2=bloombits&max=100",
@@ -118,8 +127,11 @@
"!icache",
"arg=icachedirty&graph=pct&arg2=icachesize&max=100",
"arg=icachehit&graph=pctdiff&arg2=icachelookup&max=100",
+ "arg=scachehit&graph=pctdiff&arg2=icachelookup&max=100",
+ "arg=icachemiss&graph=pctdiff&arg2=icachelookup&max=100",
"arg=icachewrite&graph=diff",
"arg=icacheprefetch&graph=diff",
+ "arg=scacheprefetch&graph=diff",
"!dcache",
"arg=dcachedirty&graph=pct&arg2=dcachesize&max=100",
@@ -144,7 +156,6 @@
"arg=lumpstall",
"!timings",
- "arg=bloomlookuptime&graph=divdiff&arg2=bloomlookup",
"arg=icachelookuptime&graph=divdiff&arg2=icachelookup",
"arg=lcachelookuptime&graph=divdiff&arg2=lcachelookup",
"arg=dcachelookuptime&graph=divdiff&arg2=dcachelookup",
@@ -154,6 +165,7 @@
"arg=rpcreaduncachedtime&graph=divdiff&arg2=rpcreaduncached",
"arg=rpcwritenewtime&graph=divdiff&arg2=rpcwritenew",
"arg=rpcwriteoldtime&graph=divdiff&arg2=rpcwriteold",
+ "arg=cigloadtime&graph=divdiff&arg2=cigload",
"END"
)
@@ -353,7 +365,7 @@
function loglinks(list) {
var s = ""
for(var i=0; i<list.length; i++){
- s = s+" <a href=\"/log/"+list[i]+"\">"+list[i]+"</a>"
+ s = s+" <a href=\"/log?log="+list[i]+"\">"+list[i]+"</a>"
}
return s
}
@@ -383,5 +395,5 @@
eval(name+"= \""+value+"\"")
redrawsettings()
// Works in FireFox, not in Safari
- parent.hidden.location.href = "/set/"+name+"/"+value
+ parent.hidden.location.href = "/set?name="+name+"&value="+value
}
diff -r 88ea8de5bdf7 sys/src/cmd/venti/write.c
--- a/sys/src/cmd/venti/write.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/cmd/venti/write.c Tue May 29 00:00:00 2012 +0200
@@ -4,6 +4,12 @@
#include <libsec.h>
#include <thread.h>
+enum
+{
+ // XXX What to do here?
+ VtMaxLumpSize = 65535,
+};
+
void
usage(void)
{
diff -r 88ea8de5bdf7 sys/src/cmd/venti/writefile.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/src/cmd/venti/writefile.c Tue May 29 00:00:00 2012 +0200
@@ -0,0 +1,106 @@
+#include <u.h>
+#include <libc.h>
+#include <venti.h>
+#include <libsec.h>
+#include <thread.h>
+
+enum
+{
+ Blocksize = 8192
+};
+
+int chatty;
+
+void
+usage(void)
+{
+ fprint(2, "usage: writefile [-v] [-h host] < data\n");
+ threadexitsall("usage");
+}
+
+void
+threadmain(int argc, char *argv[])
+{
+ int n;
+ uchar score[VtScoreSize];
+ uchar *buf;
+ char *host;
+ vlong off;
+ VtEntry e;
+ VtRoot root;
+ VtCache *c;
+ VtConn *z;
+ VtFile *f;
+
+ quotefmtinstall();
+ fmtinstall('F', vtfcallfmt);
+ fmtinstall('V', vtscorefmt);
+
+ host = nil;
+ ARGBEGIN{
+ case 'V':
+ chattyventi++;
+ break;
+ case 'h':
+ host = EARGF(usage());
+ break;
+ case 'v':
+ chatty++;
+ break;
+ default:
+ usage();
+ break;
+ }ARGEND
+
+ if(argc != 0)
+ usage();
+
+ buf = vtmallocz(Blocksize);
+
+ z = vtdial(host);
+ if(z == nil)
+ sysfatal("could not connect to server: %r");
+
+ if(vtconnect(z) < 0)
+ sysfatal("vtconnect: %r");
+
+ // write file
+ c = vtcachealloc(z, Blocksize*32);
+ if(c == nil)
+ sysfatal("vtcachealloc: %r");
+ f = vtfilecreateroot(c, Blocksize, Blocksize, VtDataType);
+ if(f == nil)
+ sysfatal("vtfilecreateroot: %r");
+ off = 0;
+ vtfilelock(f, VtOWRITE);
+ while((n = read(0, buf, Blocksize)) > 0){
+ if(vtfilewrite(f, buf, n, off) != n)
+ sysfatal("vtfilewrite: %r");
+ off += n;
+ if(vtfileflushbefore(f, off) < 0)
+ sysfatal("vtfileflushbefore: %r");
+ }
+ if(vtfileflush(f) < 0)
+ sysfatal("vtfileflush: %r");
+ if(vtfilegetentry(f, &e) < 0)
+ sysfatal("vtfilegetentry: %r");
+ vtfileunlock(f);
+
+ // write directory entry
+ memset(&root, 0, sizeof root);
+ vtentrypack(&e, buf, 0);
+ if(vtwrite(z, root.score, VtDirType, buf, VtEntrySize) < 0)
+ sysfatal("vtwrite dir: %r");
+
+ // write root
+ strcpy(root.name, "data");
+ strcpy(root.type, "file");
+ root.blocksize = Blocksize;
+ vtrootpack(&root, buf);
+ if(vtwrite(z, score, VtRootType, buf, VtRootSize) < 0)
+ sysfatal("vtwrite root: %r");
+
+ print("file:%V\n", score);
+ threadexitsall(0);
+}
+
diff -r 88ea8de5bdf7 sys/src/libventi/cache.c
--- a/sys/src/libventi/cache.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/cache.c Tue May 29 00:00:00 2012 +0200
@@ -32,7 +32,6 @@
{
QLock lk;
VtConn *z;
- u32int blocksize;
u32int now; /* ticks for usage time stamps */
VtBlock **hash; /* hash table for finding addresses */
int nhash;
@@ -40,41 +39,45 @@
int nheap;
VtBlock *block; /* all allocated blocks */
int nblock;
- uchar *mem; /* memory for all blocks and data */
int (*write)(VtConn*, uchar[VtScoreSize], uint, uchar*, int);
+ VtBlock *dead; /* blocks we don't have memory for */
+ ulong mem;
+ ulong maxmem;
};
static void cachecheck(VtCache*);
VtCache*
-vtcachealloc(VtConn *z, int blocksize, ulong nblock)
+vtcachealloc(VtConn *z, ulong maxmem)
{
- uchar *p;
VtCache *c;
int i;
+ int nblock;
VtBlock *b;
+ ulong maxmem0;
+ maxmem0 = maxmem;
c = vtmallocz(sizeof(VtCache));
-
+ nblock = maxmem/100/(sizeof(VtBlock)+2*sizeof(VtBlock*));
c->z = z;
- c->blocksize = (blocksize + 127) & ~127;
c->nblock = nblock;
c->nhash = nblock;
c->hash = vtmallocz(nblock*sizeof(VtBlock*));
c->heap = vtmallocz(nblock*sizeof(VtBlock*));
c->block = vtmallocz(nblock*sizeof(VtBlock));
- c->mem = vtmallocz(nblock*c->blocksize);
c->write = vtwrite;
+ maxmem -= nblock*(sizeof(VtBlock) + 2*sizeof(VtBlock*));
+ maxmem -= sizeof(VtCache);
+ if((long)maxmem < 0)
+ sysfatal("cache size far too small: %lud", maxmem0);
+ c->mem = maxmem;
- p = c->mem;
for(i=0; i<nblock; i++){
b = &c->block[i];
b->addr = NilBlock;
b->c = c;
- b->data = p;
b->heap = i;
c->heap[i] = b;
- p += c->blocksize;
}
c->nheap = nblock;
cachecheck(c);
@@ -102,13 +105,14 @@
qlock(&c->lk);
cachecheck(c);
- for(i=0; i<c->nblock; i++)
- assert(c->block[i].ref == 0);
+ for(i=0; i<c->nblock; i++) {
+ assert(c->block[i].data == nil || c->block[i].ref == 0);
+ vtfree(c->block[i].data);
+ }
vtfree(c->hash);
vtfree(c->heap);
vtfree(c->block);
- vtfree(c->mem);
vtfree(c);
}
@@ -128,11 +132,10 @@
static void
cachecheck(VtCache *c)
{
- u32int size, now;
+ u32int now;
int i, k, refed;
VtBlock *b;
- size = c->blocksize;
now = c->now;
for(i = 0; i < c->nheap; i++){
@@ -151,8 +154,6 @@
refed = 0;
for(i = 0; i < c->nblock; i++){
b = &c->block[i];
- if(b->data != &c->mem[i * size])
- sysfatal("mis-blocked at %d", i);
if(b->ref && b->heap == BadHeap)
refed++;
else if(b->addr != NilBlock)
@@ -299,6 +300,57 @@
}
/*
+ * evict blocks until there is enough memory for size bytes.
+ */
+static VtBlock*
+vtcacheevict(VtCache *c, ulong size)
+{
+ VtBlock *b;
+
+ /*
+ * If we were out of memory and put some blocks
+ * to the side but now we have memory, grab one.
+ */
+ if(c->mem >= size && c->dead) {
+ b = c->dead;
+ c->dead = b->next;
+ b->next = nil;
+ goto alloc;
+ }
+
+ /*
+ * Otherwise, evict until we have memory.
+ */
+ for(;;) {
+ b = vtcachebumpblock(c);
+ if(c->mem+b->size >= size)
+ break;
+ /*
+ * chain b onto dead list
+ */
+ free(b->data);
+ b->data = nil;
+ c->mem += b->size;
+ b->size = 0;
+ b->next = c->dead;
+ c->dead = b;
+ }
+
+ /*
+ * Allocate memory for block.
+ */
+alloc:
+ if(size > b->size || size <= b->size/2) {
+ free(b->data);
+ c->mem += b->size;
+ c->mem -= size;
+ b->size = size;
+ b->data = vtmalloc(size);
+ }
+ return b;
+}
+
+/*
* fetch a local block from the memory cache.
* if it's not there, load it, bumping some other Block.
* if we're out of free blocks, we're screwed.
@@ -312,7 +364,7 @@
sysfatal("vtcachelocal: asked for nonexistent block 0");
if(addr > c->nblock)
sysfatal("vtcachelocal: asked for block #%ud; only %d blocks",
- addr, c->nblock);
+ (uint)addr, c->nblock);
b = &c->block[addr-1];
if(b->addr == NilBlock || b->iostate != BioLocal)
@@ -332,16 +384,16 @@
}
VtBlock*
-vtcacheallocblock(VtCache *c, int type)
+vtcacheallocblock(VtCache *c, int type, ulong size)
{
VtBlock *b;
qlock(&c->lk);
- b = vtcachebumpblock(c);
+ b = vtcacheevict(c, size);
b->iostate = BioLocal;
b->type = type;
b->addr = (b - c->block)+1;
- vtzeroextend(type, b->data, 0, c->blocksize);
+ vtzeroextend(type, b->data, 0, size);
vtlocaltoglobal(b->addr, b->score);
qunlock(&c->lk);
@@ -356,7 +408,7 @@
* if it's not there, load it, bumping some other block.
*/
VtBlock*
-vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type)
+vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type, ulong size)
{
VtBlock *b;
ulong h;
@@ -409,7 +461,7 @@
/*
* not found
*/
- b = vtcachebumpblock(c);
+ b = vtcacheevict(c, size);
b->addr = NilBlock;
b->type = type;
memmove(b->score, score, VtScoreSize);
@@ -435,7 +487,7 @@
qunlock(&c->lk);
vtcachenread++;
- n = vtread(c->z, score, type, b->data, c->blocksize);
+ n = vtread(c->z, score, type, b->data, size);
if(n < 0){
if(chattyventi)
fprint(2, "read %V: %r\n", score);
@@ -445,7 +497,7 @@
vtblockput(b);
return nil;
}
- vtzeroextend(type, b->data, n, c->blocksize);
+ vtzeroextend(type, b->data, n, size);
b->iostate = BioVenti;
b->nlock = 1;
if(vttracelevel)
@@ -534,7 +586,7 @@
}
c = b->c;
- n = vtzerotruncate(b->type, b->data, c->blocksize);
+ n = vtzerotruncate(b->type, b->data, b->size);
vtcachenwrite++;
if(c->write(c->z, score, b->type, b->data, n) < 0)
return -1;
@@ -554,24 +606,18 @@
return 0;
}
-uint
-vtcacheblocksize(VtCache *c)
-{
- return c->blocksize;
-}
-
VtBlock*
vtblockcopy(VtBlock *b)
{
VtBlock *bb;
vtcachencopy++;
- bb = vtcacheallocblock(b->c, b->type);
+ bb = vtcacheallocblock(b->c, b->type, b->size);
if(bb == nil){
vtblockput(b);
return nil;
}
- memmove(bb->data, b->data, b->c->blocksize);
+ memmove(bb->data, b->data, b->size);
vtblockput(b);
bb->pc = getcallerpc(&b);
return bb;
diff -r 88ea8de5bdf7 sys/src/libventi/client.c
--- a/sys/src/libventi/client.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/client.c Tue May 29 00:00:00 2012 +0200
@@ -65,6 +65,15 @@
if(memcmp(score, vtzeroscore, VtScoreSize) == 0)
return packetalloc();
+ if(z == nil){
+ werrstr("not connected");
+ return nil;
+ }
+
+ if(z->version[1] == '2' && n >= (1<<16)) {
+ werrstr("read count too large for protocol");
+ return nil;
+ }
memset(&tx, 0, sizeof tx);
tx.msgtype = VtTread;
tx.blocktype = type;
diff -r 88ea8de5bdf7 sys/src/libventi/conn.c
--- a/sys/src/libventi/conn.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/conn.c Tue May 29 00:00:00 2012 +0200
@@ -27,6 +27,28 @@
return z;
}
+int
+vtreconn(VtConn *z, int infd, int outfd)
+{
+ NetConnInfo *nci;
+
+ z->state = VtStateAlloc;
+ if(z->infd >= 0)
+ close(z->infd);
+ z->infd = infd;
+ if(z->outfd >= 0)
+ close(z->outfd);
+ z->outfd = outfd;
+ nci = getnetconninfo(nil, infd);
+ if(nci == nil)
+ snprint(z->addr, sizeof z->addr, "/dev/fd/%d", infd);
+ else{
+ strecpy(z->addr, z->addr+sizeof z->addr, nci->raddr);
+ freenetconninfo(nci);
+ }
+ return 0;
+}
+
void
vtfreeconn(VtConn *z)
{
diff -r 88ea8de5bdf7 sys/src/libventi/dial.c
--- a/sys/src/libventi/dial.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/dial.c Tue May 29 00:00:00 2012 +0200
@@ -23,3 +23,21 @@
strecpy(z->addr, z->addr+sizeof z->addr, na);
return z;
}
+
+int
+vtredial(VtConn *z, char *addr)
+{
+ char *na;
+ int fd;
+
+ if(addr == nil)
+ addr = getenv("venti");
+ if(addr == nil)
+ addr = "$venti";
+
+ na = netmkaddr(addr, "tcp", "venti");
+ if((fd = dial(na, nil, nil, nil)) < 0)
+ return fd;
+
+ return vtreconn(z, fd, fd);
+}
diff -r 88ea8de5bdf7 sys/src/libventi/entry.c
--- a/sys/src/libventi/entry.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/entry.c Tue May 29 00:00:00 2012 +0200
@@ -6,13 +6,37 @@
static int
checksize(int n)
{
- if(n < 256 || n > VtMaxLumpSize) {
+ if(n < 256) {
werrstr("bad block size %#ux", n);
return -1;
}
return 0;
}
+// _VtEntryBig integer format is floating-point:
+// (n>>5) << (n&31).
+// Convert this number; must be exact or return -1.
+int
+vttobig(ulong n)
+{
+ int shift;
+ ulong n0;
+
+ n0 = n;
+ shift = 0;
+ while(n >= (1<<(16 - 5))) {
+ if(n & 1)
+ return -1;
+ shift++;
+ n >>= 1;
+ }
+
+ n = (n<<5) | shift;
+ if(((n>>5)<<(n&31)) != n0)
+ sysfatal("vttobig %#lux => %#lux failed", n0, n);
+ return n;
+}
+
void
vtentrypack(VtEntry *e, uchar *p, int index)
{
@@ -20,21 +44,31 @@
int flags;
uchar *op;
int depth;
+ int psize, dsize;
p += index * VtEntrySize;
op = p;
- U32PUT(p, e->gen);
- p += 4;
- U16PUT(p, e->psize);
- p += 2;
- U16PUT(p, e->dsize);
- p += 2;
depth = e->type&VtTypeDepthMask;
flags = (e->flags&~(_VtEntryDir|_VtEntryDepthMask));
flags |= depth << _VtEntryDepthShift;
if(e->type - depth == VtDirType)
flags |= _VtEntryDir;
+ U32PUT(p, e->gen);
+ p += 4;
+ psize = e->psize;
+ dsize = e->dsize;
+ if(psize >= (1<<16) || dsize >= (1<<16)) {
+ flags |= _VtEntryBig;
+ psize = vttobig(psize);
+ dsize = vttobig(dsize);
+ if(psize < 0 || dsize < 0)
+ sysfatal("invalid entry psize/dsize: %ld/%ld", e->psize, e->dsize);
+ }
+ U16PUT(p, psize);
+ p += 2;
+ U16PUT(p, dsize);
+ p += 2;
U8PUT(p, flags);
p++;
memset(p, 0, 5);
@@ -62,10 +96,14 @@
e->dsize = U16GET(p);
p += 2;
e->flags = U8GET(p);
+ p++;
+ if(e->flags & _VtEntryBig) {
+ e->psize = (e->psize>>5)<<(e->psize & 31);
+ e->dsize = (e->dsize>>5)<<(e->dsize & 31);
+ }
e->type = (e->flags&_VtEntryDir) ? VtDirType : VtDataType;
e->type += (e->flags & _VtEntryDepthMask) >> _VtEntryDepthShift;
- e->flags &= ~(_VtEntryDir|_VtEntryDepthMask);
- p++;
+ e->flags &= ~(_VtEntryDir|_VtEntryDepthMask|_VtEntryBig);
p += 5;
e->size = U48GET(p);
p += 6;
diff -r 88ea8de5bdf7 sys/src/libventi/fcall.c
--- a/sys/src/libventi/fcall.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/fcall.c Tue May 29 00:00:00 2012 +0200
@@ -5,7 +5,7 @@
Packet*
vtfcallpack(VtFcall *f)
{
- uchar buf[4];
+ uchar buf[10];
Packet *p;
p = packetalloc();
@@ -60,9 +60,17 @@
if(~buf[0] == 0)
goto Err;
buf[1] = 0;
- buf[2] = f->count >> 8;
- buf[3] = f->count;
- packetappend(p, buf, 4);
+ if(f->count >= (1<<16)) {
+ buf[2] = f->count >> 24;
+ buf[3] = f->count >> 16;
+ buf[4] = f->count >> 8;
+ buf[5] = f->count;
+ packetappend(p, buf, 6);
+ } else {
+ buf[2] = f->count >> 8;
+ buf[3] = f->count;
+ packetappend(p, buf, 4);
+ }
break;
case VtRread:
@@ -163,12 +171,25 @@
case VtTread:
if(packetconsume(p, f->score, VtScoreSize) < 0
- || packetconsume(p, buf, 4) < 0)
+ || packetconsume(p, buf, 2) < 0)
goto Err;
f->blocktype = vtfromdisktype(buf[0]);
if(~f->blocktype == 0)
goto Err;
- f->count = (buf[2] << 8) | buf[3];
+ switch(packetsize(p)) {
+ default:
+ goto Err;
+ case 2:
+ if(packetconsume(p, buf, 2) < 0)
+ goto Err;
+ f->count = (buf[0] << 8) | buf[1];
+ break;
+ case 4:
+ if(packetconsume(p, buf, 4) < 0)
+ goto Err;
+ f->count = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3];
+ break;
+ }
break;
case VtRread:
diff -r 88ea8de5bdf7 sys/src/libventi/file.c
--- a/sys/src/libventi/file.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/file.c Tue May 29 00:00:00 2012 +0200
@@ -36,7 +36,6 @@
vtfilealloc(VtCache *c, VtBlock *b, VtFile *p, u32int offset, int mode)
{
int epb;
- u32int size;
VtEntry e;
VtFile *r;
@@ -68,21 +67,15 @@
}
if(DEPTH(e.type) < sizetodepth(e.size, e.psize, e.dsize)){
- fprint(2, "depth %ud size %llud psize %ud dsize %ud\n",
+ fprint(2, "depth %ud size %llud psize %lud dsize %lud\n",
DEPTH(e.type), e.size, e.psize, e.dsize);
werrstr("bad depth");
return nil;
}
- size = vtcacheblocksize(c);
- if(e.dsize > size || e.psize > size){
- werrstr("block sizes %ud, %ud bigger than cache block size %ud",
- e.psize, e.dsize, size);
- return nil;
- }
-
r = vtmallocz(sizeof(VtFile));
r->c = c;
+ r->bsize = b->size;
r->mode = mode;
r->dsize = e.dsize;
r->psize = e.psize;
@@ -126,7 +119,7 @@
VtBlock *b;
VtFile *f;
- b = vtcacheallocblock(c, VtDirType);
+ b = vtcacheallocblock(c, VtDirType, VtEntrySize);
if(b == nil)
return nil;
@@ -191,8 +184,6 @@
u32int offset;
assert(ISLOCKED(r));
- assert(psize <= VtMaxLumpSize);
- assert(dsize <= VtMaxLumpSize);
assert(type == VtDirType || type == VtDataType);
if(!r->dir){
@@ -325,12 +316,12 @@
static int
shrinksize(VtFile *r, VtEntry *e, uvlong size)
{
- int i, depth, type, isdir, ppb;
+ int i, depth, bsiz, type, isdir, ppb;
uvlong ptrsz;
uchar score[VtScoreSize];
VtBlock *b;
- b = vtcacheglobal(r->c, e->score, e->type);
+ b = vtcacheglobal(r->c, e->score, e->type, r->dsize);
if(b == nil)
return -1;
@@ -342,7 +333,7 @@
ptrsz *= ppb;
isdir = r->dir;
- while(depth > 0){
+ while(DEPTH(type) > 0){
if(b->addr == NilBlock){
/* not worth copying the block just so we can zero some of it */
vtblockput(b);
@@ -369,7 +360,11 @@
type--;
memmove(score, b->data+i*VtScoreSize, VtScoreSize);
vtblockput(b);
- b = vtcacheglobal(r->c, score, type);
+ if(type == VtDataType || type == VtDirType)
+ bsiz = r->dsize;
+ else
+ bsiz = r->psize;
+ b = vtcacheglobal(r->c, score, type, bsiz);
if(b == nil)
return -1;
}
@@ -498,10 +493,10 @@
}
static VtBlock *
-blockwalk(VtBlock *p, int index, VtCache *c, int mode, VtEntry *e)
+blockwalk(VtFile *r, VtBlock *p, int index, VtCache *c, int mode, VtEntry *e)
{
VtBlock *b;
- int type;
+ int type, size;
uchar *score;
VtEntry oe;
@@ -519,12 +514,16 @@
}
/*print("walk from %V/%d ty %d to %V ty %d\n", p->score, index, p->type, score, type); */
+ if(type == VtDirType || type == VtDataType)
+ size = r->dsize;
+ else
+ size = r->psize;
if(mode == VtOWRITE && vtglobaltolocal(score) == NilBlock){
- b = vtcacheallocblock(c, type);
+ b = vtcacheallocblock(c, type, size);
if(b)
goto HaveCopy;
}else
- b = vtcacheglobal(c, score, type);
+ b = vtcacheglobal(c, score, type, size);
if(b == nil || mode == VtOREAD)
return b;
@@ -566,7 +565,7 @@
assert(ISLOCKED(r));
assert(depth <= VtPointerDepth);
- b = vtcacheglobal(r->c, e->score, e->type);
+ b = vtcacheglobal(r->c, e->score, e->type, r->dsize);
if(b == nil)
return -1;
@@ -577,7 +576,7 @@
* or an error occurs.
*/
while(DEPTH(e->type) < depth){
- bb = vtcacheallocblock(r->c, e->type+1);
+ bb = vtcacheallocblock(r->c, e->type+1, r->psize);
if(bb == nil)
break;
memmove(bb->data, b->score, VtScoreSize);
@@ -605,7 +604,7 @@
assert(ISLOCKED(r));
assert(depth <= VtPointerDepth);
- rb = vtcacheglobal(r->c, e->score, e->type);
+ rb = vtcacheglobal(r->c, e->score, e->type, r->psize);
if(rb == nil)
return -1;
@@ -618,7 +617,7 @@
ob = nil;
b = rb;
for(; DEPTH(e->type) > depth; e->type--){
- nb = vtcacheglobal(r->c, b->data, e->type-1);
+ nb = vtcacheglobal(r->c, b->data, e->type-1, r->psize);
if(nb == nil)
break;
if(ob!=nil && ob!=rb)
@@ -720,7 +719,7 @@
m = VtORDWR;
for(i=DEPTH(e.type); i>=0; i--){
- bb = blockwalk(b, index[i], r->c, i==0 ? mode : m, &e);
+ bb = blockwalk(r, b, index[i], r->c, i==0 ? mode : m, &e);
if(bb == nil)
goto Err;
vtblockput(b);
@@ -768,7 +767,7 @@
index[DEPTH(e.type)] = r->offset % r->epb;
for(i=DEPTH(e.type); i>=1; i--){
- bb = blockwalk(b, index[i], r->c, VtOREAD, &e);
+ bb = blockwalk(r, b, index[i], r->c, VtOREAD, &e);
if(bb == nil)
goto Err;
vtblockput(b);
@@ -837,7 +836,7 @@
case VtORDWR:
assert(r->mode == VtORDWR);
if(r->local == 1){
- b = vtcacheglobal(r->c, r->score, VtDirType);
+ b = vtcacheglobal(r->c, r->score, VtDirType, r->bsize);
if(b == nil)
return nil;
b->pc = getcallerpc(&r);
@@ -861,7 +860,7 @@
}
addr = vtglobaltolocal(r->score);
if(addr == NilBlock)
- return vtcacheglobal(r->c, r->score, VtDirType);
+ return vtcacheglobal(r->c, r->score, VtDirType, r->bsize);
b = vtcachelocal(r->c, addr, VtDirType);
if(b)
@@ -1220,7 +1219,7 @@
*/
index[depth] = r->offset % r->epb;
for(i=depth; i>=0; i--){
- bb = blockwalk(b, index[i], r->c, VtORDWR, &e);
+ bb = blockwalk(r, b, index[i], r->c, VtORDWR, &e);
if(bb == nil)
goto Err;
bi[i] = bb;
diff -r 88ea8de5bdf7 sys/src/libventi/log.c
--- a/sys/src/libventi/log.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/log.c Tue May 29 00:00:00 2012 +0200
@@ -2,8 +2,6 @@
#include <libc.h>
#include <venti.h>
-char *VtServerLog = "libventi/server";
-
int ventilogging;
#define log not_the_log_library_call
@@ -164,35 +162,16 @@
qunlock(&vl.lk);
}
-static int
-timefmt(Fmt *fmt)
-{
- static uvlong t0;
- uvlong t;
-
- if(t0 == 0)
- t0 = nsec();
- t = nsec()-t0;
- return fmtprint(fmt, "T+%d.%04d", (uint)(t/1000000000), (uint)(t%1000000000)/100000);
-}
-
void
vtlogvprint(VtLog *l, char *fmt, va_list arg)
{
int n;
char *p;
VtLogChunk *c;
- static int first = 1;
if(l == nil)
return;
-
- if(first){
- fmtinstall('T', timefmt);
- first = 0;
- }
-
-
+
qlock(&l->lk);
c = l->w;
n = c->ep - c->wp;
diff -r 88ea8de5bdf7 sys/src/libventi/mkfile
--- a/sys/src/libventi/mkfile Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/mkfile Tue May 29 00:00:00 2012 +0200
@@ -24,6 +24,7 @@
scorefmt.$O\
send.$O\
server.$O\
+ sha1.$O\
srvhello.$O\
strdup.$O\
string.$O\
diff -r 88ea8de5bdf7 sys/src/libventi/root.c
--- a/sys/src/libventi/root.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/root.c Tue May 29 00:00:00 2012 +0200
@@ -6,19 +6,30 @@
static int
checksize(int n)
{
- if(n < 256 || n > VtMaxLumpSize) {
+ if(n < 256) {
werrstr("bad block size");
return -1;
}
return 0;
}
+extern int vttobig(ulong);
+
void
vtrootpack(VtRoot *r, uchar *p)
{
uchar *op = p;
+ int vers, bsize;
- U16PUT(p, VtRootVersion);
+ vers = VtRootVersion;
+ bsize = r->blocksize;
+ if(bsize >= (1<<16)) {
+ vers |= _VtRootVersionBig;
+ bsize = vttobig(bsize);
+ if(bsize < 0)
+ sysfatal("invalid root blocksize: %#lx", r->blocksize);
+ }
+ U16PUT(p, vers);
p += 2;
memmove(p, r->name, sizeof(r->name));
p += sizeof(r->name);
@@ -26,7 +37,7 @@
p += sizeof(r->type);
memmove(p, r->score, VtScoreSize);
p += VtScoreSize;
- U16PUT(p, r->blocksize);
+ U16PUT(p, bsize);
p += 2;
memmove(p, r->prev, VtScoreSize);
p += VtScoreSize;
@@ -42,7 +53,7 @@
memset(r, 0, sizeof(*r));
vers = U16GET(p);
- if(vers != VtRootVersion) {
+ if((vers&~_VtRootVersionBig) != VtRootVersion) {
werrstr("unknown root version");
return -1;
}
@@ -56,6 +67,8 @@
memmove(r->score, p, VtScoreSize);
p += VtScoreSize;
r->blocksize = U16GET(p);
+ if(vers & _VtRootVersionBig)
+ r->blocksize = (r->blocksize >> 5) << (r->blocksize & 31);
if(checksize(r->blocksize) < 0)
return -1;
p += 2;
diff -r 88ea8de5bdf7 sys/src/libventi/rpc.c
--- a/sys/src/libventi/rpc.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/rpc.c Tue May 29 00:00:00 2012 +0200
@@ -37,6 +37,12 @@
uchar tag, buf[2], *top;
Rwait *r, *rr;
+ if(z == nil){
+ werrstr("not connected");
+ packetfree(p);
+ return nil;
+ }
+
/* must malloc because stack could be private */
r = vtmallocz(sizeof(Rwait));
diff -r 88ea8de5bdf7 sys/src/libventi/send.c
--- a/sys/src/libventi/send.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/send.c Tue May 29 00:00:00 2012 +0200
@@ -11,7 +11,7 @@
{
IOchunk ioc;
int n, tot;
- uchar buf[2];
+ uchar buf[4];
if(z->state != VtStateConnected) {
werrstr("session not connected");
@@ -20,15 +20,24 @@
/* add framing */
n = packetsize(p);
- if(n >= (1<<16)) {
- werrstr("packet too large");
- packetfree(p);
- return -1;
+ if(z->version[1] == '2') {
+ if(n >= (1<<16)) {
+ werrstr("packet too large");
+ packetfree(p);
+ return -1;
+ }
+ buf[0] = n>>8;
+ buf[1] = n;
+ packetprefix(p, buf, 2);
+ ventisendbytes += n+2;
+ } else {
+ buf[0] = n>>24;
+ buf[1] = n>>16;
+ buf[2] = n>>8;
+ buf[3] = n;
+ packetprefix(p, buf, 4);
+ ventisendbytes += n+4;
}
- buf[0] = n>>8;
- buf[1] = n;
- packetprefix(p, buf, 2);
- ventisendbytes += n+2;
ventisendpackets++;
tot = 0;
@@ -63,7 +72,7 @@
_vtrecv(VtConn *z)
{
uchar buf[10], *b;
- int n;
+ int n, need;
Packet *p;
int size, len;
@@ -75,11 +84,12 @@
p = z->part;
/* get enough for head size */
size = packetsize(p);
- while(size < 2) {
- b = packettrailer(p, 2);
+ need = z->version[1] - '0'; // 2 or 4
+ while(size < need) {
+ b = packettrailer(p, need);
assert(b != nil);
if(0) fprint(2, "%d read hdr\n", getpid());
- n = read(z->infd, b, 2);
+ n = read(z->infd, b, need);
if(0) fprint(2, "%d got %d (%r)\n", getpid(), n);
if(n==0 || (n<0 && !interrupted()))
goto Err;
@@ -87,10 +97,15 @@
packettrim(p, 0, size);
}
- if(packetconsume(p, buf, 2) < 0)
+ if(packetconsume(p, buf, need) < 0)
goto Err;
- len = (buf[0] << 8) | buf[1];
- size -= 2;
+ if(z->version[1] == '2') {
+ len = (buf[0] << 8) | buf[1];
+ size -= 2;
+ } else {
+ len = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3];
+ size -= 4;
+ }
while(size < len) {
n = len - size;
diff -r 88ea8de5bdf7 sys/src/libventi/server.c
--- a/sys/src/libventi/server.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/server.c Tue May 29 00:00:00 2012 +0200
@@ -31,6 +31,8 @@
static void listenproc(void*);
static void connproc(void*);
+char *VtServerLog = "libventi/server";
+
static void
scincref(VtSconn *sc)
{
diff -r 88ea8de5bdf7 sys/src/libventi/sha1.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/src/libventi/sha1.c Tue May 29 00:00:00 2012 +0200
@@ -0,0 +1,28 @@
+#include <u.h>
+#include <libc.h>
+#include <venti.h>
+#include <libsec.h>
+
+void
+vtsha1(uchar score[VtScoreSize], uchar *p, int n)
+{
+ DigestState ds;
+
+ memset(&ds, 0, sizeof ds);
+ sha1(p, n, score, &ds);
+}
+
+int
+vtsha1check(uchar score[VtScoreSize], uchar *p, int n)
+{
+ DigestState ds;
+ uchar score2[VtScoreSize];
+
+ memset(&ds, 0, sizeof ds);
+ sha1(p, n, score2, &ds);
+ if(memcmp(score, score2, VtScoreSize) != 0) {
+ werrstr("vtsha1check failed");
+ return -1;
+ }
+ return 0;
+}
diff -r 88ea8de5bdf7 sys/src/libventi/time.c
--- a/sys/src/libventi/time.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/time.c Tue May 29 00:00:00 2012 +0200
@@ -8,15 +8,18 @@
vlong ns;
Tm tm;
+ if(fmt->flags&FmtSign){
+ ns = va_arg(fmt->args, long);
+ ns *= 1000000000;
+ } else
+ ns = nsec();
+ tm = *localtime(ns/1000000000);
if(fmt->flags&FmtLong){
- ns = nsec();
- tm = *localtime(ns/1000000000);
return fmtprint(fmt, "%04d/%02d%02d %02d:%02d:%02d.%03d",
tm.year+1900, tm.mon+1, tm.mday,
tm.hour, tm.min, tm.sec,
(int)(ns%1000000000)/1000000);
}else{
- tm = *localtime(time(0));
return fmtprint(fmt, "%04d/%02d%02d %02d:%02d:%02d",
tm.year+1900, tm.mon+1, tm.mday,
tm.hour, tm.min, tm.sec);
diff -r 88ea8de5bdf7 sys/src/libventi/version.c
--- a/sys/src/libventi/version.c Sat May 26 00:00:00 2012 +0200
+++ b/sys/src/libventi/version.c Tue May 29 00:00:00 2012 +0200
@@ -3,6 +3,7 @@
#include <venti.h>
static char *okvers[] = {
+ "04",
"02",
nil,
};
|
