- Gs-commits - mailman3.cels.anl.gov

Grayskull Repository branch, master, updated. git-migration-291-g89acab2
by noreply＠mcs.anl.gov 16 Dec '09

16 Dec '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 89acab203268a051a6067d84451e6eeb03f3213c (commit) from c116f7ffd90d2b3c6c0838b1451d4c16bf664993 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 89acab203268a051a6067d84451e6eeb03f3213c Author: Phil Carns <carns(a)mcs.anl.gov> Date: Wed Dec 16 10:58:06 2009 -0500 make cosd write function not trash caller pointers The cosd write function now makes copies of input extent arrays so that it can modify them safely. Also added an optional cleanup function pointer for each cosd op for situations like this. ----------------------------------------------------------------------- Summary of changes: .../gsl/resources/cosd-prototype/cosd-prototype.c | 64 ++++++++++++++++++-- 1 files changed, 59 insertions(+), 5 deletions(-) Diff of changes: diff --git a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c index b4b745b..0bfa578 100644 --- a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c +++ b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c @@ -199,6 +199,7 @@ struct cosd_op gs_op_id_t op_id; struct gs_op op; int error_code; + void (*cleanup_fn)(struct cosd_op* c_op); }; static void* thread_fn(void* foo) @@ -214,6 +215,12 @@ static void* thread_fn(void* foo) /* spin on op worker */ while(op->op_worker(op) != 1); + /* call cleanup function if present */ + if(c_op->cleanup_fn) + { + c_op->cleanup_fn(c_op); + } + /* trigger completion of the operation */ gs_opcache_complete_op(cosd_opcache, op, c_op->error_code); @@ -370,6 +377,12 @@ static int gs_cosd_poll(gs_context_t context, int millisecs) ret = gop->op_worker(gop); if(ret == 1) { + /* call cleanup function if present */ + if(c_op->cleanup_fn) + { + c_op->cleanup_fn(c_op); + } + /* done */ gs_opcache_complete_op(cosd_opcache, gop, c_op->error_code); } @@ -629,6 +642,24 @@ int gs_cosd_finalize(void) return 0; } +/* write_op_cleanup() + * + * cleans up memory after a write operation completes + */ +static void write_op_cleanup(struct cosd_op *c_op) +{ + if(c_op->u.write.mem_offsets) + free(c_op->u.write.mem_offsets); + if(c_op->u.write.mem_sizes) + free(c_op->u.write.mem_sizes); + if(c_op->u.write.obj_sizes) + free(c_op->u.write.obj_sizes); + if(c_op->u.write.obj_offsets) + free(c_op->u.write.obj_offsets); + + return; +} + /* write_op_worker() * * writes data to an object @@ -644,7 +675,7 @@ static int write_op_worker(struct gs_op* op) int mem_index = 0; int obj_index = 0; struct logical_map_entry *tmp_update; - char* mem_ptr; + const char* mem_ptr; void* scratch_buffer = NULL; int scratch_size = 0; int64_t aligned_size = 0; @@ -1047,17 +1078,36 @@ gs_ret_t gs_cosd_write_post( c_op->op_id = gs_id_gen(gs_cosd_resource_id, (uint64_t)(op->cache_id)); c_op->u.write.oid = oid; c_op->u.write.txn_number = txn_number; - c_op->u.write.mem_offsets = mem_offsets; - c_op->u.write.mem_sizes = mem_sizes; c_op->u.write.mem_count = mem_count; - c_op->u.write.obj_offsets = obj_offsets; - c_op->u.write.obj_sizes = obj_sizes; c_op->u.write.obj_count = obj_count; + /* copy arrays so that we can modify them internally */ + /* TODO: error handling */ + c_op->u.write.mem_offsets = malloc(mem_count*sizeof(*mem_offsets)); + assert(c_op->u.write.mem_offsets); + memcpy(c_op->u.write.mem_offsets, mem_offsets, + mem_count*sizeof(*mem_offsets)); + + c_op->u.write.mem_sizes = malloc(mem_count*sizeof(*mem_sizes)); + assert(c_op->u.write.mem_sizes); + memcpy(c_op->u.write.mem_sizes, mem_sizes, + mem_count*sizeof(*mem_sizes)); + + c_op->u.write.obj_offsets = malloc(obj_count*sizeof(*obj_offsets)); + assert(c_op->u.write.obj_offsets); + memcpy(c_op->u.write.obj_offsets, obj_offsets, + obj_count*sizeof(*obj_offsets)); + + c_op->u.write.obj_sizes = malloc(obj_count*sizeof(*obj_sizes)); + assert(c_op->u.write.obj_sizes); + memcpy(c_op->u.write.obj_sizes, obj_sizes, + obj_count*sizeof(*obj_sizes)); + *op_id = c_op->op_id; /* TODO: put this in the fill function if we keep it? */ op->op_worker = write_op_worker; + c_op->cleanup_fn = write_op_cleanup; gs_cosd_launch_op(op, gs_cosd_progress_mode); @@ -1099,6 +1149,7 @@ gs_ret_t gs_cosd_dump_post( /* TODO: put this in the fill function if we keep it? */ op->op_worker = dump_op_worker; + c_op->cleanup_fn = NULL; gs_cosd_launch_op(op, gs_cosd_progress_mode); @@ -1136,6 +1187,7 @@ gs_ret_t gs_cosd_create_post( /* TODO: put this in the fill function if we keep it? */ op->op_worker = create_op_worker; + c_op->cleanup_fn = NULL; gs_cosd_launch_op(op, gs_cosd_progress_mode); @@ -1250,6 +1302,7 @@ gs_ret_t gs_cosd_get_version_post( /* TODO: put this in the fill function if we keep it? */ op->op_worker = get_version_op_worker; + c_op->cleanup_fn = NULL; gs_cosd_launch_op(op, gs_cosd_progress_mode); @@ -1536,6 +1589,7 @@ gs_ret_t gs_cosd_txn_close_post( /* TODO: put this in the fill function if we keep it? */ op->op_worker = txn_close_op_worker; + c_op->cleanup_fn = NULL; gs_cosd_launch_op(op, gs_cosd_progress_mode); hooks/post-receive -- Grayskull Repository

1 0

Grayskull Repository branch, master, updated. git-migration-290-gc116f7f
by noreply＠mcs.anl.gov 15 Dec '09

15 Dec '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via c116f7ffd90d2b3c6c0838b1451d4c16bf664993 (commit) from 612fc3601a2011e3adc30b7ef379445ed8636421 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit c116f7ffd90d2b3c6c0838b1451d4c16bf664993 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 17:24:19 2009 -0500 beginning of concurrent cosd I/O benchmark ----------------------------------------------------------------------- Summary of changes: .../resources/cosd-prototype/test/4M-w-serial.txt | 7 + .../cosd-prototype/test/cosd-bench-concurrent.gs | 240 ++++++++++++++++++++ .../gsl/resources/cosd-prototype/test/module.mk.in | 1 + 3 files changed, 248 insertions(+), 0 deletions(-) create mode 100644 code/src/gsl/resources/cosd-prototype/test/4M-w-serial.txt create mode 100644 code/src/gsl/resources/cosd-prototype/test/cosd-bench-concurrent.gs Diff of changes: diff --git a/code/src/gsl/resources/cosd-prototype/test/4M-w-serial.txt b/code/src/gsl/resources/cosd-prototype/test/4M-w-serial.txt new file mode 100644 index 0000000..22c07a2 --- /dev/null +++ b/code/src/gsl/resources/cosd-prototype/test/4M-w-serial.txt @@ -0,0 +1,7 @@ +# <op> <offset> <size> +write 0 4194304 +write 4194304 4194304 +write 8388608 4194304 +write 12582912 4194304 +write 16777216 4194304 +write 20971520 4194304 diff --git a/code/src/gsl/resources/cosd-prototype/test/cosd-bench-concurrent.gs b/code/src/gsl/resources/cosd-prototype/test/cosd-bench-concurrent.gs new file mode 100644 index 0000000..aa43467 --- /dev/null +++ b/code/src/gsl/resources/cosd-prototype/test/cosd-bench-concurrent.gs @@ -0,0 +1,240 @@ + +#include <stdio.h> +#include <errno.h> +#include "../../../include/gs.h" +#include "../cosd-prototype.gsh" +#include "gs-config.h" + +/* TODO: this needs to be in a header somewhere */ +#ifndef llu +#if SIZEOF_LONG_INT == 4 +# define llu(x) (x) +# define lld(x) (x) +# define SCANF_lld lld +#elif SIZEOF_LONG_INT == 8 +# define llu(x) (unsigned long long)(x) +# define lld(x) (long long)(x) +# define SCANF_lld ld +#else +# error Unexpected sizeof(long int) +#endif +#endif + +enum op_type +{ + WRITE, + READ +}; + +struct bench_op +{ + int64_t offset; + int64_t size; + enum op_type type; + struct gs_list_link list_link; +}; + +GS_LIST_HEAD(op_list); +gs_mutex_t op_list_mutex = GS_MUTEX_INITIALIZER; + +static __blocking int do_cosd_test(void) +{ + int ret; + + uint64_t out_oid1; + uint64_t version; + + printf("Creating oid 1 (if it doesn't already exist)...\n"); + ret = gs_cosd_create(1, &out_oid1); + if(ret != 0 && ret != -EEXIST) + { + printf("Error creating oid=1: %d\n", ret); + return 1; + } + printf("DONE\n"); + + printf("Getting version number of oid 1...\n"); + ret = gs_cosd_get_version(1, &version); + if(ret != 0) + { + printf("Error getting version number for oid=1: %d\n", ret); + return(1); + } + printf("DONE: %llu\n", llu(version)); + + ret = gs_cosd_dump(); + assert(ret == 0); + + pwait + { + int i; + /* TODO: make "16" tunable */ + for(i=0; i<16; i++) + { + pbranch + { + struct gs_list_link* tmp_link; + char* buffer = NULL; + uint64_t b_version; + struct bench_op* tmp_op; + int b_ret; + /* TODO: make buffer size configurable, and check that + * nothing in workload desc is bigger than this + */ + int64_t buffer_size = 4*1024*1024; + + buffer = malloc(buffer_size); + assert(buffer); + + gs_mutex_lock(&op_list_mutex); + while((tmp_link = gs_list_pop(&op_list)) != NULL) + { + b_version = version++; + gs_mutex_unlock(&op_list_mutex); + + tmp_op = gs_list_get_entry(tmp_link, struct bench_op, + list_link); + /* only writes for now */ + assert(tmp_op->type == WRITE); + + ret = gs_cosd_txn_open(1, b_version); + if(ret != 0) + { + /* TODO: err handling */ + assert(0); + } + + b_ret = gs_cosd_write(1, b_version, &buffer, + &buffer_size, 1, &tmp_op->offset, &tmp_op->size, 1); + if(b_ret < 0) + { + /* TODO: error handling */ + assert(0); + } + + ret = gs_cosd_txn_close(1, b_version); + if(ret != 0) + { + /* TODO: error handling */ + assert(0); + } + + gs_mutex_lock(&op_list_mutex); + } + gs_mutex_unlock(&op_list_mutex); + + free(buffer); + } + } + } + + ret = gs_cosd_dump(); + assert(ret == 0); + return 0; +} + +int done = 0; + +static void done_callback(void *up, int ret) +{ + done = 1; +} + + +int main(int argc, char *argv[]) +{ + gs_context_t ctx; + gs_op_id_t op_id; + int mode = GS_PROG_POLL; + int pc = 0; + FILE *desc = 0; + char line[2048]; + char op_string[100]; + int64_t size; + int64_t offset; + int ret; + struct bench_op* tmp_op; + + if(argc != 4) + { + fprintf(stderr, "Usage: cosd1 <1|2> <workload description file> <cosd dir>\n"); + fprintf(stderr, " # 1 for polling\n"); + fprintf(stderr, " # 2 for thread-per-op\n"); + return(-1); + } + + sscanf(argv[1], "%d", &mode); + if(mode > 2 || mode < 1) + { + fprintf(stderr, "Usage: cosd1 <1|2>\n"); + fprintf(stderr, " # 1 for polling\n"); + fprintf(stderr, " # 2 for thread-per-op\n"); + return(-1); + } + + /* parse description of workload */ + desc = fopen(argv[2], "r"); + if(!desc) + { + perror("fopen"); + return(-1); + } + while(fgets(line, 2048, desc)) + { + if(line[0] == '#') + continue; +#if SIZEOF_LONG_INT == 4 + ret = sscanf(line, "%s %lld %lld", op_string, &offset, &size); +#else + ret = sscanf(line, "%s %ld %ld", op_string, &offset, &size); +#endif + if(ret != 3) + { + fprintf(stderr, "Error: bad line: %s\n", line); + return(-1); + } + tmp_op = malloc(sizeof(*tmp_op)); + assert(tmp_op); + + /* only writes for now */ + assert(strcmp(op_string, "write") == 0); + tmp_op->type = WRITE; + tmp_op->offset = offset; + tmp_op->size = size; + gs_list_add(&tmp_op->list_link, &op_list); + } + fclose(desc); + + if(mode == 1) + printf("Initializing cosd in poll-driven mode...\n"); + else + printf("Initializing cosd in thread-per-op mode...\n"); + + gs_cosd_init(mode, argv[3]); + printf("Done.\n"); + + gs_context_create(&ctx, 1, "cosd"); + + done = 0; + do_cosd_test_post(done_callback, NULL, NULL, ctx, &op_id); + while(done == 0) + { + pc++; + gs_poll(ctx, 10000); + fflush(NULL); + printf("polled %d times...\n", pc); + } + + gs_cosd_finalize(); + gs_context_destroy(ctx); + return 0; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * End: + * + * vim: ft=c ts=8 sts=4 sw=4 expandtab + */ diff --git a/code/src/gsl/resources/cosd-prototype/test/module.mk.in b/code/src/gsl/resources/cosd-prototype/test/module.mk.in index 8ef8b52..78bca02 100644 --- a/code/src/gsl/resources/cosd-prototype/test/module.mk.in +++ b/code/src/gsl/resources/cosd-prototype/test/module.mk.in @@ -1,5 +1,6 @@ DIR := resources/cosd-prototype/test GSTESTSRC += $(DIR)/cosd1.gs +GSTESTSRC += $(DIR)/cosd-bench-concurrent.gs MODLIBS_$(DIR) = -lpthread @DB_LIB@ hooks/post-receive -- Grayskull Repository

1 0

Grayskull Repository branch, master, updated. git-migration-289-g612fc36
by noreply＠mcs.anl.gov 15 Dec '09

15 Dec '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 612fc3601a2011e3adc30b7ef379445ed8636421 (commit) from 6a72db3916e25d53b3b0b015180d79dc0cefcc72 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 612fc3601a2011e3adc30b7ef379445ed8636421 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 16:25:32 2009 -0500 another off by one bug in cosd ----------------------------------------------------------------------- Summary of changes: .../gsl/resources/cosd-prototype/cosd-prototype.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) Diff of changes: diff --git a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c index 69b4e9c..b4b745b 100644 --- a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c +++ b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c @@ -1365,7 +1365,7 @@ static int txn_close_op_worker(struct gs_op* op) assert(0); } - if(lme.logical_offset > tmp_update->logical_offset_end) + if(lme.logical_offset >= tmp_update->logical_offset_end) { /* run out of overlapping regions */ done = 1; hooks/post-receive -- Grayskull Repository

1 0

Grayskull Repository branch, master, updated. git-migration-288-g6a72db3
by noreply＠mcs.anl.gov 15 Dec '09

15 Dec '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 6a72db3916e25d53b3b0b015180d79dc0cefcc72 (commit) via 834c0c4c8b91eb4bb1d05d742f11628a162bf58f (commit) via 8fef453a8515cc48a03fa5d0ad4cd160a1387e88 (commit) via cac3eeecf5a7efa3be0c774640ed7cff182e910d (commit) via f0389f311ac8435df8b4d34437d150fb09afae27 (commit) via 62f89db1a52ebe73ce33e98a5671e5b62924249a (commit) from 088e88dd34e9d2caf5c67383440032c7d51f9a57 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 6a72db3916e25d53b3b0b015180d79dc0cefcc72 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 16:17:55 2009 -0500 fixed array index bug in cosd commit 834c0c4c8b91eb4bb1d05d742f11628a162bf58f Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 16:08:24 2009 -0500 fix variable initialization bug commit 8fef453a8515cc48a03fa5d0ad4cd160a1387e88 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 16:02:46 2009 -0500 off by one bug in logical map query commit cac3eeecf5a7efa3be0c774640ed7cff182e910d Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 15:59:38 2009 -0500 tweaks to test program commit f0389f311ac8435df8b4d34437d150fb09afae27 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 15:35:38 2009 -0500 test writing old versions commit 62f89db1a52ebe73ce33e98a5671e5b62924249a Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 15:32:41 2009 -0500 fix test program bugs ----------------------------------------------------------------------- Summary of changes: .../gsl/resources/cosd-prototype/cosd-prototype.c | 6 ++- .../src/gsl/resources/cosd-prototype/test/cosd1.gs | 50 ++++++++++++++++++-- 2 files changed, 50 insertions(+), 6 deletions(-) Diff of changes: diff --git a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c index f0f3282..69b4e9c 100644 --- a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c +++ b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c @@ -1348,7 +1348,7 @@ static int txn_close_op_worker(struct gs_op* op) tmp_update = gs_list_get_entry(iterator, struct logical_map_entry, list_link); - lmk.logical_offset_end = tmp_update->logical_offset; + lmk.logical_offset_end = tmp_update->logical_offset + 1; done = 0; while(!done) { @@ -1633,6 +1633,8 @@ static int merge_logical_map(struct gs_list_link* list1_in, int } /* loop through arrays and merge into output array */ + i=0; + j=0; while(i<list1_count || j<list2_count) { @@ -1686,7 +1688,7 @@ static int merge_logical_map(struct gs_list_link* list1_in, int if(list1_array[i].logical_offset == list2_array[j].logical_offset) { /* TODO: think about how to handle version wrap... */ - if(list1_array[i].version > list2_array[i].version) + if(list1_array[i].version > list2_array[j].version) { winner = &list1_array[i]; loser = &list2_array[j]; diff --git a/code/src/gsl/resources/cosd-prototype/test/cosd1.gs b/code/src/gsl/resources/cosd-prototype/test/cosd1.gs index 91456d9..41ef5fe 100644 --- a/code/src/gsl/resources/cosd-prototype/test/cosd1.gs +++ b/code/src/gsl/resources/cosd-prototype/test/cosd1.gs @@ -23,7 +23,7 @@ static __blocking int do_cosd_test(void) { int ret; - char buffer[1024]; + char buffer[4096]; int64_t buffer_szs[3]; char* buffer_offsets[3]; @@ -51,6 +51,9 @@ static __blocking int do_cosd_test(void) } printf("DONE: %llu\n", llu(version)); + ret = gs_cosd_dump(); + assert(ret == 0); + /********************************************************/ printf("opening txn...\n"); ret = gs_cosd_txn_open(1, (version+1)); if(ret != 0) @@ -83,6 +86,9 @@ static __blocking int do_cosd_test(void) } printf("DONE\n"); + ret = gs_cosd_dump(); + assert(ret == 0); + /********************************************************/ printf("opening txn...\n"); ret = gs_cosd_txn_open(1, (version+3)); if(ret != 0) @@ -99,8 +105,10 @@ static __blocking int do_cosd_test(void) buffer_szs[0] = 512; buffer_szs[1] = 256; buffer_szs[2] = 256; - obj_offsets[1024] = 100; - obj_offsets[2000] = 924; + obj_offsets[0] = 1024; + obj_offsets[1] = 2000; + obj_szs[0] = 100; + obj_szs[1] = 924; ret = gs_cosd_write(1, (version+3), buffer_offsets, buffer_szs, 3, obj_offsets, obj_szs, 2); if(ret != 0) @@ -119,11 +127,45 @@ static __blocking int do_cosd_test(void) } printf("DONE\n"); + ret = gs_cosd_dump(); + assert(ret == 0); + /********************************************************/ + printf("opening txn...\n"); + ret = gs_cosd_txn_open(1, (version+2)); + if(ret != 0) + { + printf("Error opening txn 1: %d\n", ret); + return 1; + } + printf("DONE\n"); + + printf("writing bigger region with old version...\n"); + buffer_offsets[0] = buffer; + buffer_szs[0] = 4096; + obj_offsets[0] = 200; + obj_szs[0] = 4096; + ret = gs_cosd_write(1, (version+2), buffer_offsets, buffer_szs, 1, + obj_offsets, obj_szs, 1); + if(ret != 0) + { + printf("Error writing with listio: %d\n", ret); + return 1; + } + printf("DONE\n"); + + printf("closing txn...\n"); + ret = gs_cosd_txn_close(1, (version+2)); + if(ret != 0) + { + printf("Error closing txn 1: %d\n", ret); + return 1; + } + printf("DONE\n"); + ret = gs_cosd_dump(); assert(ret == 0); - return 0; } hooks/post-receive -- Grayskull Repository

1 0

Grayskull Repository branch, master, updated. git-migration-282-g088e88d
by noreply＠mcs.anl.gov 15 Dec '09

15 Dec '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 088e88dd34e9d2caf5c67383440032c7d51f9a57 (commit) from 6443e3baeae94a4b1d2181b7142ea2334168bebd (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 088e88dd34e9d2caf5c67383440032c7d51f9a57 Author: Dries Kimpe <dkimpe(a)mcs.anl.gov> Date: Tue Dec 15 15:14:17 2009 -0600 Dot diagram ----------------------------------------------------------------------- Summary of changes: wiki/Makefile | 5 ++++ wiki/proto-overview.dot | 53 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 0 deletions(-) create mode 100644 wiki/Makefile create mode 100644 wiki/proto-overview.dot Diff of changes: diff --git a/wiki/Makefile b/wiki/Makefile new file mode 100644 index 0000000..485075c --- /dev/null +++ b/wiki/Makefile @@ -0,0 +1,5 @@ + +%.eps: %.dot + dot -Teps -o $@ $^ + +all:: proto-overview.eps diff --git a/wiki/proto-overview.dot b/wiki/proto-overview.dot new file mode 100644 index 0000000..a95ae52 --- /dev/null +++ b/wiki/proto-overview.dot @@ -0,0 +1,53 @@ +digraph prototype { + +overlap = false; +splines = true; + +subgraph resources +{ +color = blue; +rank = same; +node [style=filled,color=lightgray,shape=octagon]; +cosd [label="COSD"]; +net [label="MPI/Net"]; +} + +client [label="Client"]; +obj_group [label="Object/Group"]; +rebuild [label="Rebuild"]; +rosd [label="ROSD"]; +placement [label="Placement"]; +rpc [label="RPC"]; +pipelining [label="Pipelining"]; +failure [label="Failure/Gossip"]; + +client -> failure; +client -> obj_group; +client -> pipelining; +client -> placement; + +obj_group -> placement; +obj_group -> rosd; +obj_group -> net; + +rebuild -> failure; +rebuild -> rosd [ label="?"] ; +rebuild -> placement; + +failure -> rpc; + +placement -> failure; + +rosd -> placement; +rosd -> cosd; +rosd -> rpc; +rosd -> failure; + +rpc -> net; + +pipelining -> net; +pipelining -> cosd; + + + +} hooks/post-receive -- Grayskull Repository

1 0

Grayskull Repository branch, master, updated. git-migration-281-g6443e3b
by noreply＠mcs.anl.gov 15 Dec '09

15 Dec '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 6443e3baeae94a4b1d2181b7142ea2334168bebd (commit) via 933104037d1dd69e70bf092d6db891c243a5dd3b (commit) via d3f9a17a5918e0a321f5fe476cd35fba63ea9dc9 (commit) from 0fd9f3de3dfa793453f556f5951762a3348e1026 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 6443e3baeae94a4b1d2181b7142ea2334168bebd Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 14:51:06 2009 -0500 buf fix in cosd buffer alignment commit 933104037d1dd69e70bf092d6db891c243a5dd3b Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 14:33:21 2009 -0500 add missing open/close of txn for listio commit d3f9a17a5918e0a321f5fe476cd35fba63ea9dc9 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 14:30:27 2009 -0500 trivial testing of listio ----------------------------------------------------------------------- Summary of changes: .../gsl/resources/cosd-prototype/cosd-prototype.c | 4 +- .../src/gsl/resources/cosd-prototype/test/cosd1.gs | 56 +++++++++++++++++-- 2 files changed, 52 insertions(+), 8 deletions(-) Diff of changes: diff --git a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c index b501646..f0f3282 100644 --- a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c +++ b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c @@ -788,9 +788,9 @@ static int write_op_worker(struct gs_op* op) c_op->error_code = -errno; return(1); } - memcpy(scratch_buffer, mem_ptr, tmp_update->size); - mem_ptr = scratch_buffer; } + memcpy(scratch_buffer, mem_ptr, tmp_update->size); + mem_ptr = scratch_buffer; } /* actually write the data */ diff --git a/code/src/gsl/resources/cosd-prototype/test/cosd1.gs b/code/src/gsl/resources/cosd-prototype/test/cosd1.gs index 288708a..91456d9 100644 --- a/code/src/gsl/resources/cosd-prototype/test/cosd1.gs +++ b/code/src/gsl/resources/cosd-prototype/test/cosd1.gs @@ -23,11 +23,13 @@ static __blocking int do_cosd_test(void) { int ret; - int64_t buffer_sz = 1024; char buffer[1024]; - char* buffer_offset = buffer; - int64_t obj_sz = 1024; - int64_t obj_offset = 0; + + int64_t buffer_szs[3]; + char* buffer_offsets[3]; + int64_t obj_offsets[3]; + int64_t obj_szs[3]; + uint64_t out_oid1; uint64_t version; @@ -59,8 +61,12 @@ static __blocking int do_cosd_test(void) printf("DONE\n"); printf("writing 0-1024...\n"); - ret = gs_cosd_write(1, (version+1), &buffer_offset, &buffer_sz, 1, &obj_offset, - &obj_sz, 1); + buffer_offsets[0] = buffer; + buffer_szs[0] = 1024; + obj_offsets[0] = 0; + obj_szs[0] = 1024; + ret = gs_cosd_write(1, (version+1), buffer_offsets, buffer_szs, 1, + obj_offsets, obj_szs, 1); if(ret != 0) { printf("Error writing 0-1024: %d\n", ret); @@ -77,6 +83,44 @@ static __blocking int do_cosd_test(void) } printf("DONE\n"); + printf("opening txn...\n"); + ret = gs_cosd_txn_open(1, (version+3)); + if(ret != 0) + { + printf("Error opening txn 1: %d\n", ret); + return 1; + } + printf("DONE\n"); + + printf("writing with listio (3 mem regions, 2 file regions)...\n"); + buffer_offsets[0] = buffer; + buffer_offsets[1] = buffer+512; + buffer_offsets[2] = buffer+768; + buffer_szs[0] = 512; + buffer_szs[1] = 256; + buffer_szs[2] = 256; + obj_offsets[1024] = 100; + obj_offsets[2000] = 924; + ret = gs_cosd_write(1, (version+3), buffer_offsets, buffer_szs, 3, + obj_offsets, obj_szs, 2); + if(ret != 0) + { + printf("Error writing with listio: %d\n", ret); + return 1; + } + printf("DONE\n"); + + printf("closing txn...\n"); + ret = gs_cosd_txn_close(1, (version+3)); + if(ret != 0) + { + printf("Error closing txn 1: %d\n", ret); + return 1; + } + printf("DONE\n"); + + + ret = gs_cosd_dump(); assert(ret == 0); hooks/post-receive -- Grayskull Repository

1 0

Grayskull Repository branch, master, updated. git-migration-278-g0fd9f3d
by noreply＠mcs.anl.gov 15 Dec '09

15 Dec '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 0fd9f3de3dfa793453f556f5951762a3348e1026 (commit) via ec3fffe6805803edc2c39c1e52b413b2944bded8 (commit) from cb8be4c05c5cc874e93f6c4d8daf50c3d6b2eb81 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 0fd9f3de3dfa793453f556f5951762a3348e1026 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 11:47:38 2009 -0500 whitespace cleanups commit ec3fffe6805803edc2c39c1e52b413b2944bded8 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Dec 15 11:44:30 2009 -0500 cosd api function to retrieve current version ----------------------------------------------------------------------- Summary of changes: .../gsl/resources/cosd-prototype/cosd-prototype.c | 146 ++++++++++++++++---- .../resources/cosd-prototype/cosd-prototype.gsh | 9 ++ .../src/gsl/resources/cosd-prototype/test/cosd1.gs | 20 ++- 3 files changed, 145 insertions(+), 30 deletions(-) Diff of changes: diff --git a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c index 8e9b9cd..b501646 100644 --- a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c +++ b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c @@ -22,14 +22,14 @@ * - use txn_checkpoint to flush when coalescing * - consider rmw locks where appropriate * - long term: consider fallocate hint for logs to keep them linear - * - try computing cksum along the way, store in db + * - try computing cksum along the way, store in db * - either in log map itself or db that matches its keys */ /**************************************************************/ /* TODO: this needs to be in a header somewhere */ #ifndef llu -#if SIZEOF_LONG_INT == 4 +#if SIZEOF_LONG_INT == 4 # define llu(x) (x) # define lld(x) (x) # define SCANF_lld "%lld" @@ -97,7 +97,7 @@ static int gs_cosd_resource_id; static enum progress_mode gs_cosd_progress_mode = GS_PROG_NONE; static int merge_logical_map(struct gs_list_link* list1_in, int - list1_count, struct gs_list_link* list2_in, int list2_count, + list1_count, struct gs_list_link* list2_in, int list2_count, struct gs_list_link* list_out, void** free_ptr); static int compare_log_map_key(DB * dbp, const DBT * a, const DBT * b); static int compare_missing_version(DB * dbp, const DBT * a, const DBT * b); @@ -170,7 +170,10 @@ struct cosd_op uint64_t oid; uint64_t txn_number; } txn_close; - + struct get_version_op{ + uint64_t oid; + uint64_t *version; + } get_version; #if 0 struct remove_op{ uint64_t oid; @@ -364,7 +367,7 @@ static int gs_cosd_poll(gs_context_t context, int millisecs) c_op = gs_op_entry(gop, struct cosd_op, op); - ret = gop->op_worker(gop); + ret = gop->op_worker(gop); if(ret == 1) { /* done */ @@ -391,7 +394,7 @@ static int gs_cosd_cancel(gs_context_t gs_ctx, gs_op_id_t op_id) gs_mutex_lock(&cosd_mutex); struct gs_op *op; struct cosd_op *c_op; - + cache_id = gs_id_lookup(op_id, &resource_id); assert(resource_id == gs_cosd_resource_id); @@ -401,7 +404,7 @@ static int gs_cosd_cancel(gs_context_t gs_ctx, gs_op_id_t op_id) gs_oplist_del(op, &cosd_oplist); gs_opcache_put(cosd_opcache, op); #endif - + gs_mutex_unlock(&cosd_mutex); return 0; @@ -600,7 +603,7 @@ int gs_cosd_init(enum progress_mode mode, const char* path) gs_cosd_resource_id = gs_resource_register(&gs_cosd_resource); if(gs_cosd_resource_id < 0) { - gs_opcache_destroy(cosd_opcache); + gs_opcache_destroy(cosd_opcache); return(gs_cosd_resource_id); } gs_cosd_progress_mode = mode; @@ -662,7 +665,7 @@ static int write_op_worker(struct gs_op* op) /* open log file */ sprintf(log_name, "%s/%llu.dat", cosd_log_path, llu(c_op->u.write.oid)); - ret = open(log_name, O_RDWR|O_DIRECT|O_EXCL|O_NOATIME, + ret = open(log_name, O_RDWR|O_DIRECT|O_EXCL|O_NOATIME, S_IRUSR|S_IWUSR); if(ret < 0) { @@ -720,7 +723,7 @@ static int write_op_worker(struct gs_op* op) /* calculate aligned buffer size */ if(tmp_update->size % DIRECT_ALIGN != 0) - aligned_size = + aligned_size = ((tmp_update->size / DIRECT_ALIGN)+1)*DIRECT_ALIGN; else aligned_size = tmp_update->size; @@ -799,7 +802,7 @@ static int write_op_worker(struct gs_op* op) c_op->error_code = -errno; return(1); } - + /* track this update in the txn accumulator */ gs_mutex_lock(&txn_mutex); hash_link = gs_hash_search(txn_table, &c_op->u.write.txn_number); @@ -810,7 +813,7 @@ static int write_op_worker(struct gs_op* op) gs_mutex_unlock(&txn_mutex); return(1); } - txn_acc = gs_hash_get_entry(hash_link, struct txn_accumulator, + txn_acc = gs_hash_get_entry(hash_link, struct txn_accumulator, hash_link); tmp_update->version = txn_acc->txn_number; gs_list_add(&tmp_update->list_link, &txn_acc->update_list); @@ -919,7 +922,7 @@ static int dump_op_worker(struct gs_op* op) { printf("logical_off: %lld, logical_end: %lld, size: %lld, log_off: %lld, ver: %llu\n", lld(lmv.logical_offset), lld(lmv.logical_offset_end), - lld(lmv.size), lld(lmv.log_offset), llu(lmv.version)); + lld(lmv.size), lld(lmv.log_offset), llu(lmv.version)); c_get_flag = DB_NEXT; } assert(ret == DB_NOTFOUND); /* TODO: err handling */ @@ -944,7 +947,7 @@ static int dump_op_worker(struct gs_op* op) return(1); } - + /* create_op_worker() * * creates an object @@ -968,7 +971,7 @@ static int create_op_worker(struct gs_op* op) /* create a log file */ sprintf(log_name, "%s/%llu.dat", cosd_log_path, llu(c_op->u.create.requested_oid)); - ret = open(log_name, O_RDWR|O_CREAT|O_DIRECT|O_EXCL|O_NOATIME, + ret = open(log_name, O_RDWR|O_CREAT|O_DIRECT|O_EXCL|O_NOATIME, S_IRUSR|S_IWUSR); if(ret < 0) { @@ -997,7 +1000,7 @@ static int create_op_worker(struct gs_op* op) /* TODO: error handling */ assert(0); } - + /* set initial log offset of 0 */ val = 0; ret = log_offset_dbp->put(log_offset_dbp, txn, &key, &value, 0); @@ -1006,7 +1009,7 @@ static int create_op_worker(struct gs_op* op) /* TODO: error handling */ assert(0); } - + ret = txn->commit(txn, 0); if(ret != 0) { @@ -1169,6 +1172,99 @@ int gs_cosd_txn_open(uint64_t oid, uint64_t txn_number) return(0); } + +/* get_version_op_worker() + * + * retrieves the current version number of an object + */ +static int get_version_op_worker(struct gs_op* op) +{ + struct cosd_op *c_op; + int ret; + DB_TXN *txn; + uint64_t version; + DBT key, value; + + c_op = gs_op_entry(op, struct cosd_op, op); + assert(c_op); + + /* only support one object for now */ + assert(c_op->u.get_version.oid == 1); + + /* do db stuff */ + ret = envp->txn_begin(envp, NULL, &txn, 0); + if(ret != 0) + { + /* TODO: error handling */ + c_op->error_code = -1; + return(1); + } + + /* read current version */ + COSD_INIT_DBT(key, c_op->u.get_version.oid); + COSD_INIT_DBT(value, version); + + ret = ver_dbp->get(ver_dbp, txn, &key, &value, 0); + if(ret != 0) + { + /* TODO: error handling */ + assert(0); + } + + *c_op->u.get_version.version = version; + + ret = txn->commit(txn, 0); + if(ret != 0) + { + /* TODO: error handling */ + assert(0); + } + + c_op->error_code = 0; + + return(1); +} + + +gs_ret_t gs_cosd_get_version_post( + uint64_t oid, + uint64_t* version, + void (*callback)(void *ptr, int ret), + void *user_ptr, + gs_hints_t hints, + gs_context_t ctx, + gs_op_id_t *op_id) +{ + struct gs_op *op; + struct cosd_op *c_op; + + op = gs_opcache_get(cosd_opcache); + gs_op_fill(op, callback, user_ptr, hints, ctx); + + c_op = gs_op_entry(op, struct cosd_op, op); + c_op->op_id = gs_id_gen(gs_cosd_resource_id, (uint64_t)(op->cache_id)); + c_op->u.get_version.oid = oid; + c_op->u.get_version.version = version; + + *op_id = c_op->op_id; + + /* TODO: put this in the fill function if we keep it? */ + op->op_worker = get_version_op_worker; + + gs_cosd_launch_op(op, gs_cosd_progress_mode); + + return 0; +} +gs_ret_t (* gs_cosd_get_version)( + uint64_t oid, + uint64_t* version, + void (*callback)(void *ptr, int ret), + void *user_ptr, + gs_hints_t hints, + gs_context_t ctx, + gs_op_id_t *op_id) = gs_cosd_get_version_post; + + /* txn_close_op_worker() * * closes a transaction @@ -1217,7 +1313,7 @@ static int txn_close_op_worker(struct gs_op* op) gs_mutex_unlock(&txn_mutex); return(1); } - txn_acc = gs_hash_get_entry(hash_link, struct txn_accumulator, + txn_acc = gs_hash_get_entry(hash_link, struct txn_accumulator, hash_link); gs_list_del(&txn_acc->hash_link); gs_mutex_unlock(&txn_mutex); @@ -1251,7 +1347,7 @@ static int txn_close_op_worker(struct gs_op* op) { tmp_update = gs_list_get_entry(iterator, struct logical_map_entry, list_link); - + lmk.logical_offset_end = tmp_update->logical_offset; done = 0; while(!done) @@ -1339,7 +1435,7 @@ static int txn_close_op_worker(struct gs_op* op) /* TODO: error handling */ assert(0); } - + if(txn_acc->txn_number > version) { /* this update is newer than the current version of the object */ @@ -1456,7 +1552,7 @@ gs_ret_t (* gs_cosd_txn_close)( /* merge_logical_map() - * + * * merges two logical maps into one map. The two input maps are destroyed * by this function. The output map is allocated from scratch and can be * freed via the free_ptr argument. @@ -1464,7 +1560,7 @@ gs_ret_t (* gs_cosd_txn_close)( * returns 0 on success, -errno on failure */ static int merge_logical_map(struct gs_list_link* list1_in, int - list1_count, struct gs_list_link* list2_in, int list2_count, + list1_count, struct gs_list_link* list2_in, int list2_count, struct gs_list_link* list_out, void** free_ptr) { struct gs_list_link* iterator; @@ -1541,17 +1637,17 @@ static int merge_logical_map(struct gs_list_link* list1_in, int { /* consume any elements that have been entirely passed over */ - if(i < list1_count && list1_array[i].logical_offset_end <= logical_offset) + if(i < list1_count && list1_array[i].logical_offset_end <= logical_offset) { i++; continue; } - if(j < list2_count && list2_array[j].logical_offset_end <= logical_offset) + if(j < list2_count && list2_array[j].logical_offset_end <= logical_offset) { j++; continue; } - + /* shave off the beginning of any extent that has been partially * passed over */ diff --git a/code/src/gsl/resources/cosd-prototype/cosd-prototype.gsh b/code/src/gsl/resources/cosd-prototype/cosd-prototype.gsh index 212780c..3943c3f 100644 --- a/code/src/gsl/resources/cosd-prototype/cosd-prototype.gsh +++ b/code/src/gsl/resources/cosd-prototype/cosd-prototype.gsh @@ -54,6 +54,15 @@ __blocking int gs_cosd_create( ); /** + * Retrieves current version number of object + * \return 0 on success, -errno on failure + */ +__blocking int gs_cosd_get_version( + uint64_t oid, /**< object to operate on */ + uint64_t* version /**< version number of object */ +); + +/** * Opens a txn * \return 0 on success, -errno on failure */ diff --git a/code/src/gsl/resources/cosd-prototype/test/cosd1.gs b/code/src/gsl/resources/cosd-prototype/test/cosd1.gs index f4e3153..288708a 100644 --- a/code/src/gsl/resources/cosd-prototype/test/cosd1.gs +++ b/code/src/gsl/resources/cosd-prototype/test/cosd1.gs @@ -29,8 +29,9 @@ static __blocking int do_cosd_test(void) int64_t obj_sz = 1024; int64_t obj_offset = 0; uint64_t out_oid1; + uint64_t version; - printf("Creating oid 1...\n"); + printf("Creating oid 1 (if it doesn't already exist)...\n"); ret = gs_cosd_create(1, &out_oid1); if(ret != 0 && ret != -EEXIST) { @@ -39,8 +40,17 @@ static __blocking int do_cosd_test(void) } printf("DONE\n"); - printf("opening txn 1...\n"); - ret = gs_cosd_txn_open(1, 1); + printf("Getting version number of oid 1...\n"); + ret = gs_cosd_get_version(1, &version); + if(ret != 0) + { + printf("Error getting version number for oid=1: %d\n", ret); + return(1); + } + printf("DONE: %llu\n", llu(version)); + + printf("opening txn...\n"); + ret = gs_cosd_txn_open(1, (version+1)); if(ret != 0) { printf("Error opening txn 1: %d\n", ret); @@ -49,7 +59,7 @@ static __blocking int do_cosd_test(void) printf("DONE\n"); printf("writing 0-1024...\n"); - ret = gs_cosd_write(1, 1, &buffer_offset, &buffer_sz, 1, &obj_offset, + ret = gs_cosd_write(1, (version+1), &buffer_offset, &buffer_sz, 1, &obj_offset, &obj_sz, 1); if(ret != 0) { @@ -59,7 +69,7 @@ static __blocking int do_cosd_test(void) printf("DONE\n"); printf("closing txn...\n"); - ret = gs_cosd_txn_close(1, 1); + ret = gs_cosd_txn_close(1, (version+1)); if(ret != 0) { printf("Error closing txn 1: %d\n", ret); hooks/post-receive -- Grayskull Repository

1 0

Grayskull Repository branch, master, updated. git-migration-276-gcb8be4c
by noreply＠mcs.anl.gov 15 Dec '09

15 Dec '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via cb8be4c05c5cc874e93f6c4d8daf50c3d6b2eb81 (commit) via 6ec15cb1ce34c5952bc39fb7b41ab26a7c317229 (commit) via 2a327ff8bc81a495abadedfc08e1c92f2913cc6c (commit) via 4170f5d9c860e8e9b4efc07616f549a8ad567933 (commit) via 371eda30e123a069f378dc503ad84a98e919899a (commit) via 09ddd17baf13e5bcdbb5d4f74af170eb64198b72 (commit) from 39e095cdbdb876901ecb231d972127769a53e738 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit cb8be4c05c5cc874e93f6c4d8daf50c3d6b2eb81 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Dec 15 10:32:33 2009 -0600 First decent dl01-small plot. commit 6ec15cb1ce34c5952bc39fb7b41ab26a7c317229 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Thu Dec 10 22:55:05 2009 -0600 EOLs. commit 2a327ff8bc81a495abadedfc08e1c92f2913cc6c Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Thu Dec 10 22:53:13 2009 -0600 Remove unnecessary copies. commit 4170f5d9c860e8e9b4efc07616f549a8ad567933 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Thu Dec 10 22:35:05 2009 -0600 EOL fixes and minor bug fix to Rebuild.completeRepair(). commit 371eda30e123a069f378dc503ad84a98e919899a Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Thu Dec 10 22:29:24 2009 -0600 Moved more Rebuild control structures into Simulator. commit 09ddd17baf13e5bcdbb5d4f74af170eb64198b72 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Thu Dec 10 15:54:53 2009 -0600 Major refactoring. Addition of Sites class corrects a design flaw. ----------------------------------------------------------------------- Summary of changes: sim/gobs/build.xml | 8 + sim/gobs/gobs/sim/Copy.java | 8 +- sim/gobs/gobs/sim/Event.java | 6 +- sim/gobs/gobs/sim/Fault.java | 5 - sim/gobs/gobs/sim/File.java | 6 +- sim/gobs/gobs/sim/{Object.java => Gob.java} | 8 +- sim/gobs/gobs/sim/KClosest.java | 35 ++- sim/gobs/gobs/sim/Metric.java | 101 ++++--- sim/gobs/gobs/sim/Node.java | 49 ++-- sim/gobs/gobs/sim/ObjectStrip.java | 5 +- sim/gobs/gobs/sim/ParallelRUSH.java | 54 ++-- sim/gobs/gobs/sim/Placement.java | 110 +++++++- .../gobs/sim/{Primary.java => PrimaryObject.java} | 15 +- sim/gobs/gobs/sim/RUSHp.java | 29 ++- sim/gobs/gobs/sim/Rebuild.java | 281 ++++++++++++------- sim/gobs/gobs/sim/Repair.java | 5 - sim/gobs/gobs/sim/Replicated.java | 77 ++---- .../sim/{Secondary.java => SecondaryObject.java} | 14 +- sim/gobs/gobs/sim/SecondaryStrip.java | 6 + sim/gobs/gobs/sim/Simulator.java | 167 +++++++++---- sim/gobs/gobs/sim/SingleObject.java | 15 +- sim/gobs/gobs/sim/Sites.java | 133 +++++++++ sim/gobs/gobs/sim/VertexGroups.java | 22 +- sim/gobs/gobs/sweep/AbstractSweep.java | 44 +++- sim/gobs/gobs/sweep/Sweep.java | 1 + sim/gobs/gobs/test/TestELHS.java | 6 +- sim/gobs/gobs/util/BoundedSortedValues.java | 15 + sim/gobs/gobs/util/Tools.java | 1 + sim/gobs/scripts/eol.all.zsh | 12 + sim/gobs/scripts/eol.zsh | 6 + sim/gobs/sweeps/dl01-small.cfg | 11 +- sim/gobs/sweeps/rb01-small.cfg | 1 + 32 files changed, 872 insertions(+), 384 deletions(-) copy sim/gobs/gobs/sim/{Object.java => Gob.java} (69%) rename sim/gobs/gobs/sim/{Primary.java => PrimaryObject.java} (60%) rename sim/gobs/gobs/sim/{Secondary.java => SecondaryObject.java} (58%) create mode 100644 sim/gobs/gobs/sim/Sites.java create mode 100755 sim/gobs/scripts/eol.all.zsh Diff of changes: diff --git a/sim/gobs/build.xml b/sim/gobs/build.xml index 4fc4316..1599575 100644 --- a/sim/gobs/build.xml +++ b/sim/gobs/build.xml @@ -109,6 +109,14 @@ <delete dir="docs" /> </target> + <target name="clean_sim"> + <delete> + <fileset dir="gobs/sim"> + <filename name="*.class" /> + </fileset> + </delete> + </target> + <target name="clean"> <delete> <fileset dir="."> diff --git a/sim/gobs/gobs/sim/Copy.java b/sim/gobs/gobs/sim/Copy.java index 5a9c34f..c86ae17 100644 --- a/sim/gobs/gobs/sim/Copy.java +++ b/sim/gobs/gobs/sim/Copy.java @@ -9,7 +9,7 @@ import gobs.util.*; class Copy extends Event { - gobs.sim.Object object; + Gob object; Node source; Node destination; boolean complete = false; @@ -20,7 +20,7 @@ class Copy static DecimalFormat df = new DecimalFormat("0.00"); Copy(Simulator simulator, - gobs.sim.Object object, Node source, Node destination) + Gob object, Node source, Node destination) { super(simulator); assert (source != null); @@ -37,12 +37,12 @@ class Copy this.start = start; finish = start + object.size/Rebuild.rate; progress = Progress.RUNNING; - if (Simulator.current.bool("print.rebuilds")) + if (simulator.bool("print.rebuilds")) System.out.println("start: " + log()); } else if (progress == Progress.RUNNING) { - if (Simulator.current.bool("print.rebuilds")) + if (simulator.bool("print.rebuilds")) System.out.println("restart: " + log()); } else if (progress == Progress.DONE) diff --git a/sim/gobs/gobs/sim/Event.java b/sim/gobs/gobs/sim/Event.java index 3cc5f3c..7592768 100644 --- a/sim/gobs/gobs/sim/Event.java +++ b/sim/gobs/gobs/sim/Event.java @@ -32,8 +32,10 @@ class Event public int compareTo(Event event) { - return - new Double(finish).compareTo(new Double(event.finish)); + int d = new Double(finish).compareTo(new Double(event.finish)); + if (d == 0) + return toString().compareTo(event.toString()); + return d; } String log() diff --git a/sim/gobs/gobs/sim/Fault.java b/sim/gobs/gobs/sim/Fault.java index 669accd..37afae0 100644 --- a/sim/gobs/gobs/sim/Fault.java +++ b/sim/gobs/gobs/sim/Fault.java @@ -11,11 +11,6 @@ class Fault simulator.mttfExponential.nextDouble(); } - public int compareTo(Event event) - { - return new Double(finish).compareTo(new Double(event.finish)); - } - String log() { return "Fault(" + df.format(finish) + ")"; diff --git a/sim/gobs/gobs/sim/File.java b/sim/gobs/gobs/sim/File.java index 59d22c2..32b6bf8 100644 --- a/sim/gobs/gobs/sim/File.java +++ b/sim/gobs/gobs/sim/File.java @@ -19,7 +19,7 @@ public class File static int unique = 0; - Collection<gobs.sim.Object> objects; + Collection<Gob> objects; File(long size, int strip, int width, int replicas) { @@ -31,7 +31,7 @@ public class File id = unique++; } - void link(Collection<gobs.sim.Object> objects) + void link(Collection<Gob> objects) { this.objects = objects; } @@ -54,7 +54,7 @@ public class File StringBuilder sb = new StringBuilder(100+objects.size()*10); sb.append("file[").append(id).append("]: "); - for (gobs.sim.Object object : objects) + for (Gob object : objects) sb.append(object.toString(simulator)).append(" "); return sb.toString(); } diff --git a/sim/gobs/gobs/sim/Object.java b/sim/gobs/gobs/sim/Gob.java similarity index 69% copy from sim/gobs/gobs/sim/Object.java copy to sim/gobs/gobs/sim/Gob.java index 99632dd..f30bf17 100644 --- a/sim/gobs/gobs/sim/Object.java +++ b/sim/gobs/gobs/sim/Gob.java @@ -4,10 +4,12 @@ package gobs.sim; import java.math.BigInteger; /** - * Tags objects. + * Simulated Grayskull object. + * + * (Identifier "Object" is used by Java.) * */ -public abstract class Object +public abstract class Gob extends Addressable { /** @@ -20,7 +22,7 @@ public abstract class Object */ int replicas; - Object(BigInteger id, int size, int replicas) + Gob(BigInteger id, int size, int replicas) { super(id); this.size = size; diff --git a/sim/gobs/gobs/sim/KClosest.java b/sim/gobs/gobs/sim/KClosest.java index 59cd7d7..6272b61 100644 --- a/sim/gobs/gobs/sim/KClosest.java +++ b/sim/gobs/gobs/sim/KClosest.java @@ -2,7 +2,7 @@ package gobs.sim; import java.math.BigInteger; -import java.util.List; +import java.util.*; import gobs.util.BoundedSortedValues; @@ -22,18 +22,43 @@ abstract class KClosest /** Obtain a list of nodes that may host the given object with respect to the replica placement scheme used here. + Ordered from farthest to closest, thus the last, closest + site is the primary. */ - List<Node> sites(gobs.sim.Object object, List<Node> nodes) + Sites sites(Gob object, Collection<Node> nodes) { - BoundedSortedValues<Node,BigInteger> sites = + Sites result = new Sites(object); + + BoundedSortedValues<Node,BigInteger> sorter = new BoundedSortedValues<Node,BigInteger>(object.replicas); for (Node node : nodes) { BigInteger d = distance(object.id, node.id); - sites.add(node, d); + sorter.add(node, d); + } + + List<Node> list = sorter.get(); + + boolean first = true; + for (Iterator<Node> it = list.iterator(); it.hasNext(); ) + { + Node node = it.next(); + if (first) + { + result.last(node); + first = false; + } + else if (it.hasNext()) + { + result.secondary(node); + } + else + { + result.primary(node); + } } - return sites.get(); + return result; } } diff --git a/sim/gobs/gobs/sim/Metric.java b/sim/gobs/gobs/sim/Metric.java index cbec1ae..7908c51 100644 --- a/sim/gobs/gobs/sim/Metric.java +++ b/sim/gobs/gobs/sim/Metric.java @@ -2,7 +2,7 @@ package gobs.sim; import java.math.BigInteger; -import java.util.List; +import java.util.*; import gobs.util.Tools; @@ -24,7 +24,12 @@ public abstract class Metric /** Find the closest Node to this id. */ - public Node lookup(BigInteger id, List<Node> nodes) + public Node lookup(BigInteger id, Sites sites) + { + return lookup(id, sites.get()); + } + + public Node lookup(BigInteger id, Collection<Node> nodes) { Node result = null; BigInteger diff = null; @@ -42,22 +47,29 @@ public abstract class Metric /** Return the Copy that is prescribed for this object. - Removes the Copy.source from nodes. May return null. */ - Copy regimen(SingleObject object, List<Node> nodes) + Copy regimen(SingleObject object, List<Node> system) { System.out.println("regimen: " + object); - List<Node> sites = sites(object, nodes); - Secondary secondary; + Sites sites = sites(object, system); + SecondaryObject secondary; // The node that should have the primary in the current system - Node primaryNode = lookup(object.id, nodes); + Node primaryNode = sites.primary(); // The node that currently holds the primary - Node primaryLocation = locatePrimary(object, sites); + Node primaryLocation = locatePrimaryObject(object, sites); Node source = removeSource(object, sites); if (source == null) { + System.out.println("no source: " + object); + System.out.println("sites: " + sites); + source = simulator.scanObject(object); + System.out.println("source: " + source); + } + + if (source == null) + { simulator.lost(object); return null; } @@ -73,20 +85,38 @@ public abstract class Metric /** Locate the Primary node if it holds the Primary. Else returns null. + Assumes sites is sorted and that the Primary is last. */ - Node locatePrimary(SingleObject object, List<Node> sites) + Node locatePrimaryObject(SingleObject object, Sites sites) { - int p = sites.size()-1; - if (p >= 0) - { - Node result = sites.get(p); - if (result.containsPrimary(object.id)) - return result; - } + Node result = sites.primary(); + if (result.containsPrimary(object.id)) + return result; return null; } /** + Return the primary node from sites, regardless of what objects + it holds. + Returns null if sites is empty. + */ + /* + Node getPrimaryNode(SingleObject object, Sites sites) + { + return sites.primary(); + } + */ + + Node primary(Sites nodes) + { + /*List<Node> sites = sites(object, nodes); + Node result = getPrimaryNode(object, sites); + return result; + */ + return nodes.primary(); + } + + /** Obtain the Copy. Returns null if there is no destination for the Object, which indicates that the Object is back to full replica count. @@ -94,29 +124,28 @@ public abstract class Metric in the current system @param primaryLocation The node that currently holds the primary @param source The node from which to make the Copy + @param sites The list of sites that should host the object */ Copy regimen(SingleObject object, Node primaryNode, Node primaryLocation, Node source, - List<Node> sites) + Sites sites) { Copy result; Node destination = null; SingleObject payload; - if (object instanceof Primary) + if (object instanceof PrimaryObject) { if (primaryNode.contains(object.id)) { - // Replace the Secondary with a Primary if (! primaryNode.containsPrimary(object.id)) { - primaryNode.remove(object.id); - primaryNode.add(object); - payload = object.toSecondary(); + promote(primaryNode, object); + payload = object.toSecondaryObject(); } - if (! (object instanceof Secondary)) + if (! (object instanceof SecondaryObject)) { - object = object.toSecondary(); + object = object.toSecondaryObject(); } payload = object; destination = locateDestination(object, sites); @@ -131,14 +160,15 @@ public abstract class Metric { if (primaryLocation != null) { - Primary primary = (Primary) primaryLocation.get(object.id); - payload = object.toSecondary(primary); + PrimaryObject primary = + (PrimaryObject) primaryLocation.get(object.id); + payload = object.toSecondaryObject(primary); destination = locateDestination(object, sites); } else { - destination = getPrimaryNode(object, sites); - payload = object.toPrimary(); + destination = sites.primary(); + payload = object.toPrimaryObject(); } } @@ -155,21 +185,6 @@ public abstract class Metric return new Copy(simulator, payload, source, destination); } - /** - Return the primary node from sites, regardless of what objects - it holds. - This is the last site in the list, by convention. - Returns null if sites is empty. - */ - Node getPrimaryNode(SingleObject object, List<Node> sites) - { - int p = sites.size()-1; - if (p >= 0) - return sites.get(p); - else - return null; - } - void logCopy(Copy copy) { if (copy == null) diff --git a/sim/gobs/gobs/sim/Node.java b/sim/gobs/gobs/sim/Node.java index 67c974c..228e3c7 100644 --- a/sim/gobs/gobs/sim/Node.java +++ b/sim/gobs/gobs/sim/Node.java @@ -18,7 +18,7 @@ public class Node /** Object store. */ - public Map<BigInteger,gobs.sim.Object> objects; + public Map<BigInteger,Gob> objects; /** Counter for number of pushes during rebuilds. @@ -41,11 +41,12 @@ public class Node { super(id); this.simulator = simulator; - objects = new HashMap<BigInteger,gobs.sim.Object>(); + objects = new HashMap<BigInteger,Gob>(); } - void add(gobs.sim.Object object) + void add(Gob object) { + assert(object != null); Tools.check(! objects.containsKey(object.id), "Node.add(): " + id + " contains " + object.toString(Simulator.current) + "\n" + @@ -55,37 +56,38 @@ public class Node } /** - @return An gobs.sim.Object with this id. + @return An Gob with this id. */ - gobs.sim.Object get(BigInteger id) + Gob get(BigInteger id) { return objects.get(id); } /** - Remove and return a gobs.sim.Object with this id. + Remove and return a Gob with this id. */ - gobs.sim.Object remove(BigInteger id) + Gob remove(BigInteger id) { return objects.remove(id); } /** - Delete the Object. + Delete the Object asserting it exists. Like {@link #remove}. */ void delete(BigInteger id) { - System.out.println("deleting: from node: " + this + - " object: " + id); - Object o = objects.remove(id); - assert(o != null); + // System.out.println("deleting: from node: " + this + + // " object: " + id); + Gob object = remove(id); + Tools.check(object != null, "Could not delete: " + + "from node: " + this.id + "object: " + id); } /** Return a Collection of all of this Node's objects. */ - public Collection<gobs.sim.Object> collection() + public Collection<Gob> collection() { return objects.values(); } @@ -112,13 +114,22 @@ public class Node Set<Copy> copies = new LinkedHashSet<Copy>(); for (Node other : simulator.nodes) { - Collection<Object> objects = other.objects.values(); - for (gobs.sim.Object object : objects) + if (other.equals(this)) + continue; + + Collection<Gob> objects = other.objects.values(); + for (Gob object : objects) { - List<Node> sites = simulator.placer.sites(object); + Sites sites = simulator.placer.sites(object); if (sites.contains(this) && ids.add(object.id)) { Node source = simulator.placer.removeSource(object, sites); + if (source == null) + { + System.out.println("no source: " + object); + System.out.println("sites: " + sites); + source = simulator.scanObject(object); + } Copy copy = new Copy(simulator, object, source, this); System.out.println("rejoin: " + copy); copies.add(copy); @@ -150,10 +161,10 @@ public class Node */ boolean containsPrimary(BigInteger id) { - gobs.sim.Object object = get(id); + Gob object = get(id); if (object == null) return false; - if (object instanceof Primary || + if (object instanceof PrimaryObject || object instanceof PrimaryStrip) return true; return false; @@ -176,7 +187,7 @@ public class Node result.append(":\t"); for (BigInteger i : objects.keySet()) { - gobs.sim.Object object = objects.get(i); + Gob object = objects.get(i); result.append(object.toString(simulator)).append(" "); } result.append("\n"); diff --git a/sim/gobs/gobs/sim/ObjectStrip.java b/sim/gobs/gobs/sim/ObjectStrip.java index 25de536..ab1d56a 100644 --- a/sim/gobs/gobs/sim/ObjectStrip.java +++ b/sim/gobs/gobs/sim/ObjectStrip.java @@ -5,11 +5,10 @@ import java.math.BigInteger; /** * Simulates a part of striped object. - * Identifier "Object" is used by Java. * */ class ObjectStrip - extends Object + extends Gob { /** The strip number. A striped object with strip width n @@ -31,7 +30,7 @@ class ObjectStrip SecondaryStrip toSecondaryStrip(PrimaryStrip primary) { - // return new Secondary(B, id, size, replicas, primary); + // assert(primary instanceof PrimaryStrip); return new SecondaryStrip(id, number, size, replicas, primary); } diff --git a/sim/gobs/gobs/sim/ParallelRUSH.java b/sim/gobs/gobs/sim/ParallelRUSH.java index 3e5cbbb..53ca21f 100644 --- a/sim/gobs/gobs/sim/ParallelRUSH.java +++ b/sim/gobs/gobs/sim/ParallelRUSH.java @@ -36,11 +36,11 @@ public class ParallelRUSH /** Generate an object set for a file. */ - public Collection<gobs.sim.Object> + public Collection<Gob> allocateFile(File file, List<Node> nodes) { - ArrayList<gobs.sim.Object> result = - new ArrayList<gobs.sim.Object>(file.width); + ArrayList<Gob> result = + new ArrayList<Gob>(file.width); BigInteger id = randomID(); for (int i = 0; i < file.width; i++) @@ -48,7 +48,7 @@ public class ParallelRUSH int bytes = file.objectSize(i); ObjectStrip obj = new ObjectStrip(id, i, bytes, file.replicas); - result.add((gobs.sim.Object) obj); + result.add((Gob) obj); } result.trimToSize(); @@ -58,24 +58,25 @@ public class ParallelRUSH void place(ObjectStrip object, List<Node> nodes) { // System.out.println("place: " + object); - List<Node> sites = sites(object, nodes); + Sites sites = sites(object, nodes); simulator.stats.flexibilities.add(sites.size()); PrimaryStrip primary = object.toPrimaryStrip(); - Node node = sites.remove(sites.size()-1); + Node node = sites.removePrimary(); node.add(primary); for (int i = object.replicas-1; i >= 0; i--) { if (sites.size() == 0) { - System.out.println("Could not place all replicas!"); + System.out.println("ParallelRUSH.place(): " + + "Could not place all replicas!"); return; } - node = sites.remove(i); + node = sites.remove(); node.add(object.toSecondaryStrip(primary)); } } - List<Node> sites(gobs.sim.Object object, List<Node> nodes) + Sites sites(Gob object, Collection<Node> nodes) { if (object instanceof ObjectStrip) return sites((ObjectStrip) object, nodes); @@ -85,31 +86,31 @@ public class ParallelRUSH "object must be an ObjectStrip."); } - List<Node> sites(gobs.sim.ObjectStrip object, List<Node> nodes) + Sites sites(ObjectStrip object, Collection<Node> nodes) { - List<Node> result = new ArrayList<Node>(object.replicas); + Sites result = new Sites(object); - for (int r = object.replicas-1; r >= 1; r--) + Node last = lookup(object.id, object.number, + object.replicas-1, object.replicas); + if (nodes.contains(last)) + result.last(last); + + for (int r = object.replicas-2; r >= 1; r--) { Node node = - lookup(object.id, object.number, r, object.replicas, nodes); - result.add(node); + lookup(object.id, object.number, r, object.replicas); + if (nodes.contains(node)) + result.secondary(node); } - Node node = - lookup(object.id, object.number, 0, object.replicas, nodes); + Node primary = + lookup(object.id, object.number, 0, object.replicas); + if (nodes.contains(primary)) + result.primary(primary); - result.add(node); return result; } - /* - List<Node> sites(SingleObject object, List<Node> nodes) - { - return null; - } - */ - /** <a href="http://wiki.mcs.anl.gov/radix/grayskull/index.php/Stripe_placement_and_rebu…">From the wiki.</a> @@ -117,11 +118,10 @@ public class ParallelRUSH @param i The strip number @param r The replica number (in [0,x.replicas)). @param R The number of replicas for x (x.replicas). - @param nodes The set of nodes. */ - public Node lookup(BigInteger x, int i, int r, int R, - List<Node> nodes) + public Node lookup(BigInteger x, int i, int r, int R) { + List<Node> nodes = simulator.nodes; int j = 0; rng.setSeed(x.longValue()); advance(j); diff --git a/sim/gobs/gobs/sim/Placement.java b/sim/gobs/gobs/sim/Placement.java index a9f6e02..45e15f3 100644 --- a/sim/gobs/gobs/sim/Placement.java +++ b/sim/gobs/gobs/sim/Placement.java @@ -27,14 +27,14 @@ public abstract class Placement /** Generate an object set for a file. */ - public abstract Collection<gobs.sim.Object> + public abstract Collection<Gob> allocateFile(File file, List<Node> nodes); /** Obtain a Copy that will repair the loss of this Object. @return The Copy or null if the Object is lost. */ - Copy regimen(gobs.sim.Object object, List<Node> nodes) + Copy regimen(Gob object, List<Node> nodes) { Copy result; if (object instanceof SingleObject) @@ -51,7 +51,7 @@ public abstract class Placement with respect to the current Simulator nodes. @return The Copy or null if the Object is lost. */ - Copy regimen(gobs.sim.Object object) + Copy regimen(Gob object) { return regimen(object, simulator.nodes); } @@ -82,10 +82,10 @@ public abstract class Placement /** Simply place each object on a node. */ - public void place(Collection<gobs.sim.Object> objects, + public void place(Collection<Gob> objects, List<Node> nodes) { - for (gobs.sim.Object object : objects) + for (Gob object : objects) place(object, nodes); } @@ -93,7 +93,7 @@ public abstract class Placement Initial placement: allocate replicas for object and place on nodes. */ - public abstract void place(gobs.sim.Object object, + public abstract void place(Gob object, List<Node> nodes); /** @@ -101,8 +101,10 @@ public abstract class Placement with respect to the replica placement scheme used here. Closest node should be last. + + @param nodes A subset of Simulator.nodes */ - abstract List<Node> sites(gobs.sim.Object object, List<Node> nodes); + abstract Sites sites(Gob object, Collection<Node> nodes); /** Obtain a list of nodes that may host the given object @@ -111,12 +113,19 @@ public abstract class Placement Closest node should be last. */ - List<Node> sites(gobs.sim.Object object) + Sites sites(Gob object) { return sites(object, simulator.nodes); } - Node removeSource(gobs.sim.Object object, List<Node> sites) + /* + Locate the Primary node for object with respect to nodes, + regardless of what objects it holds. + Returns null if nodes is empty. + */ + // abstract Node primary(Sites nodes); + + Node removeSource(Gob object, Sites sites) { if (object instanceof SingleObject) return removeSource((SingleObject) object, sites); @@ -128,18 +137,95 @@ public abstract class Placement in accordance with the policy; remove and return that replica from sites. */ - abstract Node removeSource(SingleObject object, List<Node> sites); + abstract Node removeSource(SingleObject object, Sites sites); - /** + /* Pull a random Node from one of the given sites. */ + /* Node random(List<Node> sites) { int index = Bits.nextInt(sites.size()); return sites.get(index); } + */ + + /** + If node contains a Secondary for object, + promote the object to a Primary and return it + (as a Gob). + Else return null. + */ + Gob promote(Node node, Gob object) + { + Gob primary = null; + Gob o = node.get(object.id); + assert(o != null); + if (o instanceof PrimaryObject || + o instanceof PrimaryStrip) + return null; + + if (o instanceof SecondaryObject) + { + SecondaryObject so = (SecondaryObject) o; + primary = so.toPrimaryObject(); + } + else if (o instanceof SecondaryStrip) + { + SecondaryStrip so = (SecondaryStrip) o; + primary = so.toPrimaryStrip(); + } + else + Tools.bail("Got unknown object!"); + + node.delete(o.id); + node.add(primary); + return primary; + } + + /** + If node contains a Primary for object, + demote the object to a Secondary pointing to the given + primary. + If the node contains a Secondary, ensure that it points + to the given primary. + @param primaryNode The Node that contains the Primary + */ + void demote(Node node, Gob object, Node primaryNode) + { + Gob p = primaryNode.get(object.id); + Gob o = node.get(object.id); + assert(p != null); + if (o instanceof SecondaryObject) + { + PrimaryObject primary = (PrimaryObject) p; + ((SecondaryObject) o).setPrimary(primary); + return; + } + if (o instanceof SecondaryStrip) + { + PrimaryStrip primary = (PrimaryStrip) p; + ((SecondaryStrip) o).setPrimary(primary); + return; + } + + Gob secondary = null; + if (o instanceof PrimaryObject) + { + PrimaryObject primary = (PrimaryObject) p; + secondary = ((PrimaryObject) o).toSecondaryObject(primary); + } + else if (o instanceof PrimaryStrip) + { + PrimaryStrip primary = (PrimaryStrip) p; + secondary = ((PrimaryStrip) o).toSecondaryStrip(primary); + } + + node.delete(o.id); + node.add(secondary); + } - void printRebuild(gobs.sim.Object object, + void printRebuild(Gob object, Node source, Node destination) { if (simulator.bool("print.rebuilds")) diff --git a/sim/gobs/gobs/sim/Primary.java b/sim/gobs/gobs/sim/PrimaryObject.java similarity index 60% rename from sim/gobs/gobs/sim/Primary.java rename to sim/gobs/gobs/sim/PrimaryObject.java index 15327b8..c2494d7 100644 --- a/sim/gobs/gobs/sim/Primary.java +++ b/sim/gobs/gobs/sim/PrimaryObject.java @@ -7,24 +7,17 @@ import java.math.BigInteger; * Simulates a single primary object. * */ -class Primary +class PrimaryObject extends SingleObject { - /* - Primary(int B, BigInteger id, int size, int replicas) - { - super(B, id, size, replicas); - } - */ - - Primary(BigInteger id, int size, int replicas) + PrimaryObject(BigInteger id, int size, int replicas) { super(id, size, replicas); } - Secondary toSecondary() + SecondaryObject toSecondaryObject() { - return toSecondary(this); + return toSecondaryObject(this); } public String toString(Simulator simulator) diff --git a/sim/gobs/gobs/sim/RUSHp.java b/sim/gobs/gobs/sim/RUSHp.java index a9ddc98..98563d8 100644 --- a/sim/gobs/gobs/sim/RUSHp.java +++ b/sim/gobs/gobs/sim/RUSHp.java @@ -34,7 +34,7 @@ public class RUSHp super(simulator, source); } - List<Node> sites(gobs.sim.Object object, List<Node> nodes) + Sites sites(Gob object, Collection<Node> nodes) { if (object instanceof SingleObject) return sites((SingleObject) object, nodes); @@ -44,23 +44,26 @@ public class RUSHp "object must be an SingleObject."); } - List<Node> sites(gobs.sim.SingleObject object, List<Node> nodes) + Sites sites(SingleObject object, Collection<Node> nodes) { - List<Node> result = new ArrayList<Node>(); - Set<Node> sites = new HashSet<Node>(); + Sites result = new Sites(object); - Node primary = lookup(object.id, 0, object.replicas, nodes); + Node last = lookup(object.id, object.replicas-1, object.replicas); + if (nodes.contains(last)) + result.last(last); - for (int i = 1; i <= object.replicas-1; i++) + for (int i = 1; i <= object.replicas-2; i++) { - Node node = lookup(object.id, i, object.replicas, nodes); + Node node = lookup(object.id, i, object.replicas); - if (node != primary) - sites.add(node); + if (nodes.contains(node)) + result.secondary(node); } - result.addAll(sites); - result.add(primary); + Node primary = lookup(object.id, 0, object.replicas); + if (nodes.contains(primary)) + result.primary(primary); + return result; } @@ -70,10 +73,10 @@ public class RUSHp @param x The object id (key). @param r The replica number (in [0,x.replicas)). @param R The number of replicas for x (x.replicas). - @param nodes The set of nodes. */ - public Node lookup(BigInteger x, int r, int R, List<Node> nodes) + public Node lookup(BigInteger x, int r, int R) { + List<Node> nodes = simulator.nodes; int j = 0; rng.setSeed(x.longValue()); advance(j); diff --git a/sim/gobs/gobs/sim/Rebuild.java b/sim/gobs/gobs/sim/Rebuild.java index 4c2a1df..0899bc4 100644 --- a/sim/gobs/gobs/sim/Rebuild.java +++ b/sim/gobs/gobs/sim/Rebuild.java @@ -14,6 +14,9 @@ import gobs.util.*; */ public class Rebuild { + /** + Historical schedule of Events. + */ NavigableSet<Event> schedule = null; /** @@ -41,20 +44,9 @@ public class Rebuild Simulator simulator; /** - The Set of Nodes that are currently making copies. - */ - Set<Node> busy; - - /** The Queue of work to do. */ - Queue<Copy> queue; - - /** - The Set of known Events that will finish in the future. - Contains inflight Copies, the next Fault, and any Repairs. - */ - NavigableSet<Event> inflight = new TreeSet<Event>(); + Queue<Copy> queue = new ArrayDeque<Copy>(); /** The fault that will occur after this Rebuild has started. @@ -65,104 +57,102 @@ public class Rebuild /** @param fault The fault that may interrupt this Rebuild. */ - Rebuild(Simulator simulator, Fault fault, Set<Repair> repairs) + Rebuild(Simulator simulator, Fault fault, Repair repair) { this.simulator = simulator; this.fault = fault; - inflight.add(fault); - inflight.addAll(repairs); + simulator.inflight.add(fault); + simulator.inflight.add(repair); } /** Actually perform the rebuild. - Moves prescribed Copies from copies to done. - TODO: Handle cases where more than one copy is needed. - @param time The time at which the copies start. - @param copies The Copies prescribed for this Rebuild - @param done Return the Copies completed here + Moves prescribed Copies from todo to inflight and schedule. + @param work List of new work to do @return The time at which the next fault occurs */ - double rebuild(double time, Set<Copy> copies, Set<Copy> done) + double rebuild(List<Copy> work) { - start = time; + start = simulator.time(); schedule = new TreeSet<Event>(); - busy = new HashSet<Node>(copies.size()*2); - queue = order(copies); - while (! (queue.isEmpty() && inflight.isEmpty())) + enqueue(simulator.todo); + add(work); + while (! (queue.isEmpty() && simulator.inflight.isEmpty())) { - Copy copy = select(queue, busy); + Copy copy = select(queue); if (copy != null) { - copy.startAt(time); - schedule.add(copy); - busy.add(copy.source); - busy.add(copy.destination); - while (! inflight.add(copy)) - { - time += selectTime; - copy.startAt(time); - } + startCopy(simulator.time(), copy); } else { // dumpInflight(inflight); - Event event = pop(); - try - { - if (event instanceof Copy) - { - completeCopy(event, copies, done); - } - else if (event instanceof Repair) - { - completeRepair(event); - } - else if (event instanceof Fault) - { - System.out.println("INTERRUPTION!"); - break; - } - } - finally - { - time = event.finish; - if (simulator.bool("print.rebuilds")) - System.out.println("finish: " + event.log()); - } + boolean fault = popEvent(); + if (fault) + break; } } - finish = time; - return fault.finish; + finish = simulator.time(); + return finish; } - double drain() + /** + Remove copy from todo and add to inflight. + */ + void startCopy(double time, Copy copy) { - return 0; + simulator.todo.remove(copy); + copy.startAt(time); + schedule.add(copy); + simulator.busy.add(copy.source); + simulator.busy.add(copy.destination); + Tools.check(simulator.inflight.add(copy), + "Could not add to inflight!"); + } + + boolean popEvent() + { + Event event = pop(); + simulator.advanceTo(event.finish); + if (simulator.bool("print.rebuilds")) + System.out.println("event: " + event.log()); + + if (event instanceof Copy) + { + completeCopy((Copy) event); + } + else if (event instanceof Repair) + { + completeRepair((Repair) event); + } + else if (event instanceof Fault) + { + if (simulator.bool("print.rebuilds")) + System.out.println("INTERRUPTION!"); + return true; + } + return false; } Event pop() { - Event result = inflight.pollFirst(); + Event result = simulator.inflight.pollFirst(); Tools.check(result != null, "inflight.pollFirst(): null"); return result; } /** - Handle this Copy Event. Move copy (event) from copies to done. + Handle this Copy Event. @param event Actually a Copy */ - void completeCopy(Event event, Set<Copy> copies, Set<Copy> done) + void completeCopy(Copy copy) { - Copy copy = (Copy) event; - busy.remove(copy.source); - busy.remove(copy.destination); - copies.remove(copy); + simulator.busy.remove(copy.source); + simulator.busy.remove(copy.destination); simulator.placer.copy(copy); - done.add(copy); List<Copy> extras = validate(copy.object); - for (Copy extra : extras) - copies.add(extra); + add(extras); } /** @@ -170,29 +160,49 @@ public class Rebuild Remove extraneous replicas or return a new Copy if needed. @return The Copies needed. */ - List<Copy> validate(gobs.sim.Object object) + List<Copy> validate(Gob object) { List<Copy> result = new ArrayList<Copy>(); int replicas = object.replicas; - List<Node> sites = simulator.placer.sites(object); - Set<Node> holders = simulator.scanStorage(object); + List<Node> stores = simulator.scanStorage(object); + Sites holders = simulator.placer.sites(object, stores); + Sites sites = simulator.placer.sites(object); + + ensurePrimary(object, holders); - Copy replicaCopy = ensureReplicas(object, holders, sites); + // Copy replicaCopy = ensureReplicas(object, holders, sites); List<Copy> siteCopies = ensureSites(object, sites); - removeExtras(object, holders, sites); + removeExtras(object, stores, sites); - if (replicaCopy != null) - result.add(replicaCopy); + // if (replicaCopy != null) + // result.add(replicaCopy); result.addAll(siteCopies); return result; } - Copy ensureReplicas(gobs.sim.Object object, - Set<Node> holders, List<Node> sites) + /** + Ensure that there is one and only one current Primary + for object. + */ + void ensurePrimary(Gob object, Sites holders) + { + Node primaryNode = holders.primary(); + simulator.placer.promote(primaryNode, object); + for (Node node : holders.get()) + if (! node.equals(primaryNode)) + simulator.placer.demote(node, object, primaryNode); + } + + /* + Ensure that the replica count for object is restored. + */ + /* + Copy ensureReplicas(Gob object, + Sites holders, Sites sites) { if (holders.size() < sites.size()) { @@ -201,14 +211,19 @@ public class Rebuild } return null; } + */ /** Ensure that sites that should hold this object do. */ - List<Copy> ensureSites(gobs.sim.Object object, List<Node> sites) + List<Copy> ensureSites(Gob object, Sites sites) { List<Copy> result = new ArrayList<Copy>(); - for (Node node : sites) + + if (queued(object)) + return result; + + for (Node node : sites.get()) if (! node.contains(object.id)) { Copy copy = simulator.placer.regimen(object); @@ -218,35 +233,84 @@ public class Rebuild return result; } - void removeExtras(gobs.sim.Object object, - Set<Node> holders, List<Node> sites) + boolean queued(Gob object) + { + for (Copy copy : queue) + if (copy.object.equals(object)) + return true; + for (Event event : simulator.inflight) + if (event instanceof Copy) + if (((Copy) event).object.equals(object)) + return true; + return false; + } + + void removeExtras(Gob object, + List<Node> holders, Sites sites) { - while (holders.size() > sites.size()) - for (Iterator<Node> it = holders.iterator(); it.hasNext(); ) + int hcount = holders.size(); + for (Node node : holders) + { + if (! sites.contains(node)) { - Node node = it.next(); - if (! sites.contains(node)) - { - node.delete(object.id); - it.remove(); - } + node.delete(object.id); + hcount--; } + if (hcount <= sites.size()) + break; + } } - void completeRepair(Event event) + void completeRepair(Repair repair) { - Repair repair = (Repair) event; - System.out.println(repair); - simulator.repairs.remove(repair); + if (simulator.bool("print.rebuilds")) + System.out.println(repair); Node node = repair.node; Set<Copy> copies = node.reboot(); for (Copy copy : copies) { - copy.startAt(repair.finish+selectTime); - queue.add(copy); + add(copy); } } + void add(Collection<Copy> copies) + { + List<Copy> result = new LinkedList<Copy>(copies); + Collections.shuffle(result, Bits.rng); + for (Copy copy : result) + add(copy); + } + + void add(Copy copy) + { + if (enqueue(copy)) + simulator.todo.add(copy); + } + + boolean enqueue(Copy copy) + { + for (Iterator<Copy> it = queue.iterator(); it.hasNext(); ) + { + Copy c = it.next(); + if (copy.destination.equals(c.destination) && + copy.object.equals(c.object)) + { + simulator.print("rebuilds", "copy rejected: " + copy); + return false; + } + } + queue.add(copy); + return true; + } + + void enqueue(Collection<Copy> copies) + { + List<Copy> result = new LinkedList<Copy>(copies); + Collections.shuffle(result, Bits.rng); + for (Copy copy : result) + enqueue(copy); + } + /** Compute the load from this Rebuild at absolute simulated time t. */ @@ -292,29 +356,38 @@ public class Rebuild } } - /** + /* Randomly reorder the copies into a Queue. */ - Queue<Copy> order(Set<Copy> copies) + /* + Queue<Copy> order(Collection<Copy> copies) { LinkedList<Copy> result = new LinkedList<Copy>(copies); Collections.shuffle(result, Bits.rng); return result; } + */ /** Pull out a Copy to perform. Ensure that the Nodes involved are not busy. */ - Copy select(Queue<Copy> copies, Set<Node> busy) + Copy select(Queue<Copy> copies) throws NoSuchElementException { for (Iterator<Copy> it = copies.iterator(); it.hasNext(); ) { Copy copy = it.next(); assert(copy != null); - if (! (busy.contains(copy.source) || - busy.contains(copy.destination))) + if (copy.destination.contains(copy.object.id)) + { + System.out.println("copy unnecessary: " + copy); + it.remove(); + simulator.todo.remove(copy); + continue; + } + if (! (simulator.busy.contains(copy.source) || + simulator.busy.contains(copy.destination))) { it.remove(); return copy; diff --git a/sim/gobs/gobs/sim/Repair.java b/sim/gobs/gobs/sim/Repair.java index 340f63c..09ddff3 100644 --- a/sim/gobs/gobs/sim/Repair.java +++ b/sim/gobs/gobs/sim/Repair.java @@ -14,11 +14,6 @@ class Repair finish = time+detect; } - public int compareTo(Event event) - { - return new Double(finish).compareTo(new Double(event.finish)); - } - String log() { return "Repair(" + node.toString(simulator) + "," + diff --git a/sim/gobs/gobs/sim/Replicated.java b/sim/gobs/gobs/sim/Replicated.java index f3ad5d9..3b57015 100644 --- a/sim/gobs/gobs/sim/Replicated.java +++ b/sim/gobs/gobs/sim/Replicated.java @@ -30,11 +30,11 @@ public abstract class Replicated /** Generate an object set for a file. */ - public Collection<gobs.sim.Object> + public Collection<Gob> allocateFile(File file, List<Node> nodes) { - ArrayList<Object> result = - new ArrayList<Object>(file.width); + ArrayList<Gob> result = + new ArrayList<Gob>(file.width); BigInteger bigWidth = new BigInteger("" + file.width); BigInteger s = M.divide(bigWidth); @@ -45,7 +45,7 @@ public abstract class Replicated { BigInteger id = p; int bytes = file.objectSize(i); - gobs.sim.Object obj = new SingleObject(id, bytes, file.replicas); + Gob obj = new SingleObject(id, bytes, file.replicas); result.add(obj); p = p.add(s).mod(M); } @@ -54,11 +54,11 @@ public abstract class Replicated return result; } -// public abstract Node lookup(BigInteger id, List<Node> nodes); + // public abstract Node lookup(BigInteger id, List<Node> nodes); - public void place(Collection<Object> objects, List<Node> nodes) + public void place(Collection<Gob> objects, List<Node> nodes) { - for (Object object : objects) + for (Gob object : objects) place(object, nodes); } @@ -66,7 +66,7 @@ public abstract class Replicated Initial placement: allocate replicas for object and place on nodes. */ - public void place(gobs.sim.Object object, List<Node> nodes) + public void place(Gob object, List<Node> nodes) { if (object instanceof SingleObject) place((SingleObject) object, nodes); @@ -77,10 +77,11 @@ public abstract class Replicated public void place(SingleObject object, List<Node> nodes) { // System.out.println("place: " + object); - List<Node> sites = sites(object, nodes); + Sites sites = sites(object, nodes); simulator.stats.flexibilities.add(sites.size()); - Primary primary = object.toPrimary(); - sites.remove(sites.size()-1).add(primary); + PrimaryObject primary = object.toPrimaryObject(); + Node primaryNode = sites.removePrimary(); + primaryNode.add(primary); for (int i = 0; i < object.replicas-1; i++) { if (sites.size() == 0) @@ -88,9 +89,8 @@ public abstract class Replicated System.out.println("Could not place all replicas!"); return; } - int r = Bits.nextInt(sites.size()); - Node node = sites.remove(r); - node.add(object.toSecondary(primary)); + Node node = sites.removeRandom(); + node.add(object.toSecondaryObject(primary)); } } @@ -103,7 +103,7 @@ public abstract class Replicated abstract Copy regimen(SingleObject object, List<Node> nodes); - Node removeSource(SingleObject object, List<Node> sites) + Node removeSource(SingleObject object, Sites sites) { Node source = source(object, sites); if (source == null) @@ -112,54 +112,34 @@ public abstract class Replicated return source; } - Node source(SingleObject object, List<Node> sites) + /** + Get a replica source for this object from within sites. + Does not modify sites. + May return null. + */ + Node source(SingleObject object, Sites sites) { - int p = sites.size()-1; - if (p < 0) - return null; - Node result = null; if (replicaSource == ReplicaSource.PRIMARY) - result = sites.get(p); + result = sites.primary(); else if (replicaSource == ReplicaSource.LAST) - result = lastSource(object, sites); + result = sites.last(); if (result == null || ! result.contains(object.id)) - result = randomSource(object, sites); + result = sites.randomSource(object); return result; } - Node randomSource(SingleObject object, List<Node> sites) - { - List<Node> valid = new ArrayList<Node>(sites.size()); - - for (Node node : sites) - if (node.contains(object.id)) - valid.add(node); - - if (valid.size() == 0) - return null; - - int r = Bits.nextInt(valid.size()); - return valid.get(r); - } - - Node lastSource(SingleObject object, List<Node> sites) - { - for (Node node : sites) - if (node.contains(object.id)) - return node; - return null; - } - /** Locate a random destination in sites that does not contain the object. + Removes nodes that do have the object, and the result, + from sites. May return null. */ - Node locateDestination(SingleObject object, List<Node> sites) + Node locateDestination(SingleObject object, Sites sites) { Node destination = null; @@ -167,8 +147,7 @@ public abstract class Replicated while (sites.size() > 0) { - int r = Bits.nextInt(sites.size()); - Node node = sites.remove(r); + Node node = sites.removeRandom(); if (! node.contains(object.id)) { destination = node; diff --git a/sim/gobs/gobs/sim/Secondary.java b/sim/gobs/gobs/sim/SecondaryObject.java similarity index 58% rename from sim/gobs/gobs/sim/Secondary.java rename to sim/gobs/gobs/sim/SecondaryObject.java index e5113f6..d472ac8 100644 --- a/sim/gobs/gobs/sim/Secondary.java +++ b/sim/gobs/gobs/sim/SecondaryObject.java @@ -7,18 +7,24 @@ import java.math.BigInteger; * Simulates a single secondary object. * */ -class Secondary +class SecondaryObject extends SingleObject { - Primary primary; + PrimaryObject primary; - Secondary(BigInteger id, int size, int replicas, - Primary primary) + SecondaryObject(BigInteger id, int size, int replicas, + PrimaryObject primary) { super(id, size, replicas); this.primary = primary; } + void setPrimary(PrimaryObject primary) + { + assert(primary instanceof PrimaryObject); + this.primary = primary; + } + public String toString(Simulator simulator) { return name(simulator) + "[S](" + size + ")"; diff --git a/sim/gobs/gobs/sim/SecondaryStrip.java b/sim/gobs/gobs/sim/SecondaryStrip.java index a78d9c7..ff2f981 100644 --- a/sim/gobs/gobs/sim/SecondaryStrip.java +++ b/sim/gobs/gobs/sim/SecondaryStrip.java @@ -19,6 +19,12 @@ class SecondaryStrip this.primary = primary; } + void setPrimary(PrimaryStrip primary) + { + assert(primary instanceof PrimaryStrip); + this.primary = primary; + } + public String toString(Simulator simulator) { return name(simulator) + "[S][" + number + "](" + size + ")"; diff --git a/sim/gobs/gobs/sim/Simulator.java b/sim/gobs/gobs/sim/Simulator.java index bb2d06f..e85fbca 100644 --- a/sim/gobs/gobs/sim/Simulator.java +++ b/sim/gobs/gobs/sim/Simulator.java @@ -73,8 +73,8 @@ public class Simulator String path = null; public List<Node> nodes; - EnhancedLinkedHashSet<gobs.sim.Object> objects = - new EnhancedLinkedHashSet<gobs.sim.Object>(); + EnhancedLinkedHashSet<Gob> objects = + new EnhancedLinkedHashSet<Gob>(); List<File> files = new ArrayList<File>(); /** @@ -93,9 +93,20 @@ public class Simulator public List<Rebuild> rebuilds = new ArrayList<Rebuild>(); /** - Current Repairs in progress. + The Set of Nodes that are currently making copies. */ - Set<Repair> repairs = new HashSet<Repair>(); + Set<Node> busy = new HashSet<Node>(); + + /** + Copies that the Simulator will perform in the future. + */ + Set<Copy> todo = new LinkedHashSet<Copy>(); + + /** + The Set of known Events that will finish in the future. + Contains inflight Copies, the next Fault, and any Repairs. + */ + NavigableSet<Event> inflight = new TreeSet<Event>(); /** Current rebuild in progress. @@ -361,7 +372,7 @@ public class Simulator File file = factory.get(); int tries = 0; - Collection<gobs.sim.Object> set; + Collection<Gob> set; do { // System.out.println("allocateFile(): iterate: " + tries); @@ -423,10 +434,10 @@ public class Simulator if (printAccesses) System.out.println("read: " + file.id); - for (gobs.sim.Object object : file.objects) + for (Gob object : file.objects) { - List<Node> sites = placer.sites(object, nodes); - Node node = placer.random(sites); + Sites sites = placer.sites(object, nodes); + Node node = sites.random(); if (printAccesses) System.out.println("access: object: " + object.id + " node: " + node.id); @@ -451,14 +462,14 @@ public class Simulator if (printAccesses) System.out.println("write: " + file.id); - for (gobs.sim.Object object : file.objects) + for (Gob object : file.objects) performWrite(object); } - void performWrite(gobs.sim.Object object) + void performWrite(Gob object) { - List<Node> sites = placer.sites(object, nodes); - for (Node node : sites) + Sites sites = placer.sites(object, nodes); + for (Node node : sites.get()) { if (printAccesses) System.out.println("access: object: " + object.id + @@ -478,10 +489,11 @@ public class Simulator */ void advanceTo(double time) { - Tools.check(this.time < time, + Tools.check(time >= this.time, "Attempted to step backwards in time!"); this.time = time; - System.out.println("time: " + time); + if (bool("print.rebuilds")) + System.out.println("time: " + df.format(time)); } /** @@ -495,42 +507,39 @@ public class Simulator void induceFaults() { System.out.println("faults: " + totalFaults); - Set<Copy> copies = new LinkedHashSet<Copy>(); - Set<Copy> done = new LinkedHashSet<Copy>(); // Advance to first fault: advanceTo(mttfExponential.nextDouble()); while (! complete()) { Node failed = induceFault(); - Collection<gobs.sim.Object> damaged = failed.collection(); - rebuild(failed, damaged, copies, done); + Collection<Gob> damaged = failed.collection(); + rebuild(failed, damaged); } - performCopies(copies); + if (bool("print.rebuilds")) + System.out.println("flushing copies:"); + performCopies(todo); + if (bool("print.nodes")) + printNodeObjects(nodes); } - void rebuild(Node node, Collection<gobs.sim.Object> damaged, - Set<Copy> copies, Set<Copy> done) + void rebuild(Node failed, Collection<Gob> damaged) { - startRebuild(node); + startRebuild(failed); List<Copy> work = planCopies(damaged); - copies.addAll(work); - double t = currentRebuild.rebuild(time, copies, done); - advanceTo(t); - performCopies(done); + currentRebuild.rebuild(work); completeRebuild(); } /** Setup a Rebuild. */ - Rebuild startRebuild(Node node) + Rebuild startRebuild(Node failed) { System.out.println("fault: " + faultCount); // Schedule the next fault: Fault fault = new Fault(this); - Repair repair = new Repair(this, node, time); - repairs.add(repair); - Rebuild rebuild = new Rebuild(this, fault, repairs); + Repair repair = new Repair(this, failed, time); + Rebuild rebuild = new Rebuild(this, fault, repair); currentRebuild = rebuild; return rebuild; } @@ -548,18 +557,53 @@ public class Simulator System.out.println("\nfailed: " + failed.name(this) + " (" + failed.objects.size() + ")" + " at: " + timef.format(time)); + failCopies(failed); return failed; } /** + Cancel any copies that are currently in todo or inflight + involving the failed Node. + */ + void failCopies(Node failed) + { + for (Iterator<Copy> it = todo.iterator(); it.hasNext(); ) + { + Copy copy = it.next(); + if (copy.source.equals(failed) || + copy.destination.equals(failed)) + { + if (bool("print.rebuilds")) + System.out.println("copy failed (todo): " + copy); + it.remove(); + } + } + + for (Iterator<Event> it = inflight.iterator(); it.hasNext(); ) + { + Event event = it.next(); + if (! (event instanceof Copy)) + return; + Copy copy = (Copy) event; + if (copy.source.equals(failed) || + copy.destination.equals(failed)) + { + if (bool("print.rebuilds")) + System.out.println("copy failed (inflight): " + copy); + it.remove(); + } + } + } + + /** Build the Map of Copies to perform by calling {@link Placement#regimen}. @param damaged The Set of Objects that need to be repaired */ - List<Copy> planCopies(Collection<gobs.sim.Object> damaged) + List<Copy> planCopies(Collection<Gob> damaged) { List<Copy> result = new ArrayList<Copy>(); - for (gobs.sim.Object object : damaged) + for (Gob object : damaged) { Copy copy = placer.regimen(object, nodes); if (copy != null) @@ -576,8 +620,8 @@ public class Simulator */ void performCopies(Set<Copy> done) { - // for (Copy copy : done) - // placer.copy(copy); + for (Copy copy : done) + placer.copy(copy); done.clear(); } @@ -593,18 +637,29 @@ public class Simulator } /** - Scan all nodes and return the List of Nodes currently - holding the given object. + Brute force search for all object locations. */ - Set<Node> scanStorage(gobs.sim.Object object) + List<Node> scanStorage(Gob object) { - Set<Node> holders = new LinkedHashSet<Node>(); + List<Node> holders = new ArrayList<Node>(); for (Node node : nodes) if (node.contains(object.id)) holders.add(node); return holders; } + /** + Brute force search for an object location. + */ + Node scanObject(Gob object) + { + List<Node> holders = scanStorage(object); + if (holders.size() == 0) + return null; + int r = Bits.nextInt(holders.size()); + return holders.get(r); + } + void computeStatistics() { computeDiffs(); @@ -801,8 +856,6 @@ public class Simulator System.out.println("TRAFFIC: "); double interval = decimal("plot.traffic.interval"); - // List<List<Integer>> results = new ArrayList<List<Integer>>(); - List<Double> intervals = new ArrayList<Double>(); List<Integer> sample = new ArrayList<Integer>(); double d = 0; @@ -818,9 +871,6 @@ public class Simulator intervals.add(d); sample.add(0); - System.out.println(intervals.toString()); - System.out.println(sample.toString()); - stats.results.setProperty("traffic-x", intervals.toString()); stats.results.setProperty("traffic-y", sample.toString()); } @@ -848,9 +898,9 @@ public class Simulator return count; } - void lost(gobs.sim.Object object) + void lost(Gob object) { - new Exception().printStackTrace(); + // new Exception().printStackTrace(); System.out.println("rebuild: lost: " + object.toString(this)); stats.lostObjects++; } @@ -882,7 +932,7 @@ public class Simulator void printNodeObjects(List<Node> list) { - System.out.println("NODE_CONTENTS:"); + System.out.println("\nNODE_CONTENTS:"); for (Node node : list) System.out.println(node.display(this)); } @@ -915,6 +965,22 @@ public class Simulator } /** + Free up some memory. + */ + public void free() + { + for (Node node : nodes) + node.objects.clear(); + files.clear(); + objects.clear(); + nodes.clear(); + busy.clear(); + todo.clear(); + inflight.clear(); + System.gc(); + } + + /** Allow subordinate classes to check properties. @return true if property name exists. */ @@ -958,4 +1024,13 @@ public class Simulator { return Tools.bool(properties, name); } + + /** + Print msg iff property print.category is true. + */ + public void print(String category, String msg) + { + if (bool("print."+category)) + System.out.println(msg); + } } diff --git a/sim/gobs/gobs/sim/SingleObject.java b/sim/gobs/gobs/sim/SingleObject.java index 0347506..4699813 100644 --- a/sim/gobs/gobs/sim/SingleObject.java +++ b/sim/gobs/gobs/sim/SingleObject.java @@ -8,28 +8,27 @@ import java.math.BigInteger; * */ public class SingleObject - extends gobs.sim.Object + extends Gob { public SingleObject(BigInteger id, int size, int replicas) { super(id, size, replicas); } - Primary toPrimary() + PrimaryObject toPrimaryObject() { - // return new Primary(B, id, size, replicas); - return new Primary(id, size, replicas); + return new PrimaryObject(id, size, replicas); } - Secondary toSecondary() + SecondaryObject toSecondaryObject() { throw new UnsupportedOperationException(); } - Secondary toSecondary(Primary primary) + SecondaryObject toSecondaryObject(PrimaryObject primary) { - // return new Secondary(B, id, size, replicas, primary); - return new Secondary(id, size, replicas, primary); + assert(primary instanceof PrimaryObject); + return new SecondaryObject(id, size, replicas, primary); } public String toString(Simulator simulator) diff --git a/sim/gobs/gobs/sim/Sites.java b/sim/gobs/gobs/sim/Sites.java new file mode 100644 index 0000000..fdeba1a --- /dev/null +++ b/sim/gobs/gobs/sim/Sites.java @@ -0,0 +1,133 @@ + +package gobs.sim; + +import java.util.*; + +import gobs.util.Bits; + +/** + * Represents a list of sites on which a Gob may reside. + * */ + +class Sites +{ + Gob object; + + LinkedList<Node> sites = new LinkedList<Node>(); + + Node primary = null; + Node last = null; + + Sites(Gob object) + { + this.object = object; + } + + int size() + { + return sites.size(); + } + + void primary(Node node) + { + if (! sites.contains(node)) + sites.add(node); + primary = node; + } + + void last(Node node) + { + if (! sites.contains(node)) + sites.add(node); + last = node; + } + + void secondary(Node node) + { + if (! sites.contains(node)) + sites.add(node); + } + + Node primary() + { + return sites.getLast(); + } + + Node removePrimary() + { + Node primary = primary(); + sites.remove(primary); + return primary; + } + + Node removeRandom() + { + Node node = random(); + sites.remove(node); + return node; + } + + void remove(Node node) + { + sites.remove(node); + } + + Node remove() + { + return sites.remove(0); + } + + Node random() + { + int r = Bits.nextInt(sites.size()); + Node result = sites.get(r); + return result; + } + + /** + Return a random Node that has this object. + May return null. + */ + Node randomSource(Gob object) + { + List<Node> valid = new ArrayList<Node>(sites.size()); + + for (Node node : sites) + if (node.contains(object.id)) + valid.add(node); + + if (valid.size() == 0) + return null; + + int r = Bits.nextInt(valid.size()); + return valid.get(r); + } + + boolean contains(Node node) + { + return sites.contains(node); + } + + Collection<Node> get() + { + Collection<Node> result = new ArrayList<Node>(sites); + return result; + } + + /** + Last replica in the chain. + */ + Node last() + { + // Last replica in the chain is first in our list + return sites.getFirst(); + } + + public String toString() + { + String result = "for: " + object + " : "; + for (Node node : sites) + result += node + " "; + return result; + } +} diff --git a/sim/gobs/gobs/sim/VertexGroups.java b/sim/gobs/gobs/sim/VertexGroups.java index 5bca1a6..7ef8660 100644 --- a/sim/gobs/gobs/sim/VertexGroups.java +++ b/sim/gobs/gobs/sim/VertexGroups.java @@ -34,7 +34,7 @@ public class VertexGroups return id1.xor(id2); } - List<Node> sites(gobs.sim.Object object, List<Node> nodes) + Sites sites(Gob object, Collection<Node> nodes) { if (object instanceof SingleObject) return sites((SingleObject) object, nodes); @@ -44,24 +44,26 @@ public class VertexGroups "object must be an SingleObject."); } - List<Node> sites(SingleObject object, List<Node> nodes) + Sites sites(SingleObject object, Collection<Node> nodes) { - List<Node> result = new ArrayList<Node>(P+1); - Set<Node> sites = new HashSet<Node>(2*(P+1)); + Sites result = new Sites(object); + Set<Node> sites = new HashSet<Node>(2*(P+1)); Node primary = lookup(object.id, nodes); + result.primary(primary); - for (int i = 1; i <= P; i++) + int i = 1; + for (; i < object.replicas-2; i++) { BigInteger t = flip(simulator.B, i, object.id); Node node = lookup(t, nodes); - - if (node != primary) - sites.add(node); + result.secondary(node); } - result.addAll(sites); - result.add(primary); + BigInteger t = flip(simulator.B, i, object.id); + Node last = lookup(t, nodes); + result.last(last); + return result; } diff --git a/sim/gobs/gobs/sweep/AbstractSweep.java b/sim/gobs/gobs/sweep/AbstractSweep.java index ba7ac89..cc631c6 100644 --- a/sim/gobs/gobs/sweep/AbstractSweep.java +++ b/sim/gobs/gobs/sweep/AbstractSweep.java @@ -106,6 +106,8 @@ public abstract class AbstractSweep return exps(min,max); else if (step.equals("mids")) return mids(min,max); + else if (step.startsWith("interval")) + return interval(step, min, max); else if (step.startsWith("literal")) return literal(step); else @@ -196,6 +198,38 @@ public abstract class AbstractSweep } /** + Sweep over all integer values between min and max + stepping over the interval argument. + @param arg The argument, e.g., "interval_4". + @return The list, e.g., min, min+4, min+8, max. + */ + static List<String> interval(String arg, int min, int max) + { + String[] tokens = arg.split("_"); + + int interval = 0; + try + { + interval = Integer.parseInt(tokens[1]); + } + catch (NumberFormatException e) + { + Tools.bail("AbstractSweep: interval must be an integer!"); + } + + List<String> result = new ArrayList<String>(); + result.add(""+min); + int i = 1; + while (min + interval*i < max) + { + result.add("" + (min+interval*i)); + i++; + } + result.add(""+max); + return result; + } + + /** Sweep over all integer values given by the literal. Ignores min and max. @param arg The argument, e.g., "literal_2,4,5". @@ -203,8 +237,8 @@ public abstract class AbstractSweep */ static List<String> literal(String arg) { - int i = arg.indexOf("_"); - String vals = arg.substring(i+1); + int p = arg.indexOf("_"); + String vals = arg.substring(p+1); String[] tokens = vals.split(","); List<String> result = new ArrayList<String>(); @@ -241,6 +275,12 @@ public abstract class AbstractSweep return result; } + static void free(Simulator simulator) + { + if (Tools.bool(properties, "sweep.free")) + simulator.free(); + } + static void reportRuntime() { long t = (new Date().getTime()-launch.getTime())/(60*1000); diff --git a/sim/gobs/gobs/sweep/Sweep.java b/sim/gobs/gobs/sweep/Sweep.java index 4511bd1..cb7da80 100644 --- a/sim/gobs/gobs/sweep/Sweep.java +++ b/sim/gobs/gobs/sweep/Sweep.java @@ -110,6 +110,7 @@ public class Sweep Simulator simulator = new Simulator(properties); simulator.execute(); sims.put(tag, simulator); + free(simulator); } /** diff --git a/sim/gobs/gobs/test/TestELHS.java b/sim/gobs/gobs/test/TestELHS.java index 3ffedf1..70c86ec 100644 --- a/sim/gobs/gobs/test/TestELHS.java +++ b/sim/gobs/gobs/test/TestELHS.java @@ -17,12 +17,12 @@ class TestELHS { int n = 10; - EnhancedLinkedHashSet<gobs.sim.Object> set = - new EnhancedLinkedHashSet<gobs.sim.Object>(); + EnhancedLinkedHashSet<Gob> set = + new EnhancedLinkedHashSet<Gob>(); for (int i = 0; i < n; i++) { - List<gobs.sim.Object> list = new ArrayList<gobs.sim.Object>(); + List<Gob> list = new ArrayList<gobs.sim.Object>(); for (int j = 0; j < n; j++) list.add(new SingleObject(new BigInteger("" + j), 0, 0)); if (set.containsAny(list)) diff --git a/sim/gobs/gobs/util/BoundedSortedValues.java b/sim/gobs/gobs/util/BoundedSortedValues.java index 7cebb2c..88fd5f7 100644 --- a/sim/gobs/gobs/util/BoundedSortedValues.java +++ b/sim/gobs/gobs/util/BoundedSortedValues.java @@ -3,6 +3,12 @@ package gobs.util; import java.util.*; +/** + * Like a Map. + * Given (t,v) pairs, only retains the <tt>bound</tt> + * lowest values of v. + * */ + public class BoundedSortedValues<T,V extends Comparable<V>> { List<Entry> list = new LinkedList<Entry>(); @@ -104,6 +110,15 @@ public class BoundedSortedValues<T,V extends Comparable<V>> return list.size(); } + /** + Return t from (t,v) such that v is the smallest. + */ + public T peek() + { + Entry pair = list.get(list.size()-1); + return pair.getKey(); + } + public List<T> get() { List<T> result = new ArrayList<T>(list.size()); diff --git a/sim/gobs/gobs/util/Tools.java b/sim/gobs/gobs/util/Tools.java index 4e270e5..b7a03c5 100644 --- a/sim/gobs/gobs/util/Tools.java +++ b/sim/gobs/gobs/util/Tools.java @@ -86,6 +86,7 @@ public class Tools /** Call bail(msg) if condition is false. + Like assert() but does not require -ea. */ public static void check(boolean condition, String msg) { diff --git a/sim/gobs/scripts/eol.all.zsh b/sim/gobs/scripts/eol.all.zsh new file mode 100755 index 0000000..292b1ae --- /dev/null +++ b/sim/gobs/scripts/eol.all.zsh @@ -0,0 +1,12 @@ +#!/bin/zsh + +files=() +for f in **/*.zsh + do + if [[ ${f} != $0 ]] + then + files+=${f} + fi +done + +sed -i 's/ *$//' **/*.java **/*.cfg ${files} diff --git a/sim/gobs/scripts/eol.zsh b/sim/gobs/scripts/eol.zsh index 0a65445..6a0fc61 100644 --- a/sim/gobs/scripts/eol.zsh +++ b/sim/gobs/scripts/eol.zsh @@ -20,6 +20,12 @@ do fi done +if [[ ${#FILES} == 0 ]] +then + print "Nothing to do." + exit +fi + print -l $FILES sed -i 's/ *$//' ${FILES} diff --git a/sim/gobs/sweeps/dl01-small.cfg b/sim/gobs/sweeps/dl01-small.cfg index ec10bcc..afb01cc 100644 --- a/sim/gobs/sweeps/dl01-small.cfg +++ b/sim/gobs/sweeps/dl01-small.cfg @@ -7,6 +7,9 @@ include = rb_output.cfg # Statistics: +# Clear some memory after each simulator run +sweep.free = true + # Rebuild traffic over time analysis: plot.traffic.interval = 10 @@ -22,9 +25,9 @@ nodes.max = 16 # Step through nodes: See AbstractSweep.values() nodes.step = literal_14,16,18 # Number of files: -files = 6 +files = 10 # Number of faults: -faults = 4 +faults = 30 # Number of accesses: reads = 0 writes = 0 @@ -32,7 +35,9 @@ writes = 0 # Step through MTTF values: See AbstractSweep.values() mttf.min = 150 mttf.max = 600 -mttf.step = exps +mttf.step = interval_50 + +mttr = 200 ### Choose a PlacementScheme implementation: diff --git a/sim/gobs/sweeps/rb01-small.cfg b/sim/gobs/sweeps/rb01-small.cfg index 6474dc5..96a8180 100644 --- a/sim/gobs/sweeps/rb01-small.cfg +++ b/sim/gobs/sweeps/rb01-small.cfg @@ -30,6 +30,7 @@ reads = 0 writes = 0 mttf = 500 +mttr = 400 ### Choose a PlacementScheme implementation: hooks/post-receive -- Grayskull Repository

1 0

Grayskull Repository branch, master, updated. git-migration-270-g39e095c
by noreply＠mcs.anl.gov 08 Dec '09

08 Dec '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 39e095cdbdb876901ecb231d972127769a53e738 (commit) via 0a4be2be06fcefc4c1c87c80124be486bd1335aa (commit) via 17818dcab75ee56fad1ad53fefbf1b5a129871c5 (commit) via 64daac7dfccc81d15117033e696b07a9de2901d1 (commit) via 2c09992d25210b0f4bd8c2cd199b590b2de56eeb (commit) via d9cf3a116194186489c35838e4db245c1e362da2 (commit) via 773c7bb41b660b0badbf4d45a8cd7ec44179e120 (commit) via c4b17c2baab47680d74f2966da314af8106c081f (commit) via 1c8ac6117e70b4fe0923200557cd77b3ce20e4eb (commit) via f5ad79137af191cdf96b9fe15905e4f7de6d58d3 (commit) via 8d87f640e6d9bd3bb416a9a937ef150e0e0977d3 (commit) via 897f889c340a3d7c548df41036135060cb4279e3 (commit) via 0db6653ab56dd8d394c2a9a2d9634560db3f25f8 (commit) via 00cab39f4c98e97a2b37fc6fb3151a9f694817d9 (commit) via d1fbbaea9b0a2c804d43469df371ccd4df523b90 (commit) from c8fbc01d3ba901fb39fbb042e74ca51aad5847b3 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 39e095cdbdb876901ecb231d972127769a53e738 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Dec 8 10:23:50 2009 -0600 EOL fixes. commit 0a4be2be06fcefc4c1c87c80124be486bd1335aa Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Dec 8 10:20:31 2009 -0600 New Rebuild.validate() covers some corner cases during Rebuilds. commit 17818dcab75ee56fad1ad53fefbf1b5a129871c5 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 20:00:55 2009 -0600 Basic support for Node Repairs. commit 64daac7dfccc81d15117033e696b07a9de2901d1 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 20:00:32 2009 -0600 doc tag fix. commit 2c09992d25210b0f4bd8c2cd199b590b2de56eeb Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 19:58:41 2009 -0600 build.xml: Add veryclean target. Now clean does not remove javadoc. commit d9cf3a116194186489c35838e4db245c1e362da2 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 18:32:22 2009 -0600 Better javadoc. commit 773c7bb41b660b0badbf4d45a8cd7ec44179e120 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 16:53:11 2009 -0600 Fixes for new subpackage x. commit c4b17c2baab47680d74f2966da314af8106c081f Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 15:51:49 2009 -0600 Post new DataLoss01 plot to wiki. commit 1c8ac6117e70b4fe0923200557cd77b3ce20e4eb Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 15:43:36 2009 -0600 Separate reusable gobs.sweep classes from eXperiments in gobs.sweep.x . commit f5ad79137af191cdf96b9fe15905e4f7de6d58d3 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 15:41:56 2009 -0600 Fixes to DataLoss01. commit 8d87f640e6d9bd3bb416a9a937ef150e0e0977d3 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 15:40:08 2009 -0600 New object loss tests. commit 897f889c340a3d7c548df41036135060cb4279e3 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 15:39:35 2009 -0600 New basic 2D parameter sweep. commit 0db6653ab56dd8d394c2a9a2d9634560db3f25f8 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 15:36:07 2009 -0600 Better handling of lost objects. Some output improvements as well. commit 00cab39f4c98e97a2b37fc6fb3151a9f694817d9 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 15:35:18 2009 -0600 Sanity output. commit d1fbbaea9b0a2c804d43469df371ccd4df523b90 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sat Dec 5 14:17:43 2009 -0600 Faster EOL fixer. ----------------------------------------------------------------------- Summary of changes: sim/gobs/build.xml | 13 +- sim/gobs/gobs/plot/MultiplePlot.java | 1 + sim/gobs/gobs/sim/Copy.java | 30 +- sim/gobs/gobs/sim/Event.java | 13 +- sim/gobs/gobs/sim/Fault.java | 9 +- sim/gobs/gobs/sim/Metric.java | 56 +- sim/gobs/gobs/sim/Node.java | 59 +- sim/gobs/gobs/sim/Object.java | 14 +- sim/gobs/gobs/sim/Placement.java | 37 + sim/gobs/gobs/sim/Progress.java | 7 + sim/gobs/gobs/sim/RandomNodes.java | 2 +- sim/gobs/gobs/sim/Rebuild.java | 167 ++- sim/gobs/gobs/sim/Repair.java | 32 + sim/gobs/gobs/sim/Replicated.java | 26 +- sim/gobs/gobs/sim/SimpleNodes.java | 2 +- sim/gobs/gobs/sim/Simulator.java | 147 +- sim/gobs/gobs/sim/Status.java | 12 + sim/gobs/gobs/sweep/AbstractSweep.java | 18 +- sim/gobs/gobs/sweep/Affected01.java | 38 - sim/gobs/gobs/sweep/ArraySweep.java | 23 +- sim/gobs/gobs/sweep/Balance01.java | 77 - sim/gobs/gobs/sweep/Balance02.java | 66 - sim/gobs/gobs/sweep/Decluster01.java | 33 - sim/gobs/gobs/sweep/Decluster01P.java | 33 - sim/gobs/gobs/sweep/Decluster02.java | 69 - sim/gobs/gobs/sweep/Decluster03.java | 64 - sim/gobs/gobs/sweep/Decluster04.java | 69 - sim/gobs/gobs/sweep/ParallelSweep.java | 7 +- sim/gobs/gobs/sweep/Parallelism01.java | 63 - sim/gobs/gobs/sweep/Rebuild01.java | 74 - sim/gobs/gobs/sweep/Rebuild02.java | 126 - sim/gobs/gobs/sweep/Sweep.java | 82 +- sim/gobs/gobs/sweep/x/Affected01.java | 42 + sim/gobs/gobs/sweep/x/Balance01.java | 78 + sim/gobs/gobs/sweep/x/Balance02.java | 67 + sim/gobs/gobs/sweep/x/DataLoss01.java | 143 ++ sim/gobs/gobs/sweep/x/Decluster01.java | 37 + sim/gobs/gobs/sweep/x/Decluster01P.java | 36 + sim/gobs/gobs/sweep/x/Decluster02.java | 69 + sim/gobs/gobs/sweep/x/Decluster03.java | 64 + sim/gobs/gobs/sweep/x/Decluster04.java | 71 + sim/gobs/gobs/sweep/x/Parallelism01.java | 67 + sim/gobs/gobs/sweep/x/Rebuild01.java | 76 + sim/gobs/gobs/sweep/x/Rebuild02.java | 127 + sim/gobs/scripts/eol.zsh | 25 +- sim/gobs/sweeps/dl01-small.cfg | 79 + sim/gobs/wiki/dl01-small.cfg | 79 + sim/gobs/wiki/dl01-small.eps | 3771 ++++++++++++++++++++++++++++++ 48 files changed, 5401 insertions(+), 899 deletions(-) create mode 100644 sim/gobs/gobs/sim/Progress.java create mode 100644 sim/gobs/gobs/sim/Repair.java create mode 100644 sim/gobs/gobs/sim/Status.java delete mode 100644 sim/gobs/gobs/sweep/Affected01.java delete mode 100644 sim/gobs/gobs/sweep/Balance01.java delete mode 100644 sim/gobs/gobs/sweep/Balance02.java delete mode 100644 sim/gobs/gobs/sweep/Decluster01.java delete mode 100644 sim/gobs/gobs/sweep/Decluster01P.java delete mode 100644 sim/gobs/gobs/sweep/Decluster02.java delete mode 100644 sim/gobs/gobs/sweep/Decluster03.java delete mode 100644 sim/gobs/gobs/sweep/Decluster04.java delete mode 100644 sim/gobs/gobs/sweep/Parallelism01.java delete mode 100644 sim/gobs/gobs/sweep/Rebuild01.java delete mode 100644 sim/gobs/gobs/sweep/Rebuild02.java create mode 100644 sim/gobs/gobs/sweep/x/Affected01.java create mode 100644 sim/gobs/gobs/sweep/x/Balance01.java create mode 100644 sim/gobs/gobs/sweep/x/Balance02.java create mode 100644 sim/gobs/gobs/sweep/x/DataLoss01.java create mode 100644 sim/gobs/gobs/sweep/x/Decluster01.java create mode 100644 sim/gobs/gobs/sweep/x/Decluster01P.java create mode 100644 sim/gobs/gobs/sweep/x/Decluster02.java create mode 100644 sim/gobs/gobs/sweep/x/Decluster03.java create mode 100644 sim/gobs/gobs/sweep/x/Decluster04.java create mode 100644 sim/gobs/gobs/sweep/x/Parallelism01.java create mode 100644 sim/gobs/gobs/sweep/x/Rebuild01.java create mode 100644 sim/gobs/gobs/sweep/x/Rebuild02.java mode change 100755 => 100644 sim/gobs/scripts/eol.zsh create mode 100644 sim/gobs/sweeps/dl01-small.cfg create mode 100644 sim/gobs/wiki/dl01-small.cfg create mode 100755 sim/gobs/wiki/dl01-small.eps Diff of changes: diff --git a/sim/gobs/build.xml b/sim/gobs/build.xml index f5c57cf..4fc4316 100644 --- a/sim/gobs/build.xml +++ b/sim/gobs/build.xml @@ -93,7 +93,9 @@ classpath="${classpath}" breakiterator="yes" linksource="yes" - private="true"> + private="true" + doctitle="GOBS Docs" + windowtitle="GOBS"> <fileset dir="." includes="**/*.java"/> <link href="http://java.sun.com/javase/6/docs/api" /> @@ -102,6 +104,11 @@ </javadoc> </target> + <target name="veryclean" + depends="clean"> + <delete dir="docs" /> + </target> + <target name="clean"> <delete> <fileset dir="."> @@ -116,11 +123,13 @@ <fileset dir="gobs/sweep"> <filename name="*.class" /> </fileset> + <fileset dir="gobs/sweep/x"> + <filename name="*.class" /> + </fileset> <fileset dir="gobs/util"> <filename name="*.class" /> </fileset> </delete> - <delete dir="docs" /> </target> </project> diff --git a/sim/gobs/gobs/plot/MultiplePlot.java b/sim/gobs/gobs/plot/MultiplePlot.java index bc18fe7..44cf14b 100644 --- a/sim/gobs/gobs/plot/MultiplePlot.java +++ b/sim/gobs/gobs/plot/MultiplePlot.java @@ -100,6 +100,7 @@ public class MultiplePlot } IOUtils.closeQuietly(out); + System.out.println("plotted: " + output); return true; } diff --git a/sim/gobs/gobs/sim/Copy.java b/sim/gobs/gobs/sim/Copy.java index 213bb42..5a9c34f 100644 --- a/sim/gobs/gobs/sim/Copy.java +++ b/sim/gobs/gobs/sim/Copy.java @@ -4,7 +4,7 @@ package gobs.sim; import java.text.DecimalFormat; import java.util.*; -import gobs.util.Bits; +import gobs.util.*; class Copy extends Event @@ -15,22 +15,38 @@ class Copy boolean complete = false; double start = 0; + Progress progress; + static DecimalFormat df = new DecimalFormat("0.00"); - Copy(gobs.sim.Object object, Node source, Node destination) + Copy(Simulator simulator, + gobs.sim.Object object, Node source, Node destination) { + super(simulator); assert (source != null); this.object = object; this.source = source; this.destination = destination; + progress = Progress.PROTO; } void startAt(double start) { - this.start = start; - finish = start + object.size/Rebuild.rate; - if (Simulator.current.bool("print.rebuilds")) - System.out.println("start: " + log()); + if (progress == Progress.PROTO) + { + this.start = start; + finish = start + object.size/Rebuild.rate; + progress = Progress.RUNNING; + if (Simulator.current.bool("print.rebuilds")) + System.out.println("start: " + log()); + } + else if (progress == Progress.RUNNING) + { + if (Simulator.current.bool("print.rebuilds")) + System.out.println("restart: " + log()); + } + else if (progress == Progress.DONE) + Tools.bail("Copy.startAt(): Copy is DONE: " + log()); } public String toString() @@ -44,7 +60,7 @@ class Copy sb.append("copy: ") .append(object).append(": ") .append(source).append(" -> ").append(destination) - .append("(").append(df.format(start)); + .append(" (").append(df.format(start)); if (finish == -1) sb.append(")"); else diff --git a/sim/gobs/gobs/sim/Event.java b/sim/gobs/gobs/sim/Event.java index 5831a1f..3cc5f3c 100644 --- a/sim/gobs/gobs/sim/Event.java +++ b/sim/gobs/gobs/sim/Event.java @@ -6,6 +6,8 @@ import java.text.DecimalFormat; class Event implements Comparable<Event> { + Simulator simulator; + /** The simulated time at which the Event finishes or, in the case of an instantaneous event, occurs. @@ -17,12 +19,15 @@ class Event */ static DecimalFormat df = new DecimalFormat("0.00"); - Event() - {} + Event(Simulator simulator) + { + this.simulator = simulator; + } - Event(double finish) + Event(Simulator simulator, double finish) { - this.finish = finish; + this(simulator); + this.finish = finish; } public int compareTo(Event event) diff --git a/sim/gobs/gobs/sim/Fault.java b/sim/gobs/gobs/sim/Fault.java index ee67abb..669accd 100644 --- a/sim/gobs/gobs/sim/Fault.java +++ b/sim/gobs/gobs/sim/Fault.java @@ -4,12 +4,11 @@ package gobs.sim; class Fault extends Event { - Fault() - {} - - Fault(double finish) + Fault(Simulator simulator) { - this.finish = finish; + super(simulator); + this.finish = simulator.time() + + simulator.mttfExponential.nextDouble(); } public int compareTo(Event event) diff --git a/sim/gobs/gobs/sim/Metric.java b/sim/gobs/gobs/sim/Metric.java index 0a9f5cf..cbec1ae 100644 --- a/sim/gobs/gobs/sim/Metric.java +++ b/sim/gobs/gobs/sim/Metric.java @@ -4,6 +4,8 @@ package gobs.sim; import java.math.BigInteger; import java.util.List; +import gobs.util.Tools; + /** * Superclass for Placements with some concept of * a distance metric. @@ -19,6 +21,9 @@ public abstract class Metric public abstract BigInteger distance(BigInteger id1, BigInteger id2); + /** + Find the closest Node to this id. + */ public Node lookup(BigInteger id, List<Node> nodes) { Node result = null; @@ -37,10 +42,12 @@ public abstract class Metric /** Return the Copy that is prescribed for this object. + Removes the Copy.source from nodes. May return null. */ Copy regimen(SingleObject object, List<Node> nodes) { + System.out.println("regimen: " + object); List<Node> sites = sites(object, nodes); Secondary secondary; // The node that should have the primary in the current system @@ -50,24 +57,39 @@ public abstract class Metric Node source = removeSource(object, sites); if (source == null) + { + simulator.lost(object); return null; + } Copy result = regimen(object, primaryNode, primaryLocation, source, sites); - logCopy(result); + // assert (result != null); + if (result != null) + logCopy(result); return result; } + /** + Locate the Primary node if it holds the Primary. + Else returns null. + */ Node locatePrimary(SingleObject object, List<Node> sites) { - Node result = sites.get(sites.size()-1); - if (result.containsPrimary(object.id)) - return result; + int p = sites.size()-1; + if (p >= 0) + { + Node result = sites.get(p); + if (result.containsPrimary(object.id)) + return result; + } return null; } /** - Obtain the Copy. May return null. + Obtain the Copy. + Returns null if there is no destination for the Object, + which indicates that the Object is back to full replica count. @param primaryNode The node that should have the primary in the current system @param primaryLocation The node that currently holds the primary @@ -120,20 +142,32 @@ public abstract class Metric } } - if (payload == null || - source == null || - destination == null) + if (destination == null) { - System.out.println("regimen(): could not handle: " + object); + System.out.println("no destination: " + object); return null; } - return new Copy(payload, source, destination); + if (payload == null || + source == null) + Tools.bail("regimen(): could not handle: " + object); + + return new Copy(simulator, payload, source, destination); } + /** + Return the primary node from sites, regardless of what objects + it holds. + This is the last site in the list, by convention. + Returns null if sites is empty. + */ Node getPrimaryNode(SingleObject object, List<Node> sites) { - return sites.get(sites.size()-1); + int p = sites.size()-1; + if (p >= 0) + return sites.get(p); + else + return null; } void logCopy(Copy copy) diff --git a/sim/gobs/gobs/sim/Node.java b/sim/gobs/gobs/sim/Node.java index c5ec5eb..67c974c 100644 --- a/sim/gobs/gobs/sim/Node.java +++ b/sim/gobs/gobs/sim/Node.java @@ -13,6 +13,8 @@ import gobs.util.Tools; public class Node extends Addressable { + Simulator simulator; + /** Object store. */ @@ -33,9 +35,12 @@ public class Node */ int accesses = 0; - Node(BigInteger id) + Status status = Status.UP; + + Node(Simulator simulator, BigInteger id) { super(id); + this.simulator = simulator; objects = new HashMap<BigInteger,gobs.sim.Object>(); } @@ -58,7 +63,7 @@ public class Node } /** - Remove and return an gobs.sim.Object with this id. + Remove and return a gobs.sim.Object with this id. */ gobs.sim.Object remove(BigInteger id) { @@ -66,6 +71,18 @@ public class Node } /** + Delete the Object. + Like {@link #remove}. + */ + void delete(BigInteger id) + { + System.out.println("deleting: from node: " + this + + " object: " + id); + Object o = objects.remove(id); + assert(o != null); + } + + /** Return a Collection of all of this Node's objects. */ public Collection<gobs.sim.Object> collection() @@ -74,6 +91,44 @@ public class Node } /** + After a Repair, remove all objects. + @return A Set of Copies to be performed to restore this Node. + */ + Set<Copy> reboot() + { + simulator.nodes.add(this); + replaceDisk(); + status = Status.UP; + return rejoin(); + } + + /** + Rejoin the cluster. + */ + Set<Copy> rejoin() + { + // Ensure no Object is being copied twice simultaneously + Set<BigInteger> ids = new HashSet<BigInteger>(); + Set<Copy> copies = new LinkedHashSet<Copy>(); + for (Node other : simulator.nodes) + { + Collection<Object> objects = other.objects.values(); + for (gobs.sim.Object object : objects) + { + List<Node> sites = simulator.placer.sites(object); + if (sites.contains(this) && ids.add(object.id)) + { + Node source = simulator.placer.removeSource(object, sites); + Copy copy = new Copy(simulator, object, source, this); + System.out.println("rejoin: " + copy); + copies.add(copy); + } + } + } + return copies; + } + + /** Act as though this Node's disk failed and was replaced. Essentially just empties {@link #objects}. */ diff --git a/sim/gobs/gobs/sim/Object.java b/sim/gobs/gobs/sim/Object.java index 455a216..99632dd 100644 --- a/sim/gobs/gobs/sim/Object.java +++ b/sim/gobs/gobs/sim/Object.java @@ -10,13 +10,6 @@ import java.math.BigInteger; public abstract class Object extends Addressable { - Object(BigInteger id, int size, int replicas) - { - super(id); - this.size = size; - this.replicas = replicas; - } - /** Simulated size in bytes. */ @@ -27,5 +20,12 @@ public abstract class Object */ int replicas; + Object(BigInteger id, int size, int replicas) + { + super(id); + this.size = size; + this.replicas = replicas; + } + abstract String toString(Simulator simulator); } diff --git a/sim/gobs/gobs/sim/Placement.java b/sim/gobs/gobs/sim/Placement.java index d1ccb07..a9f6e02 100644 --- a/sim/gobs/gobs/sim/Placement.java +++ b/sim/gobs/gobs/sim/Placement.java @@ -47,6 +47,16 @@ public abstract class Placement } /** + Obtain a Copy that will repair the loss of this Object + with respect to the current Simulator nodes. + @return The Copy or null if the Object is lost. + */ + Copy regimen(gobs.sim.Object object) + { + return regimen(object, simulator.nodes); + } + + /** Obtain a Copy that will repair the loss of this SingleObject. */ abstract Copy regimen(SingleObject object, List<Node> nodes); @@ -65,6 +75,7 @@ public abstract class Placement */ void copy(Copy copy) { + System.out.println("perform: " + copy); copy.destination.add(copy.object); } @@ -94,6 +105,32 @@ public abstract class Placement abstract List<Node> sites(gobs.sim.Object object, List<Node> nodes); /** + Obtain a list of nodes that may host the given object + with respect to the replica placement scheme used here + and the current Simulator node list. + + Closest node should be last. + */ + List<Node> sites(gobs.sim.Object object) + { + return sites(object, simulator.nodes); + } + + Node removeSource(gobs.sim.Object object, List<Node> sites) + { + if (object instanceof SingleObject) + return removeSource((SingleObject) object, sites); + throw new UnsupportedOperationException("Cannot do this yet!"); + } + + /** + Locate a replica of the given object on one of the given nodes + in accordance with the policy; + remove and return that replica from sites. + */ + abstract Node removeSource(SingleObject object, List<Node> sites); + + /** Pull a random Node from one of the given sites. */ Node random(List<Node> sites) diff --git a/sim/gobs/gobs/sim/Progress.java b/sim/gobs/gobs/sim/Progress.java new file mode 100644 index 0000000..466f224 --- /dev/null +++ b/sim/gobs/gobs/sim/Progress.java @@ -0,0 +1,7 @@ + +package gobs.sim; + +enum Progress +{ + PROTO, RUNNING, DONE +} diff --git a/sim/gobs/gobs/sim/RandomNodes.java b/sim/gobs/gobs/sim/RandomNodes.java index 3baf3a1..7a17559 100644 --- a/sim/gobs/gobs/sim/RandomNodes.java +++ b/sim/gobs/gobs/sim/RandomNodes.java @@ -31,7 +31,7 @@ class RandomNodes Tools.bail("Could not assign unique node id!"); id = new BigInteger(simulator.B, Bits.rng); - node = new Node(id); + node = new Node(simulator, id); } while (nodes.contains(node)); nodes.add(node); diff --git a/sim/gobs/gobs/sim/Rebuild.java b/sim/gobs/gobs/sim/Rebuild.java index 829f57b..4c2a1df 100644 --- a/sim/gobs/gobs/sim/Rebuild.java +++ b/sim/gobs/gobs/sim/Rebuild.java @@ -2,6 +2,7 @@ package gobs.sim; // import java.text.DecimalFormat; +import java.math.BigInteger; import java.util.*; import gobs.util.*; @@ -45,8 +46,13 @@ public class Rebuild Set<Node> busy; /** + The Queue of work to do. + */ + Queue<Copy> queue; + + /** The Set of known Events that will finish in the future. - Contains inflight Copies and the next Fault. + Contains inflight Copies, the next Fault, and any Repairs. */ NavigableSet<Event> inflight = new TreeSet<Event>(); @@ -56,34 +62,33 @@ public class Rebuild */ Fault fault; - Rebuild(Simulator simulator) - { - this.simulator = simulator; - } - /** - Schedule the next fault. + @param fault The fault that may interrupt this Rebuild. */ - void nextFault(double t) + Rebuild(Simulator simulator, Fault fault, Set<Repair> repairs) { - fault = new Fault(t); + this.simulator = simulator; + this.fault = fault; inflight.add(fault); + inflight.addAll(repairs); } /** Actually perform the rebuild. + Moves prescribed Copies from copies to done. TODO: Handle cases where more than one copy is needed. @param time The time at which the copies start. + @param copies The Copies prescribed for this Rebuild + @param done Return the Copies completed here @return The time at which the next fault occurs */ - double rebuild(double time, Map<gobs.sim.Object,Copy> copies, - Set<gobs.sim.Object> todo, Set<gobs.sim.Object> done) + double rebuild(double time, Set<Copy> copies, Set<Copy> done) { - start = time; + start = time; schedule = new TreeSet<Event>(); busy = new HashSet<Node>(copies.size()*2); - Queue<Copy> queue = order(copies); - while (done.size() < copies.size()) + queue = order(copies); + while (! (queue.isEmpty() && inflight.isEmpty())) { Copy copy = select(queue, busy); if (copy != null) @@ -94,9 +99,6 @@ public class Rebuild busy.add(copy.destination); while (! inflight.add(copy)) { - // System.out.println("add failed!"); - // System.out.println("" + copy); - // dumpInflight(inflight); time += selectTime; copy.startAt(time); } @@ -109,7 +111,11 @@ public class Rebuild { if (event instanceof Copy) { - completeCopy(event, todo, done); + completeCopy(event, copies, done); + } + else if (event instanceof Repair) + { + completeRepair(event); } else if (event instanceof Fault) { @@ -129,6 +135,11 @@ public class Rebuild return fault.finish; } + double drain() + { + return 0; + } + Event pop() { Event result = inflight.pollFirst(); @@ -138,17 +149,102 @@ public class Rebuild } /** - Handle this Copy Event. Move copy from todo to done. + Handle this Copy Event. Move copy (event) from copies to done. @param event Actually a Copy */ - void completeCopy(Event event, Set<gobs.sim.Object> todo, - Set<gobs.sim.Object> done) + void completeCopy(Event event, Set<Copy> copies, Set<Copy> done) { Copy copy = (Copy) event; busy.remove(copy.source); busy.remove(copy.destination); - todo.remove(copy.object); - done.add(copy.object); + copies.remove(copy); + simulator.placer.copy(copy); + done.add(copy); + List<Copy> extras = validate(copy.object); + for (Copy extra : extras) + copies.add(extra); + } + + /** + Ensure that the replica count is restored. + Remove extraneous replicas or return a new Copy if needed. + @return The Copies needed. + */ + List<Copy> validate(gobs.sim.Object object) + { + List<Copy> result = new ArrayList<Copy>(); + + int replicas = object.replicas; + + List<Node> sites = simulator.placer.sites(object); + Set<Node> holders = simulator.scanStorage(object); + + Copy replicaCopy = ensureReplicas(object, holders, sites); + List<Copy> siteCopies = ensureSites(object, sites); + + removeExtras(object, holders, sites); + + if (replicaCopy != null) + result.add(replicaCopy); + result.addAll(siteCopies); + + return result; + } + + Copy ensureReplicas(gobs.sim.Object object, + Set<Node> holders, List<Node> sites) + { + if (holders.size() < sites.size()) + { + Copy copy = simulator.placer.regimen(object); + return copy; + } + return null; + } + + /** + Ensure that sites that should hold this object do. + */ + List<Copy> ensureSites(gobs.sim.Object object, List<Node> sites) + { + List<Copy> result = new ArrayList<Copy>(); + for (Node node : sites) + if (! node.contains(object.id)) + { + Copy copy = simulator.placer.regimen(object); + assert(copy != null); + result.add(copy); + } + return result; + } + + void removeExtras(gobs.sim.Object object, + Set<Node> holders, List<Node> sites) + { + while (holders.size() > sites.size()) + for (Iterator<Node> it = holders.iterator(); it.hasNext(); ) + { + Node node = it.next(); + if (! sites.contains(node)) + { + node.delete(object.id); + it.remove(); + } + } + } + + void completeRepair(Event event) + { + Repair repair = (Repair) event; + System.out.println(repair); + simulator.repairs.remove(repair); + Node node = repair.node; + Set<Copy> copies = node.reboot(); + for (Copy copy : copies) + { + copy.startAt(repair.finish+selectTime); + queue.add(copy); + } } /** @@ -157,7 +253,8 @@ public class Rebuild public int loadAt(double t) { int result = 0; - NavigableSet<Event> view = schedule.tailSet(new Event(t), false); + NavigableSet<Event> view = + schedule.tailSet(new Event(simulator, t), false); for (Iterator<Event> it = view.iterator(); it.hasNext(); ) { Copy copy = (Copy) it.next(); @@ -195,24 +292,24 @@ public class Rebuild } } - Queue<Copy> order(Map<gobs.sim.Object,Copy> copies) + /** + Randomly reorder the copies into a Queue. + */ + Queue<Copy> order(Set<Copy> copies) { - for (gobs.sim.Object object : copies.keySet()) - if (copies.get(object) == null) - { - System.out.println(object); - assert(false); - } - - LinkedList<Copy> result = new LinkedList<Copy>(copies.values()); + LinkedList<Copy> result = new LinkedList<Copy>(copies); Collections.shuffle(result, Bits.rng); return result; } - Copy select(Queue<Copy> todo, Set<Node> busy) + /** + Pull out a Copy to perform. + Ensure that the Nodes involved are not busy. + */ + Copy select(Queue<Copy> copies, Set<Node> busy) throws NoSuchElementException { - for (Iterator<Copy> it = todo.iterator(); it.hasNext(); ) + for (Iterator<Copy> it = copies.iterator(); it.hasNext(); ) { Copy copy = it.next(); assert(copy != null); diff --git a/sim/gobs/gobs/sim/Repair.java b/sim/gobs/gobs/sim/Repair.java new file mode 100644 index 0000000..340f63c --- /dev/null +++ b/sim/gobs/gobs/sim/Repair.java @@ -0,0 +1,32 @@ + +package gobs.sim; + +class Repair + extends Event +{ + Node node; + + Repair(Simulator simulator, Node node, double time) + { + super(simulator); + this.node = node; + double detect = simulator.mttrExponential.nextDouble(); + finish = time+detect; + } + + public int compareTo(Event event) + { + return new Double(finish).compareTo(new Double(event.finish)); + } + + String log() + { + return "Repair(" + node.toString(simulator) + "," + + df.format(finish) + ")"; + } + + public String toString() + { + return log(); + } +} diff --git a/sim/gobs/gobs/sim/Replicated.java b/sim/gobs/gobs/sim/Replicated.java index 9071d2b..f3ad5d9 100644 --- a/sim/gobs/gobs/sim/Replicated.java +++ b/sim/gobs/gobs/sim/Replicated.java @@ -103,17 +103,24 @@ public abstract class Replicated abstract Copy regimen(SingleObject object, List<Node> nodes); - /** - Locate a replica of the given object on one of the given nodes - in accordance with the {@link #replicaSource} policy; - remove and return that replica from sites. - */ Node removeSource(SingleObject object, List<Node> sites) { - Node result = null; + Node source = source(object, sites); + if (source == null) + return null; + sites.remove(source); + return source; + } + Node source(SingleObject object, List<Node> sites) + { + int p = sites.size()-1; + if (p < 0) + return null; + + Node result = null; if (replicaSource == ReplicaSource.PRIMARY) - result = sites.get(sites.size()-1); + result = sites.get(p); else if (replicaSource == ReplicaSource.LAST) result = lastSource(object, sites); @@ -121,11 +128,6 @@ public abstract class Replicated ! result.contains(object.id)) result = randomSource(object, sites); - if (result == null) - simulator.lost(object); - else - sites.remove(result); - return result; } diff --git a/sim/gobs/gobs/sim/SimpleNodes.java b/sim/gobs/gobs/sim/SimpleNodes.java index 11841aa..2c31f67 100644 --- a/sim/gobs/gobs/sim/SimpleNodes.java +++ b/sim/gobs/gobs/sim/SimpleNodes.java @@ -34,7 +34,7 @@ public class SimpleNodes BigInteger bigi = new BigInteger("" + i); BigInteger id = offset.add(bigi.multiply(fraction)).mod(bigM); - Node node = new Node(id); + Node node = new Node(simulator, id); if (nodes.contains(node)) Tools.bail("Could not assign unique node id!"); diff --git a/sim/gobs/gobs/sim/Simulator.java b/sim/gobs/gobs/sim/Simulator.java index ad23799..bb2d06f 100644 --- a/sim/gobs/gobs/sim/Simulator.java +++ b/sim/gobs/gobs/sim/Simulator.java @@ -31,6 +31,7 @@ public class Simulator int writes; double mttf; + double mttr; public Placement placer; public NodeFactory hub; @@ -39,7 +40,7 @@ public class Simulator /** Simulated time. Starts at 0. */ - double time = 0; + private double time = 0; /** Time at which to terminate, if set. @@ -59,7 +60,7 @@ public class Simulator /** Useful for helpful error messages: */ - String reading; + // String reading; /** Aggregate properties from all input files: @@ -87,11 +88,16 @@ public class Simulator LinkedHashMap<Node,Integer> sortedDiffs; /** - History of rebuilds. + History of Rebuilds. */ public List<Rebuild> rebuilds = new ArrayList<Rebuild>(); /** + Current Repairs in progress. + */ + Set<Repair> repairs = new HashSet<Repair>(); + + /** Current rebuild in progress. */ Rebuild currentRebuild = null; @@ -117,7 +123,13 @@ public class Simulator Source of exponentially distributed random numbers for MTTF samples. */ - Exponential exponential; + Exponential mttfExponential; + + /** + Source of exponentially distributed random numbers + for MTTF samples. + */ + Exponential mttrExponential; public Simulator() { @@ -221,7 +233,7 @@ public class Simulator writes = integer("writes"); setupTermination(); - setupFaults(); + setupExponentials(); } void setupTermination() @@ -242,12 +254,13 @@ public class Simulator } } - void setupFaults() + void setupExponentials() { - mttf = decimal("mttf"); - DRand engine = new DRand(Bits.nextInt()); - double lambda = 1.0/mttf; - exponential = new Exponential(lambda, engine); + mttf = decimal("mttf"); + mttr = decimal("mttr"); + DRand engine = new DRand(Bits.nextInt()); + mttfExponential = new Exponential(1.0/mttf, engine); + mttrExponential = new Exponential(1.0/mttr, engine); } void setupFileFactory() @@ -316,7 +329,7 @@ public class Simulator else bail("Bad NodeFactory name: " + nodesName); - nodes = new ArrayList<Node>(N); + nodes = new LinkedList<Node>(); } void initializeNodes() @@ -324,7 +337,7 @@ public class Simulator hub.generate(N, nodes); stats.nodes = N; - System.out.println("nodes: " + N); + System.out.println("NODES: " + N); if (bool("print.initialnodes")) printNodes(nodes); } @@ -460,50 +473,65 @@ public class Simulator return files.get(index); } + /** + Advance the Simulator to time. + */ + void advanceTo(double time) + { + Tools.check(this.time < time, + "Attempted to step backwards in time!"); + this.time = time; + System.out.println("time: " + time); + } + + /** + Return the current simulated time. + */ + double time() + { + return time; + } + void induceFaults() { System.out.println("faults: " + totalFaults); - Set<gobs.sim.Object> todo = - new LinkedHashSet<gobs.sim.Object>(); - Set<gobs.sim.Object> done = - new LinkedHashSet<gobs.sim.Object>(); - Map<gobs.sim.Object,Copy> copies = - new HashMap<gobs.sim.Object,Copy>(); - time = exponential.nextDouble(); + Set<Copy> copies = new LinkedHashSet<Copy>(); + Set<Copy> done = new LinkedHashSet<Copy>(); + // Advance to first fault: + advanceTo(mttfExponential.nextDouble()); while (! complete()) { Node failed = induceFault(); - todo.addAll(failed.collection()); - rebuild(todo, done, copies); - } - while (! todo.isEmpty()) - { - Node failed = induceFault(); - todo.addAll(failed.collection()); - rebuild(todo, done, copies); + Collection<gobs.sim.Object> damaged = failed.collection(); + rebuild(failed, damaged, copies, done); } + performCopies(copies); } - void rebuild(Set<gobs.sim.Object> todo, Set<gobs.sim.Object> done, - Map<gobs.sim.Object,Copy> copies) + void rebuild(Node node, Collection<gobs.sim.Object> damaged, + Set<Copy> copies, Set<Copy> done) { - startRebuild(); - copies.putAll(planCopies(todo)); - time = currentRebuild.rebuild(time, copies, todo, done); - performCopies(copies, done); + startRebuild(node); + List<Copy> work = planCopies(damaged); + copies.addAll(work); + double t = currentRebuild.rebuild(time, copies, done); + advanceTo(t); + performCopies(done); completeRebuild(); } /** Setup a Rebuild. */ - Rebuild startRebuild() + Rebuild startRebuild(Node node) { - double nextFault = time + exponential.nextDouble(); - Rebuild rebuild = new Rebuild(this); - currentRebuild = rebuild; - rebuild.nextFault(nextFault); System.out.println("fault: " + faultCount); + // Schedule the next fault: + Fault fault = new Fault(this); + Repair repair = new Repair(this, node, time); + repairs.add(repair); + Rebuild rebuild = new Rebuild(this, fault, repairs); + currentRebuild = rebuild; return rebuild; } @@ -515,8 +543,9 @@ public class Simulator { int index = Bits.nextInt(nodes.size()); Node failed = nodes.remove(index); + failed.status = Status.DOWN; faultCount++; - System.out.println("failed: " + failed.name(this) + + System.out.println("\nfailed: " + failed.name(this) + " (" + failed.objects.size() + ")" + " at: " + timef.format(time)); return failed; @@ -525,17 +554,16 @@ public class Simulator /** Build the Map of Copies to perform by calling {@link Placement#regimen}. - @param todo The Set of Objects that need to be copied + @param damaged The Set of Objects that need to be repaired */ - Map<gobs.sim.Object,Copy> planCopies(Set<gobs.sim.Object> todo) + List<Copy> planCopies(Collection<gobs.sim.Object> damaged) { - Map<gobs.sim.Object,Copy> result = - new HashMap<gobs.sim.Object,Copy>(); - for (gobs.sim.Object object : todo) + List<Copy> result = new ArrayList<Copy>(); + for (gobs.sim.Object object : damaged) { Copy copy = placer.regimen(object, nodes); if (copy != null) - result.put(object, copy); + result.add(copy); } return result; } @@ -546,19 +574,15 @@ public class Simulator @param copies Index of {@link Placement#regimen} output @param done Set of Objects that were copied */ - void performCopies(Map<gobs.sim.Object,Copy> copies, - Set<gobs.sim.Object> done) + void performCopies(Set<Copy> done) { - for (gobs.sim.Object object : done) - { - Copy copy = copies.remove(object); - placer.copy(copy); - } + // for (Copy copy : done) + // placer.copy(copy); done.clear(); } /** - Store current Rebuild; advance time to next fault. + Store current Rebuild and clean up. */ void completeRebuild() { @@ -568,6 +592,19 @@ public class Simulator printNodeObjects(nodes); } + /** + Scan all nodes and return the List of Nodes currently + holding the given object. + */ + Set<Node> scanStorage(gobs.sim.Object object) + { + Set<Node> holders = new LinkedHashSet<Node>(); + for (Node node : nodes) + if (node.contains(object.id)) + holders.add(node); + return holders; + } + void computeStatistics() { computeDiffs(); @@ -581,6 +618,7 @@ public class Simulator stats.results.setProperty("lostObjects", df.format(stats.lostObjects)); + System.out.println("FINAL_NODES: " + nodes.size()); System.out.println("FLEXIBILITIES: " + df.format(stats.avgFlexibility())); System.out.println("LOST_OBJECTS: " + stats.lostObjects); @@ -838,9 +876,8 @@ public class Simulator void printNodes(List<Node> list) { - System.out.println("NODES:"); for (Node node : list) - System.out.println(node.name(this)); + System.out.println("NODE: " + node.name(this)); } void printNodeObjects(List<Node> list) diff --git a/sim/gobs/gobs/sim/Status.java b/sim/gobs/gobs/sim/Status.java new file mode 100644 index 0000000..5ab87b0 --- /dev/null +++ b/sim/gobs/gobs/sim/Status.java @@ -0,0 +1,12 @@ + +package gobs.sim; + +/** + Node status. +*/ + +enum Status +{ + UP, DOWN +} + diff --git a/sim/gobs/gobs/sweep/AbstractSweep.java b/sim/gobs/gobs/sweep/AbstractSweep.java index f43a4c5..ba7ac89 100644 --- a/sim/gobs/gobs/sweep/AbstractSweep.java +++ b/sim/gobs/gobs/sweep/AbstractSweep.java @@ -12,12 +12,12 @@ import gobs.util.*; * Reusable functions for parameter sweeps. * */ -abstract class AbstractSweep +public abstract class AbstractSweep { - Properties properties = new Properties(); + public Properties properties = new Properties(); List<String> values; - int repeats; + public int repeats; static String path = null; static Date launch = new Date(); @@ -40,7 +40,7 @@ abstract class AbstractSweep repeat count from command line, initialize random number generator, etc. */ - static Properties getopts(String[] args) + public static Properties getopts(String[] args) { Properties result = new Properties(); @@ -96,7 +96,7 @@ abstract class AbstractSweep path = p; } - static List<String> values(String step, int min, int max) + public static List<String> values(String step, int min, int max) { if (step.equals("all")) return all(min,max); @@ -128,7 +128,7 @@ abstract class AbstractSweep Sweep over all integer values between min and max that are multiples of 10. */ - static List<String> tens(int min, int max) + public static List<String> tens(int min, int max) { List<String> result = new ArrayList<String>(max-min+1); result.add(new Integer(min).toString()); @@ -144,7 +144,7 @@ abstract class AbstractSweep Sweep over all integer values between min and max that are the product of min and 2^i for some i. */ - static List<String> exps(int min, int max) + public static List<String> exps(int min, int max) { List<String> result = new ArrayList<String>(); @@ -166,7 +166,7 @@ abstract class AbstractSweep are the product of min and 2^i for some i, and the values between those. */ - static List<String> mids(int min, int max) + public static List<String> mids(int min, int max) { List<String> result = new ArrayList<String>(); @@ -250,7 +250,7 @@ abstract class AbstractSweep /** Seed the random number generator based on the "seed" property. */ - static void seed(Properties properties) + public static void seed(Properties properties) { if (Tools.has(properties, "seed")) Bits.init(Tools.integer(properties, "seed")); diff --git a/sim/gobs/gobs/sweep/Affected01.java b/sim/gobs/gobs/sweep/Affected01.java deleted file mode 100644 index f18efe2..0000000 --- a/sim/gobs/gobs/sweep/Affected01.java +++ /dev/null @@ -1,38 +0,0 @@ - -package gobs.sweep; - -import java.util.*; - -import gobs.sim.*; -import gobs.util.Tools; -import gobs.plot.SimplePlot; - -/** - * Modifies node count. - * Reports fraction of servers involved in rebuild. - * - * Usage: Affected01 -n <repeats> gobs.cfg params.cfg Decluster01.cfg - * */ - -class Affected01 - extends Sweep -{ - Affected01(String[] args) - { - super(args); - } - - public static void main(String[] args) - { - Affected01 sweep = new Affected01(args); - - int minNodes = Tools.integer(sweep.properties, "minNodes"); - int maxNodes = Tools.integer(sweep.properties, "maxNodes"); - List<String> values = Sweep.tens(minNodes, maxNodes); - - sweep.execute("Affected", "nodes", values, "affected"); - - SimplePlot.plot(sweep.series, "Number of Affected Servers", - "nodes", "load", "Affected01.eps"); - } -} diff --git a/sim/gobs/gobs/sweep/ArraySweep.java b/sim/gobs/gobs/sweep/ArraySweep.java index 18a3930..83a4389 100644 --- a/sim/gobs/gobs/sweep/ArraySweep.java +++ b/sim/gobs/gobs/sweep/ArraySweep.java @@ -13,25 +13,24 @@ import gobs.util.*; * Returns a set of XYSeriesCollection * */ -class ArraySweep +public class ArraySweep extends AbstractSweep { Map<String,Simulator> sims = new Hashtable<String,Simulator>(); List<Statistics> stats = new Vector<Statistics>(); - List<XYSeries> collection = new ArrayList<XYSeries>(); static String path = null; static Date launch = new Date(); - ArraySweep() + public ArraySweep() {} - ArraySweep(Properties properties) + public ArraySweep(Properties properties) { super(properties); } - ArraySweep(String[] args) + public ArraySweep(String[] args) { super(args); } @@ -42,10 +41,10 @@ class ArraySweep properties object. Run the simulation and extract the yname property for the plot. */ - void execute(String description, String xname, - List<String> values, String yname) + public List<XYSeries> execute(String description, String xname, + List<String> values, String yname) { - execute(description, xname, values, null, yname); + return execute(description, xname, values, null, yname); } /** @@ -61,9 +60,12 @@ class ArraySweep @param yname The property name to read to obtain y coordinate values for the output series */ - void execute(String description, String xname, - List<String> values, String dname, String yname) + public List<XYSeries> execute(String description, String xname, + List<String> values, String dname, + String yname) { + List<XYSeries> collection = new ArrayList<XYSeries>(); + repeats = Tools.integer(properties, "repeats"); System.out.println("repeats: " + repeats); @@ -96,6 +98,7 @@ class ArraySweep arrayAvgToSeries(longest(xresult), yresult, series); collection.add(series); } + return collection; } /** diff --git a/sim/gobs/gobs/sweep/Balance01.java b/sim/gobs/gobs/sweep/Balance01.java deleted file mode 100644 index ce0bd5e..0000000 --- a/sim/gobs/gobs/sweep/Balance01.java +++ /dev/null @@ -1,77 +0,0 @@ - -package gobs.sweep; - -import java.util.*; - -import org.jfree.data.xy.XYSeries; -import org.jfree.data.xy.XYSeriesCollection; - -import gobs.sim.*; -import gobs.util.*; -import gobs.plot.*; - -/** - * Reports object load on each server. Sorted. - * - * Usage: Balance01 gobs.cfg params.cfg - * */ - -class Balance01 -{ - public static void main(String[] args) - { - Bits.init(true); - - Properties properties = Sweep.getopts(args); - properties.setProperty("faults", "0"); - - String output = Tools.string(properties, "output"); - - final XYSeriesCollection collection = new XYSeriesCollection(); - - String[] schemes = - /* new String[]{ "Nearest", "Kademlia", "VertexGroups", - "RUSHp" }; */ - new String[]{ "ParallelRUSH" }; - // new String[]{ "Nearest" }; - for (String scheme : schemes) - { - Simulator simulator = execute(properties, scheme); - XYSeries series = extract(scheme, simulator); - collection.addSeries(series); - } - - MultiplePlot.plot(collection, "Load by Node", - "node", "load", output); - - /* - AsciiPlot.plot(collection, "Load by Node", - "nodes", "load", output); - */ - } - - static Simulator execute(Properties properties, String scheme) - { - properties.setProperty("impl.placement", scheme); - Simulator simulator = new Simulator(); - simulator.initialize(properties); - simulator.execute(); - - return simulator; - } - - static XYSeries extract(String scheme, Simulator simulator) - { - XYSeries series = new XYSeries(scheme); - series.setDescription(scheme); - - MapSorter<Node> sorter = new MapSorter<Node>(); - Map<Node,Integer> sorted = sorter.sort(simulator.origCount); - - int i = 0; - for (Node node : sorted.keySet()) - series.add(i++, sorted.get(node)); - - return series; - } -} diff --git a/sim/gobs/gobs/sweep/Balance02.java b/sim/gobs/gobs/sweep/Balance02.java deleted file mode 100644 index 5363bd5..0000000 --- a/sim/gobs/gobs/sweep/Balance02.java +++ /dev/null @@ -1,66 +0,0 @@ - -package gobs.sweep; - -import java.math.BigInteger; -import java.util.*; - -import org.jfree.data.xy.XYSeries; - -import gobs.sim.*; -import gobs.util.*; -import gobs.plot.SimplePlot; - -/** - * Reports object load on each server. Unsorted. - * - * Usage: Balance02 -n <repeats> gobs.cfg params.cfg - * */ - -class Balance02 -{ - public static void main(String[] args) - { - Bits.init(); - - Properties properties = Sweep.getopts(args); - properties.setProperty("faults", "0"); - - Simulator simulator = new Simulator(); - simulator.initialize(properties); - simulator.execute(); - - XYSeries series = new XYSeries("Balance"); - - Metric metric = (Metric) simulator.placer; - - int i = 0; - for (Node node : simulator.nodes) - { - series.add(i++, node.objects.size()); - - Node closestNode = null; - BigInteger closestDistance = simulator.M; - for (Node neighbor : simulator.nodes) - { - if (neighbor != node) - { - BigInteger distance = metric.distance(node.id,neighbor.id); - if (distance.compareTo(closestDistance) < 0) - { - closestNode = neighbor; - closestDistance = distance; - } - } - } - - System.out.println - ("i: " + i + - " node: " + node + - " load: " + node.objects.size() + - "\n\t distance: " + closestDistance); - } - - SimplePlot.plot(series, "Load Distribution", "nodes", "load", - "Balance02.eps"); - } -} diff --git a/sim/gobs/gobs/sweep/Decluster01.java b/sim/gobs/gobs/sweep/Decluster01.java deleted file mode 100644 index 12c3ac1..0000000 --- a/sim/gobs/gobs/sweep/Decluster01.java +++ /dev/null @@ -1,33 +0,0 @@ - -package gobs.sweep; - -import java.util.*; - -import gobs.plot.SimplePlot; -import gobs.util.*; - -/** - * Modifies node count. Reports object movement load on the - * heaviest server during rebuild. - * - * Usage: Decluster01 -n <repeats> gobs.cfg params.cfg Decluster01.cfg - * */ - -class Decluster01 -{ - public static void main(String[] args) - { - Bits.init(); - - Sweep sweep = new Sweep(args); - - int minNodes = Tools.integer(sweep.properties, "minNodes"); - int maxNodes = Tools.integer(sweep.properties, "maxNodes"); - List<String> values = Sweep.exps(minNodes, maxNodes); - - sweep.execute("Declustering", "nodes", values, "maxPulls"); - - SimplePlot.plot(sweep.series, "Maximum Rebuild Load", "nodes", "load", - "Decluster01.eps"); - } -} diff --git a/sim/gobs/gobs/sweep/Decluster01P.java b/sim/gobs/gobs/sweep/Decluster01P.java deleted file mode 100644 index 4387e61..0000000 --- a/sim/gobs/gobs/sweep/Decluster01P.java +++ /dev/null @@ -1,33 +0,0 @@ - -package gobs.sweep; - -import java.util.*; - -import gobs.plot.SimplePlot; -import gobs.util.*; - -/** - * Modifies node count. Reports object movement load on the - * heaviest server during rebuild. - * - * Usage: Decluster01 -n <repeats> gobs.cfg params.cfg Decluster01.cfg - * */ - -class Decluster01P -{ - public static void main(String[] args) - { - Bits.init(); - - Sweep sweep = new ParallelSweep(args); - - int minNodes = Tools.integer(sweep.properties, "minNodes"); - int maxNodes = Tools.integer(sweep.properties, "maxNodes"); - List<String> values = Sweep.tens(minNodes, maxNodes); - - sweep.execute("Declustering", "nodes", values, "maxPulls"); - - SimplePlot.plot(sweep.series, "Maximum Rebuild Load", "nodes", "load", - "Decluster01.eps"); - } -} diff --git a/sim/gobs/gobs/sweep/Decluster02.java b/sim/gobs/gobs/sweep/Decluster02.java deleted file mode 100644 index 59cad0f..0000000 --- a/sim/gobs/gobs/sweep/Decluster02.java +++ /dev/null @@ -1,69 +0,0 @@ - -package gobs.sweep; - -import java.util.*; - -import org.jfree.data.xy.XYSeries; -import org.jfree.data.xy.XYSeriesCollection; - -import gobs.util.*; -import gobs.plot.*; - -/** - * Modifies node count. Reports object movement load on the - * heaviest server during rebuild. - * - * Requires properties: - * - * output - EPS target output - * minNodes, maxNodes - * step - For Sweep.values() - * repeats - Number of repetitions for average, may be set with -n. - * - * Usage: Decluster02 -n <repeats> *.cfg - * */ - -class Decluster02 -{ - static Properties properties; - - public static void main(String[] args) - { - properties = Sweep.getopts(args); - AbstractSweep.seed(properties); - - String output = Tools.string(properties, "output"); - int minNodes = Tools.integer(properties, "minNodes"); - int maxNodes = Tools.integer(properties, "maxNodes"); - String step = Tools.string(properties, "step"); - List<String> values = Sweep.values(step, minNodes, maxNodes); - - final XYSeriesCollection collection = new XYSeriesCollection(); - - properties.setProperty("impl.placement", "Nearest"); - Sweep nearestSweep = new Sweep(properties); - nearestSweep.repeats = Tools.integer(properties, "repeats"); - nearestSweep.execute("PAST", "nodes", values, "maxPulls"); - collection.addSeries(nearestSweep.series); - - properties.setProperty("impl.placement", "Kademlia"); - Sweep kademliaSweep = new Sweep(properties); - kademliaSweep.repeats = Tools.integer(properties, "repeats"); - kademliaSweep.execute("Kademlia", "nodes", values, - "maxPulls"); - collection.addSeries(kademliaSweep.series); - - properties.setProperty("impl.placement", "VertexGroups"); - Sweep vgSweep = new Sweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "maxPulls"); - collection.addSeries(vgSweep.series); - - MultiLogXPlot.plot(collection, "Maximum Rebuild Load", - "nodes", "load fraction (%)", output); - // AsciiPlot.plot(nearestSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); - - Sweep.reportRuntime(); - } -} diff --git a/sim/gobs/gobs/sweep/Decluster03.java b/sim/gobs/gobs/sweep/Decluster03.java deleted file mode 100644 index 7812a69..0000000 --- a/sim/gobs/gobs/sweep/Decluster03.java +++ /dev/null @@ -1,64 +0,0 @@ - -package gobs.sweep; - -import java.util.*; - -import org.jfree.data.xy.XYSeries; -import org.jfree.data.xy.XYSeriesCollection; - -import gobs.util.*; -import gobs.plot.*; - -/** - * Modifies node count. - * Reports object movement load (pushes+pulls) on the - * busiest server during rebuild. - * - * Usage: Decluster03 -n <repeats> *.cfg - * */ - -class Decluster03 -{ - static Properties properties; - - public static void main(String[] args) - { - Bits.init(); - - properties = Sweep.getopts(args); - - int minNodes = Tools.integer(properties, "minNodes"); - int maxNodes = Tools.integer(properties, "maxNodes"); - String step = Tools.string(properties, "step"); - List<String> values = Sweep.values(step, minNodes, maxNodes); - - System.out.println("Decluster03: " + values); - - final XYSeriesCollection collection = new XYSeriesCollection(); - - properties.setProperty("impl.placement", "Nearest"); - Sweep nearestSweep = new Sweep(properties); - nearestSweep.repeats = Tools.integer(properties, "repeats"); - nearestSweep.execute("PAST", "nodes", values, "maxTotal"); - collection.addSeries(nearestSweep.series); - - properties.setProperty("impl.placement", "Kademlia"); - Sweep kademliaSweep = new Sweep(properties); - kademliaSweep.repeats = Tools.integer(properties, "repeats"); - kademliaSweep.execute("Kademlia", "nodes", values, - "maxTotal"); - collection.addSeries(kademliaSweep.series); - - properties.setProperty("impl.placement", "VertexGroups"); - Sweep vgSweep = new Sweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "maxTotal"); - collection.addSeries(vgSweep.series); - - String output = Tools.string(properties, "output"); - MultiLogXPlot.plot(collection, "Maximum Rebuild Load", - "nodes", "load fraction(%)", output); - // AsciiPlot.plot(nearestSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); - } -} diff --git a/sim/gobs/gobs/sweep/Decluster04.java b/sim/gobs/gobs/sweep/Decluster04.java deleted file mode 100644 index 541633b..0000000 --- a/sim/gobs/gobs/sweep/Decluster04.java +++ /dev/null @@ -1,69 +0,0 @@ - -package gobs.sweep; - -import java.util.*; - -import org.jfree.data.xy.XYSeries; -import org.jfree.data.xy.XYSeriesCollection; - -import gobs.util.*; -import gobs.plot.*; - -/** - * Modifies node count. - * Reports object movement load (pushes+pulls) on all servers - * during rebuild. - * - * Usage: Decluster04 -n <repeats> *.cfg - * */ - -class Decluster04 -{ - static Properties properties; - - public static void main(String[] args) - { - Bits.init(); - - properties = Sweep.getopts(args); - - String output = Tools.string(properties, "output"); - - int minNodes = Tools.integer(properties, "minNodes"); - int maxNodes = Tools.integer(properties, "maxNodes"); - String step = Tools.string(properties, "step"); - List<String> values = Sweep.values(step, minNodes, maxNodes); - - System.out.println("Decluster04: " + values); - - List<XYSeries> allSeries = new ArrayList<XYSeries>(); - final XYSeriesCollection collection = new XYSeriesCollection(); - - properties.setProperty("impl.placement", "Nearest"); - ArraySweep pSweep = new ArraySweep(properties); - pSweep.repeats = Tools.integer(properties, "repeats"); - pSweep.execute("PAST", "nodes", values, "distribution"); - allSeries.addAll(pSweep.collection); - - properties.setProperty("impl.placement", "Kademlia"); - ArraySweep kSweep = new ArraySweep(properties); - kSweep.repeats = Tools.integer(properties, "repeats"); - kSweep.execute("Kademlia", "nodes", values, "distribution"); - allSeries.addAll(kSweep.collection); - - properties.setProperty("impl.placement", "VertexGroups"); - ArraySweep vgSweep = new ArraySweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); - allSeries.addAll(vgSweep.collection); - - for (XYSeries series : allSeries) - collection.addSeries(series); - - MultiplePlot.plot(collection, "Rebuild Load Distribution", - "nodes involved", "load fraction(%)", output); - - // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); - } -} diff --git a/sim/gobs/gobs/sweep/ParallelSweep.java b/sim/gobs/gobs/sweep/ParallelSweep.java index b11b523..b698c12 100644 --- a/sim/gobs/gobs/sweep/ParallelSweep.java +++ b/sim/gobs/gobs/sweep/ParallelSweep.java @@ -61,8 +61,8 @@ class ParallelSweep properties object. Run the simulation and extract the yname property for the plot. */ - void execute(String description, String xname, - List<String> values, String yname) + public XYSeries execute(String description, String xname, + List<String> values, String yname) { repeats = Tools.integer(properties, "repeats"); @@ -73,7 +73,7 @@ class ParallelSweep ExecutorService service = Executors.newFixedThreadPool(cores); - series = new XYSeries(description); + XYSeries series = new XYSeries(description); for (String v : values) { double total = 0.0; @@ -125,6 +125,7 @@ class ParallelSweep System.out.println("avg: " + total/repeats); series.add(Double.parseDouble(v), total/repeats); } + return series; } /** diff --git a/sim/gobs/gobs/sweep/Parallelism01.java b/sim/gobs/gobs/sweep/Parallelism01.java deleted file mode 100644 index ca5d378..0000000 --- a/sim/gobs/gobs/sweep/Parallelism01.java +++ /dev/null @@ -1,63 +0,0 @@ - -package gobs.sweep; - -import java.util.*; - -import org.jfree.data.xy.XYSeries; -import org.jfree.data.xy.XYSeriesCollection; - -import gobs.util.*; -import gobs.plot.*; - -/** - * Modifies node count N and Placement. - * Reports file access load balancing for files and replicas. - * - * Usage: Parallelism01 -n <repeats> *.cfg - * */ - -class Parallelism01 -{ - static Properties properties; - - public static void main(String[] args) - { - Bits.init(); - - properties = Sweep.getopts(args); - - String output = Tools.string(properties, "output"); - - int minNodes = Tools.integer(properties, "minNodes"); - int maxNodes = Tools.integer(properties, "maxNodes"); - String step = Tools.string(properties, "step"); - List<String> values = Sweep.values(step, minNodes, maxNodes); - - System.out.println("Parallelism01: " + values); - - final XYSeriesCollection collection = new XYSeriesCollection(); - - properties.setProperty("impl.placement", "Nearest"); - Sweep nearestSweep = new Sweep(properties); - nearestSweep.repeats = Tools.integer(properties, "repeats"); - nearestSweep.execute("PAST", "nodes", values, "accessbalance"); - collection.addSeries(nearestSweep.series); - - properties.setProperty("impl.placement", "Kademlia"); - Sweep kademliaSweep = new Sweep(properties); - kademliaSweep.repeats = Tools.integer(properties, "repeats"); - kademliaSweep.execute("Kademlia", "nodes", values, "accessbalance"); - collection.addSeries(kademliaSweep.series); - - properties.setProperty("impl.placement", "VertexGroups"); - Sweep vgSweep = new Sweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "accessbalance"); - collection.addSeries(vgSweep.series); - - MultiLogXPlot.plot(collection, "Load Imbalance", - "nodes", "imbalance", output); - // AsciiPlot.plot(nearestSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); - } -} diff --git a/sim/gobs/gobs/sweep/Rebuild01.java b/sim/gobs/gobs/sweep/Rebuild01.java deleted file mode 100644 index b23f171..0000000 --- a/sim/gobs/gobs/sweep/Rebuild01.java +++ /dev/null @@ -1,74 +0,0 @@ - -package gobs.sweep; - -import java.util.*; - -import org.jfree.data.xy.XYSeries; -import org.jfree.data.xy.XYSeriesCollection; - -import gobs.util.*; -import gobs.plot.*; - -/** - * Modifies node count. - * Reports rebuild activity over time during rebuild. - * - * Usage: Rebuild01 -n <repeats> *.cfg - * */ - -// good runs: -// seed=828 -n 1 - -class Rebuild01 -{ - static Properties properties; - - public static void main(String[] args) - { - properties = Sweep.getopts(args); - - AbstractSweep.seed(properties); - - String output = Tools.string(properties, "output"); - - int minNodes = Tools.integer(properties, "nodes.min"); - int maxNodes = Tools.integer(properties, "nodes.max"); - String step = Tools.string(properties, "nodes.step"); - List<String> values = Sweep.values(step, minNodes, maxNodes); - - System.out.println("Rebuild01: " + values); - - List<XYSeries> allSeries = new ArrayList<XYSeries>(); - final XYSeriesCollection collection = new XYSeriesCollection(); - - properties.setProperty("impl.placement", "Nearest"); - ArraySweep pSweep = new ArraySweep(properties); - pSweep.repeats = Tools.integer(properties, "repeats"); - pSweep.execute("PAST", "nodes", values, "traffic-x", "traffic-y"); - allSeries.addAll(pSweep.collection); - - /* - properties.setProperty("impl.placement", "Kademlia"); - ArraySweep kSweep = new ArraySweep(properties); - kSweep.repeats = Tools.integer(properties, "repeats"); - kSweep.execute("Kademlia", "nodes", values, "distribution"); - allSeries.addAll(kSweep.collection); - - properties.setProperty("impl.placement", "VertexGroups"); - ArraySweep vgSweep = new ArraySweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); - allSeries.addAll(vgSweep.collection); - */ - - for (XYSeries series : allSeries) - collection.addSeries(series); - - MultiplePlot.plot(collection, "Rebuild Traffic", - "time", "copies in flight", - output); - - // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); - } -} diff --git a/sim/gobs/gobs/sweep/Rebuild02.java b/sim/gobs/gobs/sweep/Rebuild02.java deleted file mode 100644 index bd1a9fd..0000000 --- a/sim/gobs/gobs/sweep/Rebuild02.java +++ /dev/null @@ -1,126 +0,0 @@ - -package gobs.sweep; - -import java.util.*; - -import org.jfree.data.xy.XYSeries; -import org.jfree.data.xy.XYSeriesCollection; - -import gobs.sim.*; -import gobs.util.*; -import gobs.plot.*; - -/** - * Modifies node count. - * Reports rebuild activity over time during rebuild. - * - * Usage: Rebuild02 -n <repeats> -o <output.eps> *.cfg - * */ - -class Rebuild02 -{ - static Properties properties; - - public static void main(String[] args) - { - properties = Sweep.getopts(args); - - AbstractSweep.seed(properties); - - String output = Tools.string(properties, "output"); - double interval = Tools.decimal(properties, - "plot.traffic.interval"); - - int minNodes = Tools.integer(properties, "nodes.min"); - int maxNodes = Tools.integer(properties, "nodes.max"); - String step = Tools.string(properties, "nodes.step"); - List<String> values = Sweep.values(step, minNodes, maxNodes); - - System.out.println("Rebuild02: " + values); - - final XYSeriesCollection collection = new XYSeriesCollection(); - - properties.setProperty("impl.placement", "Nearest"); - Sweep pSweep = new Sweep(properties); - pSweep.repeats = Tools.integer(properties, "repeats"); - pSweep.execute("nodes", values); - - List<XYSeries> allSeries = new ArrayList<XYSeries>(); - - List<XYSeries> pSeries = plots(pSweep, "PAST", "nodes", - values, interval); - allSeries.addAll(pSeries); - - /* - properties.setProperty("impl.placement", "Kademlia"); - ArraySweep kSweep = new ArraySweep(properties); - kSweep.repeats = Tools.integer(properties, "repeats"); - kSweep.execute("Kademlia", "nodes", values, "distribution"); - allSeries.addAll(kSweep.collection); - - properties.setProperty("impl.placement", "VertexGroups"); - ArraySweep vgSweep = new ArraySweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); - allSeries.addAll(vgSweep.collection); - */ - - for (XYSeries series : allSeries) - collection.addSeries(series); - - MultiplePlot.plot(collection, "Rebuild Traffic", - "time", "copies in flight", - output); - - // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); - } - - static List<XYSeries> plots(Sweep sweep, String label, - String xname, List<String> values, - double interval) - { - List<XYSeries> results = new ArrayList<XYSeries>(); - - for (String v : values) - { - XYSeries series = new XYSeries(label + " N=" + v); - List<Rebuild> rebuilds = new ArrayList<Rebuild>(); - for (Simulator simulator : sweep.sims.values()) - { - if (Tools.string(simulator.properties, xname).equals(v)) - { - rebuilds.addAll(simulator.rebuilds); - } - } - loadsToSeries(series, rebuilds, interval); - results.add(series); - } - return results; - } - - static void loadsToSeries(XYSeries series, List<Rebuild> rebuilds, - double interval) - { - double longest = longestRebuild(rebuilds); - double d = 0.0; - while (d <= longest) - { - double total = 0.0; - for (Rebuild rebuild : rebuilds) - total += rebuild.loadOffset(d); - double sample = total/rebuilds.size(); - series.add(d, sample); - d += interval; - } - } - - static double longestRebuild(List<Rebuild> rebuilds) - { - double max = 0.0; - for (Rebuild rebuild : rebuilds) - if (rebuild.length() > max) - max = rebuild.length(); - return max; - } -} diff --git a/sim/gobs/gobs/sweep/Sweep.java b/sim/gobs/gobs/sweep/Sweep.java index 85525f1..4511bd1 100644 --- a/sim/gobs/gobs/sweep/Sweep.java +++ b/sim/gobs/gobs/sweep/Sweep.java @@ -14,25 +14,23 @@ import gobs.util.*; * Experiments that subclass this class * */ -class Sweep +public class Sweep extends AbstractSweep { - Map<String,Simulator> sims = new Hashtable<String,Simulator>(); - XYSeries series; + public Map<String,Simulator> sims = + new Hashtable<String,Simulator>(); - int repeats; - - Sweep() + public Sweep() { super(); } - Sweep(Properties properties) + public Sweep(Properties properties) { super(properties); } - Sweep(String[] args) + public Sweep(String[] args) { super(args); } @@ -45,7 +43,7 @@ class Sweep @param xname The property that will take the varying value. @param values The values to plug into property xname. */ - void execute(String xname, List<String> values) + public void execute(String xname, List<String> values) { repeats = Tools.integer(properties, "repeats"); @@ -67,24 +65,73 @@ class Sweep } /** + Execute the 2D parameter sweep. + For each value in values, substitute it for name in the + properties object. Run the simulation and store it in + sims under String "name1=value1;name2=value2". + @param name1 The property that will take the values from values1. + @param values1 The values to plug into property name1. + @param name2 The property that will take the values from values2. + @param values2 The values to plug into property name2. + */ + public void execute(String name1, List<String> values1, + String name2, List<String> values2) + { + repeats = Tools.integer(properties, "repeats"); + + if (repeats <= 0) + System.out.println("WARNING: repeats=" + repeats); + + for (String v1 : values1) + { + for (String v2 : values2) + { + for (int i = 0; i < repeats; i++) + { + execute(name1,v1,name2,v2,i); + } + } + } + } + + /** + Helper for execute(). + */ + void execute(String name1, String value1, + String name2, String value2, int unique) + { + String tag = name1+"="+value1+";"+name2+"="+value2+ + ";unique="+unique; + System.out.println("execute: " + tag); + properties.setProperty("tag", tag); + properties.setProperty("unique", new Integer(unique).toString()); + properties.setProperty(name1, value1); + properties.setProperty(name2, value2); + Simulator simulator = new Simulator(properties); + simulator.execute(); + sims.put(tag, simulator); + } + + /** Execute the parameter sweep. For each value in values, substitute it for xname in the properties object. Run the simulation and extract the yname property for the plot. - @param description Tag passed into plot legend. - @param xname The property that will take the varying value. - @param values The values to plug into property xname. - @param yname The property to extract into {@link #series}. + @param description Tag passed into plot legend + @param xname The property that will take the varying value + @param values The values to plug into property xname + @param yname The property to extract into the result + @return The series (values,yname(values)) */ - void execute(String description, String xname, - List<String> values, String yname) + public XYSeries execute(String description, String xname, + List<String> values, String yname) { repeats = Tools.integer(properties, "repeats"); if (repeats <= 0) System.out.println("WARNING: repeats=" + repeats); - series = new XYSeries(description); + XYSeries series = new XYSeries(description); for (String v : values) { double total = 0.0; @@ -101,6 +148,7 @@ class Sweep System.out.println("avg: " + total/repeats); series.add(Double.parseDouble(v), total/repeats); } + return series; } /** @@ -114,7 +162,7 @@ class Sweep return Double.parseDouble(result); } - static void reportRuntime() + public static void reportRuntime() { long t = (new Date().getTime()-launch.getTime())/(60*1000); System.out.println("took " + t + " minutes"); diff --git a/sim/gobs/gobs/sweep/x/Affected01.java b/sim/gobs/gobs/sweep/x/Affected01.java new file mode 100644 index 0000000..4f2b4aa --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Affected01.java @@ -0,0 +1,42 @@ + +package gobs.sweep.x; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; + +import gobs.sim.*; +import gobs.sweep.*; +import gobs.util.Tools; +import gobs.plot.SimplePlot; + +/** + * Modifies node count. + * Reports fraction of servers involved in rebuild. + * + * Usage: Affected01 -n <repeats> gobs.cfg params.cfg Decluster01.cfg + * */ + +class Affected01 + extends Sweep +{ + Affected01(String[] args) + { + super(args); + } + + public static void main(String[] args) + { + Affected01 sweep = new Affected01(args); + + int minNodes = Tools.integer(sweep.properties, "minNodes"); + int maxNodes = Tools.integer(sweep.properties, "maxNodes"); + List<String> values = Sweep.tens(minNodes, maxNodes); + + XYSeries series = + sweep.execute("Affected", "nodes", values, "affected"); + + SimplePlot.plot(series, "Number of Affected Servers", + "nodes", "load", "Affected01.eps"); + } +} diff --git a/sim/gobs/gobs/sweep/x/Balance01.java b/sim/gobs/gobs/sweep/x/Balance01.java new file mode 100644 index 0000000..5f1e12f --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Balance01.java @@ -0,0 +1,78 @@ + +package gobs.sweep.x; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.sim.*; +import gobs.sweep.*; +import gobs.util.*; +import gobs.plot.*; + +/** + * Reports object load on each server. Sorted. + * + * Usage: Balance01 gobs.cfg params.cfg + * */ + +class Balance01 +{ + public static void main(String[] args) + { + Bits.init(true); + + Properties properties = Sweep.getopts(args); + properties.setProperty("faults", "0"); + + String output = Tools.string(properties, "output"); + + final XYSeriesCollection collection = new XYSeriesCollection(); + + String[] schemes = + /* new String[]{ "Nearest", "Kademlia", "VertexGroups", + "RUSHp" }; */ + new String[]{ "ParallelRUSH" }; + // new String[]{ "Nearest" }; + for (String scheme : schemes) + { + Simulator simulator = execute(properties, scheme); + XYSeries series = extract(scheme, simulator); + collection.addSeries(series); + } + + MultiplePlot.plot(collection, "Load by Node", + "node", "load", output); + + /* + AsciiPlot.plot(collection, "Load by Node", + "nodes", "load", output); + */ + } + + static Simulator execute(Properties properties, String scheme) + { + properties.setProperty("impl.placement", scheme); + Simulator simulator = new Simulator(); + simulator.initialize(properties); + simulator.execute(); + + return simulator; + } + + static XYSeries extract(String scheme, Simulator simulator) + { + XYSeries series = new XYSeries(scheme); + series.setDescription(scheme); + + MapSorter<Node> sorter = new MapSorter<Node>(); + Map<Node,Integer> sorted = sorter.sort(simulator.origCount); + + int i = 0; + for (Node node : sorted.keySet()) + series.add(i++, sorted.get(node)); + + return series; + } +} diff --git a/sim/gobs/gobs/sweep/x/Balance02.java b/sim/gobs/gobs/sweep/x/Balance02.java new file mode 100644 index 0000000..3b35272 --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Balance02.java @@ -0,0 +1,67 @@ + +package gobs.sweep.x; + +import java.math.BigInteger; +import java.util.*; + +import org.jfree.data.xy.XYSeries; + +import gobs.sim.*; +import gobs.sweep.*; +import gobs.util.*; +import gobs.plot.SimplePlot; + +/** + * Reports object load on each server. Unsorted. + * + * Usage: Balance02 -n <repeats> gobs.cfg params.cfg + * */ + +class Balance02 +{ + public static void main(String[] args) + { + Bits.init(); + + Properties properties = Sweep.getopts(args); + properties.setProperty("faults", "0"); + + Simulator simulator = new Simulator(); + simulator.initialize(properties); + simulator.execute(); + + XYSeries series = new XYSeries("Balance"); + + Metric metric = (Metric) simulator.placer; + + int i = 0; + for (Node node : simulator.nodes) + { + series.add(i++, node.objects.size()); + + Node closestNode = null; + BigInteger closestDistance = simulator.M; + for (Node neighbor : simulator.nodes) + { + if (neighbor != node) + { + BigInteger distance = metric.distance(node.id,neighbor.id); + if (distance.compareTo(closestDistance) < 0) + { + closestNode = neighbor; + closestDistance = distance; + } + } + } + + System.out.println + ("i: " + i + + " node: " + node + + " load: " + node.objects.size() + + "\n\t distance: " + closestDistance); + } + + SimplePlot.plot(series, "Load Distribution", "nodes", "load", + "Balance02.eps"); + } +} diff --git a/sim/gobs/gobs/sweep/x/DataLoss01.java b/sim/gobs/gobs/sweep/x/DataLoss01.java new file mode 100644 index 0000000..6f5a66c --- /dev/null +++ b/sim/gobs/gobs/sweep/x/DataLoss01.java @@ -0,0 +1,143 @@ + +package gobs.sweep.x; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.sim.*; +import gobs.sweep.*; +import gobs.util.*; +import gobs.plot.*; + +/** + * Modifies MTTF. + * Reports number of objects lost for varying MTTF + * + * Usage: DataLoss01 -n <repeats> -o <output.eps> *.cfg + * */ + +class DataLoss01 +{ + static Properties properties; + + public static void main(String[] args) + { + properties = Sweep.getopts(args); + + AbstractSweep.seed(properties); + + String output = Tools.string(properties, "output"); + + int minNodes = Tools.integer(properties, "nodes.min"); + int maxNodes = Tools.integer(properties, "nodes.max"); + String stepNodes = Tools.string(properties, "nodes.step"); + List<String> valuesNodes = + Sweep.values(stepNodes, minNodes, maxNodes); + + int minMTTF = Tools.integer(properties, "mttf.min"); + int maxMTTF = Tools.integer(properties, "mttf.max"); + String stepMTTF = Tools.string(properties, "mttf.step"); + List<String> valuesMTTF = + Sweep.values(stepMTTF, minMTTF, maxMTTF); + + System.out.println("DataLoss01: nodes: " + valuesNodes + "\n" + + " mttf: " + valuesMTTF); + + final XYSeriesCollection collection = new XYSeriesCollection(); + + properties.setProperty("impl.placement", "Nearest"); + Sweep pSweep = new Sweep(properties); + pSweep.repeats = Tools.integer(properties, "repeats"); + pSweep.execute("nodes", valuesNodes, "mttf", valuesMTTF); + + List<XYSeries> allSeries = new ArrayList<XYSeries>(); + + List<XYSeries> pSeries = plots(pSweep, "PAST", + valuesNodes, valuesMTTF); + allSeries.addAll(pSeries); + + /* + properties.setProperty("impl.placement", "Kademlia"); + ArraySweep kSweep = new ArraySweep(properties); + kSweep.repeats = Tools.integer(properties, "repeats"); + kSweep.execute("Kademlia", "nodes", values, "distribution"); + allSeries.addAll(kSweep.collection); + + properties.setProperty("impl.placement", "VertexGroups"); + ArraySweep vgSweep = new ArraySweep(properties); + vgSweep.repeats = Tools.integer(properties, "repeats"); + vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); + allSeries.addAll(vgSweep.collection); + */ + + for (XYSeries series : allSeries) + collection.addSeries(series); + + MultiplePlot.plot(collection, "Objects Lost", + "MTTF", "objects", output); + + // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", + // "nodes", "load", "tmp.txt"); + } + + /** + Plot average number of lost objects for each MTTF. + @param label E.g., "PAST" + */ + static List<XYSeries> plots(Sweep sweep, String label, + List<String> valuesNodes, + List<String> valuesMTTF) + { + List<XYSeries> results = new ArrayList<XYSeries>(); + + for (String nodes : valuesNodes) + { + XYSeries series = new XYSeries(label + " N=" + nodes); + for (String mttf : valuesMTTF) + { + double losses = 0.0; + double total = 0.0; + for (Simulator simulator : sweep.sims.values()) + { + if (simulator.string("nodes").equals(nodes) && + simulator.string("mttf").equals(mttf)) + { + System.out.println(simulator.string("tag")); + losses += simulator.stats.lostObjects; + total++; + } + } + series.add(Integer.parseInt(mttf), losses/total); + } + results.add(series); + } + return results; + } + + static void loadsToSeries(XYSeries series, List<Rebuild> rebuilds, + double interval) + { + double longest = longestRebuild(rebuilds); + double d = 0.0; + while (d <= longest) + { + double total = 0.0; + for (Rebuild rebuild : rebuilds) + total += rebuild.loadOffset(d); + double sample = total/rebuilds.size(); + series.add(d, sample); + d += interval; + } + } + + static double longestRebuild(List<Rebuild> rebuilds) + { + double max = 0.0; + for (Rebuild rebuild : rebuilds) + if (rebuild.length() > max) + max = rebuild.length(); + return max; + } +} diff --git a/sim/gobs/gobs/sweep/x/Decluster01.java b/sim/gobs/gobs/sweep/x/Decluster01.java new file mode 100644 index 0000000..5d7f4e3 --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Decluster01.java @@ -0,0 +1,37 @@ + +package gobs.sweep.x; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; + +import gobs.plot.SimplePlot; +import gobs.sweep.*; +import gobs.util.*; + +/** + * Modifies node count. Reports object movement load on the + * heaviest server during rebuild. + * + * Usage: Decluster01 -n <repeats> gobs.cfg params.cfg Decluster01.cfg + * */ + +class Decluster01 +{ + public static void main(String[] args) + { + Bits.init(); + + Sweep sweep = new Sweep(args); + + int minNodes = Tools.integer(sweep.properties, "minNodes"); + int maxNodes = Tools.integer(sweep.properties, "maxNodes"); + List<String> values = Sweep.exps(minNodes, maxNodes); + + XYSeries series = + sweep.execute("Declustering", "nodes", values, "maxPulls"); + + SimplePlot.plot(series, "Maximum Rebuild Load", "nodes", "load", + "Decluster01.eps"); + } +} diff --git a/sim/gobs/gobs/sweep/x/Decluster01P.java b/sim/gobs/gobs/sweep/x/Decluster01P.java new file mode 100644 index 0000000..4a17d80 --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Decluster01P.java @@ -0,0 +1,36 @@ + +package gobs.sweep; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; + +import gobs.plot.SimplePlot; +import gobs.util.*; + +/** + * Modifies node count. Reports object movement load on the + * heaviest server during rebuild. + * + * Usage: Decluster01 -n <repeats> gobs.cfg params.cfg Decluster01.cfg + * */ + +class Decluster01P +{ + public static void main(String[] args) + { + Bits.init(); + + Sweep sweep = new ParallelSweep(args); + + int minNodes = Tools.integer(sweep.properties, "minNodes"); + int maxNodes = Tools.integer(sweep.properties, "maxNodes"); + List<String> values = Sweep.tens(minNodes, maxNodes); + + XYSeries series = + sweep.execute("Declustering", "nodes", values, "maxPulls"); + + SimplePlot.plot(series, "Maximum Rebuild Load", "nodes", "load", + "Decluster01.eps"); + } +} diff --git a/sim/gobs/gobs/sweep/x/Decluster02.java b/sim/gobs/gobs/sweep/x/Decluster02.java new file mode 100644 index 0000000..fab775c --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Decluster02.java @@ -0,0 +1,69 @@ + +package gobs.sweep.x; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.plot.*; +import gobs.sweep.*; +import gobs.util.*; + +/** + * Modifies node count. Reports object movement load on the + * heaviest server during rebuild. + * + * Requires properties: + * + * output - EPS target output + * minNodes, maxNodes + * step - For Sweep.values() + * repeats - Number of repetitions for average, may be set with -n. + * + * Usage: Decluster02 -n <repeats> *.cfg + * */ + +class Decluster02 +{ + static Properties properties; + + public static void main(String[] args) + { + properties = Sweep.getopts(args); + AbstractSweep.seed(properties); + + String output = Tools.string(properties, "output"); + int minNodes = Tools.integer(properties, "minNodes"); + int maxNodes = Tools.integer(properties, "maxNodes"); + String step = Tools.string(properties, "step"); + List<String> values = Sweep.values(step, minNodes, maxNodes); + + final XYSeriesCollection collection = new XYSeriesCollection(); + + properties.setProperty("impl.placement", "Nearest"); + Sweep nearestSweep = new Sweep(properties); + nearestSweep.repeats = Tools.integer(properties, "repeats"); + collection.addSeries + (nearestSweep.execute("PAST", "nodes", values, "maxPulls")); + + properties.setProperty("impl.placement", "Kademlia"); + Sweep kademliaSweep = new Sweep(properties); + kademliaSweep.repeats = Tools.integer(properties, "repeats"); + collection.addSeries + (kademliaSweep.execute("Kademlia", "nodes", values, "maxPulls")); + + properties.setProperty("impl.placement", "VertexGroups"); + Sweep vgSweep = new Sweep(properties); + vgSweep.repeats = Tools.integer(properties, "repeats"); + collection.addSeries + (vgSweep.execute("Vertex Groups", "nodes", values, "maxPulls")); + + MultiLogXPlot.plot(collection, "Maximum Rebuild Load", + "nodes", "load fraction (%)", output); + // AsciiPlot.plot(nearestSweep.series, "Maximum Rebuild Load", + // "nodes", "load", "tmp.txt"); + + Sweep.reportRuntime(); + } +} diff --git a/sim/gobs/gobs/sweep/x/Decluster03.java b/sim/gobs/gobs/sweep/x/Decluster03.java new file mode 100644 index 0000000..a2080ac --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Decluster03.java @@ -0,0 +1,64 @@ + +package gobs.sweep.x; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.plot.*; +import gobs.sweep.*; +import gobs.util.*; + +/** + * Modifies node count. + * Reports object movement load (pushes+pulls) on the + * busiest server during rebuild. + * + * Usage: Decluster03 -n <repeats> *.cfg + * */ + +class Decluster03 +{ + static Properties properties; + + public static void main(String[] args) + { + Bits.init(); + + properties = Sweep.getopts(args); + + int minNodes = Tools.integer(properties, "minNodes"); + int maxNodes = Tools.integer(properties, "maxNodes"); + String step = Tools.string(properties, "step"); + List<String> values = Sweep.values(step, minNodes, maxNodes); + + System.out.println("Decluster03: " + values); + + final XYSeriesCollection collection = new XYSeriesCollection(); + + properties.setProperty("impl.placement", "Nearest"); + Sweep nearestSweep = new Sweep(properties); + nearestSweep.repeats = Tools.integer(properties, "repeats"); + collection.addSeries + (nearestSweep.execute("PAST", "nodes", values, "maxTotal")); + + properties.setProperty("impl.placement", "Kademlia"); + Sweep kademliaSweep = new Sweep(properties); + kademliaSweep.repeats = Tools.integer(properties, "repeats"); + collection.addSeries + (kademliaSweep.execute("Kademlia", "nodes", values, "maxTotal")); + + properties.setProperty("impl.placement", "VertexGroups"); + Sweep vgSweep = new Sweep(properties); + vgSweep.repeats = Tools.integer(properties, "repeats"); + collection.addSeries + (vgSweep.execute("Vertex Groups", "nodes", values, "maxTotal")); + + String output = Tools.string(properties, "output"); + MultiLogXPlot.plot(collection, "Maximum Rebuild Load", + "nodes", "load fraction(%)", output); + // AsciiPlot.plot(nearestSweep.series, "Maximum Rebuild Load", + // "nodes", "load", "tmp.txt"); + } +} diff --git a/sim/gobs/gobs/sweep/x/Decluster04.java b/sim/gobs/gobs/sweep/x/Decluster04.java new file mode 100644 index 0000000..fb03d73 --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Decluster04.java @@ -0,0 +1,71 @@ + +package gobs.sweep.x; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.plot.*; +import gobs.sweep.*; +import gobs.util.*; + +/** + * Modifies node count. + * Reports object movement load (pushes+pulls) on all servers + * during rebuild. + * + * Usage: Decluster04 -n <repeats> *.cfg + * */ + +class Decluster04 +{ + static Properties properties; + + public static void main(String[] args) + { + Bits.init(); + + properties = Sweep.getopts(args); + + String output = Tools.string(properties, "output"); + + int minNodes = Tools.integer(properties, "minNodes"); + int maxNodes = Tools.integer(properties, "maxNodes"); + String step = Tools.string(properties, "step"); + List<String> values = Sweep.values(step, minNodes, maxNodes); + + System.out.println("Decluster04: " + values); + + List<XYSeries> allSeries = new ArrayList<XYSeries>(); + final XYSeriesCollection collection = new XYSeriesCollection(); + + properties.setProperty("impl.placement", "Nearest"); + ArraySweep pSweep = new ArraySweep(properties); + pSweep.repeats = Tools.integer(properties, "repeats"); + allSeries.addAll + (pSweep.execute("PAST", "nodes", values, "distribution")); + + properties.setProperty("impl.placement", "Kademlia"); + ArraySweep kSweep = new ArraySweep(properties); + kSweep.repeats = Tools.integer(properties, "repeats"); + allSeries.addAll + (kSweep.execute("Kademlia", "nodes", values, "distribution")); + + properties.setProperty("impl.placement", "VertexGroups"); + ArraySweep vgSweep = new ArraySweep(properties); + vgSweep.repeats = Tools.integer(properties, "repeats"); + allSeries.addAll + (vgSweep.execute + ("Vertex Groups", "nodes", values, "distribution")); + + for (XYSeries series : allSeries) + collection.addSeries(series); + + MultiplePlot.plot(collection, "Rebuild Load Distribution", + "nodes involved", "load fraction(%)", output); + + // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", + // "nodes", "load", "tmp.txt"); + } +} diff --git a/sim/gobs/gobs/sweep/x/Parallelism01.java b/sim/gobs/gobs/sweep/x/Parallelism01.java new file mode 100644 index 0000000..40fe7e3 --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Parallelism01.java @@ -0,0 +1,67 @@ + +package gobs.sweep.x; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.plot.*; +import gobs.sweep.*; +import gobs.util.*; + +/** + * Modifies node count N and Placement. + * Reports file access load balancing for files and replicas. + * + * Usage: Parallelism01 -n <repeats> *.cfg + * */ + +class Parallelism01 +{ + static Properties properties; + + public static void main(String[] args) + { + Bits.init(); + + properties = Sweep.getopts(args); + + String output = Tools.string(properties, "output"); + + int minNodes = Tools.integer(properties, "minNodes"); + int maxNodes = Tools.integer(properties, "maxNodes"); + String step = Tools.string(properties, "step"); + List<String> values = Sweep.values(step, minNodes, maxNodes); + + System.out.println("Parallelism01: " + values); + + final XYSeriesCollection collection = new XYSeriesCollection(); + + properties.setProperty("impl.placement", "Nearest"); + Sweep nearestSweep = new Sweep(properties); + nearestSweep.repeats = Tools.integer(properties, "repeats"); + collection.addSeries + (nearestSweep.execute + ("PAST", "nodes", values, "accessbalance")); + + properties.setProperty("impl.placement", "Kademlia"); + Sweep kademliaSweep = new Sweep(properties); + kademliaSweep.repeats = Tools.integer(properties, "repeats"); + collection.addSeries + (kademliaSweep.execute + ("Kademlia", "nodes", values, "accessbalance")); + + properties.setProperty("impl.placement", "VertexGroups"); + Sweep vgSweep = new Sweep(properties); + vgSweep.repeats = Tools.integer(properties, "repeats"); + collection.addSeries + (vgSweep.execute + ("Vertex Groups", "nodes", values, "accessbalance")); + + MultiLogXPlot.plot(collection, "Load Imbalance", + "nodes", "imbalance", output); + // AsciiPlot.plot(nearestSweep.series, "Maximum Rebuild Load", + // "nodes", "load", "tmp.txt"); + } +} diff --git a/sim/gobs/gobs/sweep/x/Rebuild01.java b/sim/gobs/gobs/sweep/x/Rebuild01.java new file mode 100644 index 0000000..2bed2d1 --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Rebuild01.java @@ -0,0 +1,76 @@ + +package gobs.sweep.x; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.plot.*; +import gobs.sweep.*; +import gobs.util.*; + +/** + * Modifies node count. + * Reports rebuild activity over time during rebuild. + * + * Usage: Rebuild01 -n <repeats> *.cfg + * */ + +// good runs: +// seed=828 -n 1 + +class Rebuild01 +{ + static Properties properties; + + public static void main(String[] args) + { + properties = Sweep.getopts(args); + + AbstractSweep.seed(properties); + + String output = Tools.string(properties, "output"); + + int minNodes = Tools.integer(properties, "nodes.min"); + int maxNodes = Tools.integer(properties, "nodes.max"); + String step = Tools.string(properties, "nodes.step"); + List<String> values = Sweep.values(step, minNodes, maxNodes); + + System.out.println("Rebuild01: " + values); + + List<XYSeries> allSeries = new ArrayList<XYSeries>(); + final XYSeriesCollection collection = new XYSeriesCollection(); + + properties.setProperty("impl.placement", "Nearest"); + ArraySweep pSweep = new ArraySweep(properties); + pSweep.repeats = Tools.integer(properties, "repeats"); + allSeries.addAll + (pSweep.execute + ("PAST", "nodes", values, "traffic-x", "traffic-y")); + + /* + properties.setProperty("impl.placement", "Kademlia"); + ArraySweep kSweep = new ArraySweep(properties); + kSweep.repeats = Tools.integer(properties, "repeats"); + kSweep.execute("Kademlia", "nodes", values, "distribution"); + allSeries.addAll(kSweep.collection); + + properties.setProperty("impl.placement", "VertexGroups"); + ArraySweep vgSweep = new ArraySweep(properties); + vgSweep.repeats = Tools.integer(properties, "repeats"); + vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); + allSeries.addAll(vgSweep.collection); + */ + + for (XYSeries series : allSeries) + collection.addSeries(series); + + MultiplePlot.plot(collection, "Rebuild Traffic", + "time", "copies in flight", + output); + + // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", + // "nodes", "load", "tmp.txt"); + } +} diff --git a/sim/gobs/gobs/sweep/x/Rebuild02.java b/sim/gobs/gobs/sweep/x/Rebuild02.java new file mode 100644 index 0000000..f2d98d9 --- /dev/null +++ b/sim/gobs/gobs/sweep/x/Rebuild02.java @@ -0,0 +1,127 @@ + +package gobs.sweep.x; + +import java.util.*; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.sim.*; +import gobs.sweep.*; +import gobs.util.*; +import gobs.plot.*; + +/** + * Modifies node count. + * Reports rebuild activity over time during rebuild. + * + * Usage: Rebuild02 -n <repeats> -o <output.eps> *.cfg + * */ + +class Rebuild02 +{ + static Properties properties; + + public static void main(String[] args) + { + properties = Sweep.getopts(args); + + AbstractSweep.seed(properties); + + String output = Tools.string(properties, "output"); + double interval = Tools.decimal(properties, + "plot.traffic.interval"); + + int minNodes = Tools.integer(properties, "nodes.min"); + int maxNodes = Tools.integer(properties, "nodes.max"); + String step = Tools.string(properties, "nodes.step"); + List<String> values = Sweep.values(step, minNodes, maxNodes); + + System.out.println("Rebuild02: " + values); + + final XYSeriesCollection collection = new XYSeriesCollection(); + + properties.setProperty("impl.placement", "Nearest"); + Sweep pSweep = new Sweep(properties); + pSweep.repeats = Tools.integer(properties, "repeats"); + pSweep.execute("nodes", values); + + List<XYSeries> allSeries = new ArrayList<XYSeries>(); + + List<XYSeries> pSeries = plots(pSweep, "PAST", "nodes", + values, interval); + allSeries.addAll(pSeries); + + /* + properties.setProperty("impl.placement", "Kademlia"); + ArraySweep kSweep = new ArraySweep(properties); + kSweep.repeats = Tools.integer(properties, "repeats"); + kSweep.execute("Kademlia", "nodes", values, "distribution"); + allSeries.addAll(kSweep.collection); + + properties.setProperty("impl.placement", "VertexGroups"); + ArraySweep vgSweep = new ArraySweep(properties); + vgSweep.repeats = Tools.integer(properties, "repeats"); + vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); + allSeries.addAll(vgSweep.collection); + */ + + for (XYSeries series : allSeries) + collection.addSeries(series); + + MultiplePlot.plot(collection, "Rebuild Traffic", + "time", "copies in flight", + output); + + // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", + // "nodes", "load", "tmp.txt"); + } + + static List<XYSeries> plots(Sweep sweep, String label, + String xname, List<String> values, + double interval) + { + List<XYSeries> results = new ArrayList<XYSeries>(); + + for (String v : values) + { + XYSeries series = new XYSeries(label + " N=" + v); + List<Rebuild> rebuilds = new ArrayList<Rebuild>(); + for (Simulator simulator : sweep.sims.values()) + { + if (Tools.string(simulator.properties, xname).equals(v)) + { + rebuilds.addAll(simulator.rebuilds); + } + } + loadsToSeries(series, rebuilds, interval); + results.add(series); + } + return results; + } + + static void loadsToSeries(XYSeries series, List<Rebuild> rebuilds, + double interval) + { + double longest = longestRebuild(rebuilds); + double d = 0.0; + while (d <= longest) + { + double total = 0.0; + for (Rebuild rebuild : rebuilds) + total += rebuild.loadOffset(d); + double sample = total/rebuilds.size(); + series.add(d, sample); + d += interval; + } + } + + static double longestRebuild(List<Rebuild> rebuilds) + { + double max = 0.0; + for (Rebuild rebuild : rebuilds) + if (rebuild.length() > max) + max = rebuild.length(); + return max; + } +} diff --git a/sim/gobs/scripts/eol.zsh b/sim/gobs/scripts/eol.zsh old mode 100755 new mode 100644 index 292b1ae..0a65445 --- a/sim/gobs/scripts/eol.zsh +++ b/sim/gobs/scripts/eol.zsh @@ -1,12 +1,25 @@ #!/bin/zsh -files=() -for f in **/*.zsh - do - if [[ ${f} != $0 ]] +print "Fixing EOL: " + +typeset -aU FILES +STATUS=( $( git status ) ) + +for (( i=0 ; i<${#STATUS} ; i++ )) +do + if [[ ${STATUS[i]} == "file:" || + ${STATUS[i]} == "modified:" ]] then - files+=${f} + FILE=${STATUS[i+1]} + if [[ ${FILE} == $0 || + ${FILE} == *.eps ]] + then + continue + fi + FILES+=${FILE} fi done -sed -i 's/ *$//' **/*.java **/*.cfg ${files} +print -l $FILES + +sed -i 's/ *$//' ${FILES} diff --git a/sim/gobs/sweeps/dl01-small.cfg b/sim/gobs/sweeps/dl01-small.cfg new file mode 100644 index 0000000..ec10bcc --- /dev/null +++ b/sim/gobs/sweeps/dl01-small.cfg @@ -0,0 +1,79 @@ + +# Simulator configuration + +# Basics: + +include = rb_output.cfg + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 10 + +### Parameters for GOBS Simulator run + +### General parameters: + +# Bit-length of addresses: +B = 16 +# Number of servers: +nodes.min = 12 +nodes.max = 16 +# Step through nodes: See AbstractSweep.values() +nodes.step = literal_14,16,18 +# Number of files: +files = 6 +# Number of faults: +faults = 4 +# Number of accesses: +reads = 0 +writes = 0 + +# Step through MTTF values: See AbstractSweep.values() +mttf.min = 150 +mttf.max = 600 +mttf.step = exps + +### Choose a PlacementScheme implementation: + +#impl.placement = Kademlia +impl.placement = Nearest +#impl.placement = VertexGroups + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes + +### Choose one of the following FileFactory sections: + +# FileFactory implementation PlainFiles +impl.filefactory = PlainFiles +# File size: +file.size = 140 +# Strip size: +file.strip = 4 +# File/object replicas: +file.replicas = 3 +# Number of objects per file: +file.width = 3 + +# FileFactory implementation TraceFiles +#impl.filefactory = TraceFiles +# Trace filename: +#filetrace = traces/test01.trace + +# FileFactory implementation RandomFiles +#impl.filefactory = RandomFiles +# Max size of big files: +#filebigmax = 100 +# Min size of big files: +#filebigmin = 50 +# Max size of small files: +#filesmallmax = 20 +# Min size of small files: +#filesmallmin = 0 diff --git a/sim/gobs/wiki/dl01-small.cfg b/sim/gobs/wiki/dl01-small.cfg new file mode 100644 index 0000000..3d78181 --- /dev/null +++ b/sim/gobs/wiki/dl01-small.cfg @@ -0,0 +1,79 @@ + +# Simulator configuration + +# Basics: + +include = rb_output.cfg + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 10 + +### Parameters for GOBS Simulator run + +### General parameters: + +# Bit-length of addresses: +B = 16 +# Number of servers: +nodes.min = 12 +nodes.max = 16 +# Step through nodes: See AbstractSweep.values() +nodes.step = literal_14,16,18 +# Number of files: +files = 6 +# Number of faults: +faults = 2 +# Number of accesses: +reads = 0 +writes = 0 + +# Step through MTTF values: See AbstractSweep.values() +mttf.min = 150 +mttf.max = 600 +mttf.step = exps + +### Choose a PlacementScheme implementation: + +#impl.placement = Kademlia +impl.placement = Nearest +#impl.placement = VertexGroups + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes + +### Choose one of the following FileFactory sections: + +# FileFactory implementation PlainFiles +impl.filefactory = PlainFiles +# File size: +file.size = 140 +# Strip size: +file.strip = 4 +# File/object replicas: +file.replicas = 3 +# Number of objects per file: +file.width = 3 + +# FileFactory implementation TraceFiles +#impl.filefactory = TraceFiles +# Trace filename: +#filetrace = traces/test01.trace + +# FileFactory implementation RandomFiles +#impl.filefactory = RandomFiles +# Max size of big files: +#filebigmax = 100 +# Min size of big files: +#filebigmin = 50 +# Max size of small files: +#filesmallmax = 20 +# Min size of small files: +#filesmallmin = 0 diff --git a/sim/gobs/wiki/dl01-small.eps b/sim/gobs/wiki/dl01-small.eps new file mode 100755 index 0000000..01043d9 --- /dev/null +++ b/sim/gobs/wiki/dl01-small.eps @@ -0,0 +1,3771 @@ +%!PS-Adobe-3.0 EPSF-3.0 +%%Creator: (Apache XML Graphics Commons: EPS Generator for Java2D) +%%CreationDate: 2009-12-05T15:30:00 +%%Pages: (atend) +%%BoundingBox: 0 0 400 400 +%%LanguageLevel: 3 +%%EndComments +%%BeginProlog +%%BeginResource: procset (Apache XML Graphics Std ProcSet) 1.0 0 +%%Version: 1.0 0 +%%Copyright: (Copyright 2001-2003 The Apache Software Foundation. License terms: http://www.apache.org/licenses/LICENSE-2.0) +%%Title: (Basic set of procedures used by the XML Graphics project $Batik and FOP$) +/bd{bind def}bind def +/ld{load def}bd +/M/moveto ld +/RM/rmoveto ld +/t/show ld +/A/ashow ld +/cp/closepath ld +/re {4 2 roll M +1 index 0 rlineto +0 exch rlineto +neg 0 rlineto +cp } bd +/_ctm matrix def +/_tm matrix def +/BT { _ctm currentmatrix pop matrix _tm copy pop 0 0 moveto } bd +/ET { _ctm setmatrix } bd +/iTm { _ctm setmatrix _tm concat } bd +/Tm { _tm astore pop iTm 0 0 moveto } bd +/ux 0.0 def +/uy 0.0 def +/F { + /Tp exch def + /Tf exch def + Tf findfont Tp scalefont setfont + /cf Tf def /cs Tp def /cw ( ) stringwidth pop def +} bd +/ULS {currentpoint /uy exch def /ux exch def} bd +/ULE { + /Tcx currentpoint pop def + gsave + newpath + cf findfont cs scalefont dup + /FontMatrix get 0 get /Ts exch def /FontInfo get dup + /UnderlinePosition get Ts mul /To exch def + /UnderlineThickness get Ts mul /Tt exch def + ux uy To add moveto Tcx uy To add lineto + Tt setlinewidth stroke + grestore +} bd +/OLE { + /Tcx currentpoint pop def + gsave + newpath + cf findfont cs scalefont dup + /FontMatrix get 0 get /Ts exch def /FontInfo get dup + /UnderlinePosition get Ts mul /To exch def + /UnderlineThickness get Ts mul /Tt exch def + ux uy To add cs add moveto Tcx uy To add cs add lineto + Tt setlinewidth stroke + grestore +} bd +/SOE { + /Tcx currentpoint pop def + gsave + newpath + cf findfont cs scalefont dup + /FontMatrix get 0 get /Ts exch def /FontInfo get dup + /UnderlinePosition get Ts mul /To exch def + /UnderlineThickness get Ts mul /Tt exch def + ux uy To add cs 10 mul 26 idiv add moveto Tcx uy To add cs 10 mul 26 idiv add lineto + Tt setlinewidth stroke + grestore +} bd +/QUADTO { +/Y22 exch store +/X22 exch store +/Y21 exch store +/X21 exch store +currentpoint +/Y21 load 2 mul add 3 div exch +/X21 load 2 mul add 3 div exch +/X21 load 2 mul /X22 load add 3 div +/Y21 load 2 mul /Y22 load add 3 div +/X22 load /Y22 load curveto +} bd +/SSPD { +dup length /d exch dict def +{ +/v exch def +/k exch def +currentpagedevice k known { +/cpdv currentpagedevice k get def +v cpdv ne { +/upd false def +/nullv v type /nulltype eq def +/nullcpdv cpdv type /nulltype eq def +nullv nullcpdv or +{ +/upd true def +} { +/sametype v type cpdv type eq def +sametype { +v type /arraytype eq { +/vlen v length def +/cpdvlen cpdv length def +vlen cpdvlen eq { +0 1 vlen 1 sub { +/i exch def +/obj v i get def +/cpdobj cpdv i get def +obj cpdobj ne { +/upd true def +exit +} if +} for +} { +/upd true def +} ifelse +} { +v type /dicttype eq { +v { +/dv exch def +/dk exch def +/cpddv cpdv dk get def +dv cpddv ne { +/upd true def +exit +} if +} forall +} { +/upd true def +} ifelse +} ifelse +} if +} ifelse +upd true eq { +d k v put +} if +} if +} if +} forall +d length 0 gt { +d setpagedevice +} if +} bd +%%EndResource +%%BeginResource: procset (Apache XML Graphics EPS ProcSet) 1.0 0 +%%Version: 1.0 0 +%%Copyright: (Copyright 2002-2003 The Apache Software Foundation. License terms: http://www.apache.org/licenses/LICENSE-2.0) +%%Title: (EPS procedures used by the Apache XML Graphics project $Batik and FOP$) +/BeginEPSF { %def +/b4_Inc_state save def % Save state for cleanup +/dict_count countdictstack def % Count objects on dict stack +/op_count count 1 sub def % Count objects on operand stack +userdict begin % Push userdict on dict stack +/showpage { } def % Redefine showpage, { } = null proc +0 setgray 0 setlinecap % Prepare graphics state +1 setlinewidth 0 setlinejoin +10 setmiterlimit [ ] 0 setdash newpath +/languagelevel where % If level not equal to 1 then +{pop languagelevel % set strokeadjust and +1 ne % overprint to their defaults. +{false setstrokeadjust false setoverprint +} if +} if +} bd +/EndEPSF { %def +count op_count sub {pop} repeat % Clean up stacks +countdictstack dict_count sub {end} repeat +b4_Inc_state restore +} bd +%%EndResource +%%EndProlog +%%Page: 1 1 +%%PageBoundingBox: 0 0 400 400 +%%BeginPageSetup +[1 0 0 -1 0 400] concat +%%EndPageSetup +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +1 setgray +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +newpath +152.465 13.744 M +152.465 17.275 150.582 19.291 QUADTO +148.699 21.307 145.418 21.307 QUADTO +142.121 21.307 140.246 19.26 QUADTO +138.371 17.213 138.371 13.744 QUADTO +138.371 10.244 140.246 8.205 QUADTO +142.121 6.166 145.418 6.166 QUADTO +148.699 6.166 150.582 8.189 QUADTO +152.465 10.213 152.465 13.744 QUADTO +closepath +148.699 13.729 M +148.699 12.432 148.426 11.518 QUADTO +148.152 10.604 147.699 10.025 QUADTO +147.23 9.432 146.645 9.174 QUADTO +146.059 8.916 145.418 8.916 QUADTO +144.746 8.916 144.199 9.15 QUADTO +143.652 9.385 143.152 10.01 QUADTO +142.699 10.572 142.418 11.51 QUADTO +142.137 12.447 142.137 13.744 QUADTO +142.137 15.088 142.402 15.971 QUADTO +142.668 16.854 143.121 17.447 QUADTO +143.59 18.057 144.191 18.314 QUADTO +144.793 18.572 145.418 18.572 QUADTO +146.059 18.572 146.668 18.291 QUADTO +147.277 18.01 147.715 17.432 QUADTO +148.184 16.791 148.441 15.947 QUADTO +148.699 15.104 148.699 13.729 QUADTO +closepath +161.678 15.447 M +161.678 13.807 161.178 13.104 QUADTO +160.678 12.4 159.646 12.4 QUADTO +159.193 12.4 158.74 12.541 QUADTO +158.287 12.682 157.85 12.979 QUADTO +157.85 18.494 lineto +158.178 18.619 158.529 18.674 QUADTO +158.881 18.729 159.225 18.729 QUADTO +160.459 18.729 161.068 17.908 QUADTO +161.678 17.088 161.678 15.447 QUADTO +closepath +165.178 15.369 M +165.178 17.994 163.85 19.611 QUADTO +162.521 21.229 160.553 21.229 QUADTO +159.74 21.229 159.131 21.041 QUADTO +158.521 20.854 157.85 20.463 QUADTO +157.693 21.01 lineto +154.428 21.01 lineto +154.428 5.807 lineto +157.85 5.807 lineto +157.85 11.182 lineto +158.6 10.541 159.381 10.143 QUADTO +160.162 9.744 161.178 9.744 QUADTO +163.1 9.744 164.139 11.236 QUADTO +165.178 12.729 165.178 15.369 QUADTO +closepath +171.705 21.213 M +171.705 23.057 170.596 24.096 QUADTO +169.486 25.135 167.752 25.135 QUADTO +166.814 25.135 166.049 25.057 QUADTO +165.283 24.979 164.924 24.885 QUADTO +164.924 22.369 lineto +165.189 22.369 lineto +165.533 22.479 165.939 22.541 QUADTO +166.346 22.604 166.58 22.604 QUADTO +167.564 22.604 167.924 21.994 QUADTO +168.283 21.385 168.283 20.01 QUADTO +168.283 12.432 lineto +166.314 12.432 lineto +166.314 10.041 lineto +171.705 10.041 lineto +171.705 21.213 lineto +closepath +171.705 8.479 M +168.064 8.479 lineto +168.064 5.807 lineto +171.705 5.807 lineto +171.705 8.479 lineto +closepath +179.93 21.307 M +176.82 21.307 175.172 19.822 QUADTO +173.523 18.338 173.523 15.588 QUADTO +173.523 12.9 175.062 11.299 QUADTO +176.602 9.697 179.383 9.697 QUADTO +181.898 9.697 183.164 11.057 QUADTO +184.43 12.416 184.43 14.947 QUADTO +184.43 16.182 lineto +176.945 16.182 lineto +176.992 16.932 177.266 17.447 QUADTO +177.539 17.963 177.992 18.26 QUADTO +178.43 18.572 179.016 18.705 QUADTO +179.602 18.838 180.289 18.838 QUADTO +180.883 18.838 181.461 18.705 QUADTO +182.039 18.572 182.523 18.354 QUADTO +182.945 18.166 183.297 17.955 QUADTO +183.648 17.744 183.883 17.572 QUADTO +184.242 17.572 lineto +184.242 20.4 lineto +183.82 20.557 183.477 20.697 QUADTO +183.133 20.838 182.523 20.994 QUADTO +181.961 21.135 181.375 21.221 QUADTO +180.789 21.307 179.93 21.307 QUADTO +closepath +181.07 14.119 M +181.039 13.025 180.562 12.463 QUADTO +180.086 11.9 179.117 11.9 QUADTO +178.117 11.9 177.555 12.494 QUADTO +176.992 13.088 176.93 14.119 QUADTO +181.07 14.119 lineto +closepath +191.57 21.307 M +190.211 21.307 189.086 20.963 QUADTO +187.961 20.619 187.133 19.9 QUADTO +186.305 19.197 185.852 18.119 QUADTO +185.398 17.041 185.398 15.572 QUADTO +185.398 13.994 185.891 12.893 QUADTO +186.383 11.791 187.242 11.072 QUADTO +188.055 10.4 189.164 10.072 QUADTO +190.273 9.744 191.477 9.744 QUADTO +192.383 9.744 193.289 9.955 QUADTO +194.195 10.166 195.117 10.619 QUADTO +195.117 13.541 lineto +194.68 13.541 lineto +194.477 13.354 194.18 13.119 QUADTO +193.883 12.885 193.523 12.682 QUADTO +193.148 12.479 192.68 12.338 QUADTO +192.211 12.197 191.648 12.197 QUADTO +190.336 12.197 189.617 13.111 QUADTO +188.898 14.025 188.898 15.572 QUADTO +188.898 17.213 189.664 18.025 QUADTO +190.43 18.838 191.695 18.838 QUADTO +192.352 18.838 192.836 18.689 QUADTO +193.32 18.541 193.68 18.338 QUADTO +194.008 18.135 194.258 17.916 QUADTO +194.508 17.697 194.68 17.525 QUADTO +195.117 17.525 lineto +195.117 20.447 lineto +194.82 20.572 194.438 20.744 QUADTO +194.055 20.916 193.648 21.025 QUADTO +193.148 21.15 192.719 21.229 QUADTO +192.289 21.307 191.57 21.307 QUADTO +closepath +200.945 21.229 M +198.852 21.229 197.898 20.369 QUADTO +196.945 19.51 196.945 17.557 QUADTO +196.945 12.432 lineto +195.633 12.432 lineto +195.633 10.041 lineto +196.945 10.041 lineto +196.945 6.9 lineto +200.367 6.9 lineto +200.367 10.041 lineto +203.648 10.041 lineto +203.648 12.432 lineto +200.367 12.432 lineto +200.367 16.307 lineto +200.367 16.885 200.375 17.314 QUADTO +200.383 17.744 200.523 18.088 QUADTO +200.664 18.432 201.008 18.635 QUADTO +201.352 18.838 202.023 18.838 QUADTO +202.289 18.838 202.734 18.721 QUADTO +203.18 18.604 203.367 18.494 QUADTO +203.648 18.494 lineto +203.648 20.916 lineto +203.102 21.057 202.477 21.143 QUADTO +201.852 21.229 200.945 21.229 QUADTO +closepath +208.396 21.307 M +207.178 21.307 206.1 21.049 QUADTO +205.021 20.791 204.318 20.432 QUADTO +204.318 17.541 lineto +204.6 17.541 lineto +204.85 17.729 205.162 17.955 QUADTO +205.475 18.182 206.021 18.416 QUADTO +206.506 18.65 207.107 18.807 QUADTO +207.709 18.963 208.412 18.963 QUADTO +209.131 18.963 209.686 18.744 QUADTO +210.24 18.525 210.24 18.025 QUADTO +210.24 17.65 209.982 17.455 QUADTO +209.725 17.26 208.99 17.088 QUADTO +208.6 16.979 208.006 16.869 QUADTO +207.412 16.76 206.959 16.619 QUADTO +205.662 16.213 205.014 15.424 QUADTO +204.365 14.635 204.365 13.369 QUADTO +204.365 12.635 204.701 11.986 QUADTO +205.037 11.338 205.693 10.822 QUADTO +206.35 10.322 207.311 10.033 QUADTO +208.271 9.744 209.49 9.744 QUADTO +210.631 9.744 211.6 9.971 QUADTO +212.568 10.197 213.24 10.525 QUADTO +213.24 13.291 lineto +212.959 13.291 lineto +212.787 13.15 212.404 12.924 QUADTO +212.021 12.697 211.662 12.525 QUADTO +211.225 12.338 210.693 12.213 QUADTO +210.162 12.088 209.615 12.088 QUADTO +208.865 12.088 208.357 12.338 QUADTO +207.85 12.588 207.85 13.01 QUADTO +207.85 13.385 208.1 13.596 QUADTO +208.35 13.807 209.193 14.01 QUADTO +209.646 14.119 210.217 14.221 QUADTO +210.787 14.322 211.303 14.51 QUADTO +212.49 14.885 213.1 15.619 QUADTO +213.709 16.354 213.709 17.588 QUADTO +213.709 18.354 213.35 19.049 QUADTO +212.99 19.744 212.318 20.229 QUADTO +211.615 20.744 210.67 21.025 QUADTO +209.725 21.307 208.396 21.307 QUADTO +closepath +231.111 21.01 M +221.393 21.01 lineto +221.393 6.463 lineto +225.049 6.463 lineto +225.049 18.229 lineto +231.111 18.229 lineto +231.111 21.01 lineto +closepath +243.182 15.525 M +243.182 18.229 241.697 19.791 QUADTO +240.213 21.354 237.51 21.354 QUADTO +234.822 21.354 233.338 19.791 QUADTO +231.854 18.229 231.854 15.525 QUADTO +231.854 12.791 233.338 11.244 QUADTO +234.822 9.697 237.51 9.697 QUADTO +240.213 9.697 241.697 11.26 QUADTO +243.182 12.822 243.182 15.525 QUADTO +closepath +239.666 15.557 M +239.666 14.572 239.51 13.924 QUADTO +239.354 13.275 239.057 12.885 QUADTO +238.76 12.479 238.377 12.314 QUADTO +237.994 12.15 237.51 12.15 QUADTO +237.057 12.15 236.682 12.291 QUADTO +236.307 12.432 235.994 12.838 QUADTO +235.697 13.213 235.525 13.877 QUADTO +235.354 14.541 235.354 15.557 QUADTO +235.354 16.557 235.518 17.174 QUADTO +235.682 17.791 235.947 18.166 QUADTO +236.229 18.541 236.635 18.721 QUADTO +237.041 18.9 237.541 18.9 QUADTO +237.947 18.9 238.377 18.721 QUADTO +238.807 18.541 239.057 18.197 QUADTO +239.354 17.775 239.51 17.182 QUADTO +239.666 16.588 239.666 15.557 QUADTO +closepath +248.338 21.307 M +247.119 21.307 246.041 21.049 QUADTO +244.963 20.791 244.26 20.432 QUADTO +244.26 17.541 lineto +244.541 17.541 lineto +244.791 17.729 245.104 17.955 QUADTO +245.416 18.182 245.963 18.416 QUADTO +246.447 18.65 247.049 18.807 QUADTO +247.65 18.963 248.354 18.963 QUADTO +249.072 18.963 249.627 18.744 QUADTO +250.182 18.525 250.182 18.025 QUADTO +250.182 17.65 249.924 17.455 QUADTO +249.666 17.26 248.932 17.088 QUADTO +248.541 16.979 247.947 16.869 QUADTO +247.354 16.76 246.9 16.619 QUADTO +245.604 16.213 244.955 15.424 QUADTO +244.307 14.635 244.307 13.369 QUADTO +244.307 12.635 244.643 11.986 QUADTO +244.979 11.338 245.635 10.822 QUADTO +246.291 10.322 247.252 10.033 QUADTO +248.213 9.744 249.432 9.744 QUADTO +250.572 9.744 251.541 9.971 QUADTO +252.51 10.197 253.182 10.525 QUADTO +253.182 13.291 lineto +252.9 13.291 lineto +252.729 13.15 252.346 12.924 QUADTO +251.963 12.697 251.604 12.525 QUADTO +251.166 12.338 250.635 12.213 QUADTO +250.104 12.088 249.557 12.088 QUADTO +248.807 12.088 248.299 12.338 QUADTO +247.791 12.588 247.791 13.01 QUADTO +247.791 13.385 248.041 13.596 QUADTO +248.291 13.807 249.135 14.01 QUADTO +249.588 14.119 250.158 14.221 QUADTO +250.729 14.322 251.244 14.51 QUADTO +252.432 14.885 253.041 15.619 QUADTO +253.65 16.354 253.65 17.588 QUADTO +253.65 18.354 253.291 19.049 QUADTO +252.932 19.744 252.26 20.229 QUADTO +251.557 20.744 250.611 21.025 QUADTO +249.666 21.307 248.338 21.307 QUADTO +closepath +259.49 21.229 M +257.396 21.229 256.443 20.369 QUADTO +255.49 19.51 255.49 17.557 QUADTO +255.49 12.432 lineto +254.178 12.432 lineto +254.178 10.041 lineto +255.49 10.041 lineto +255.49 6.9 lineto +258.912 6.9 lineto +258.912 10.041 lineto +262.193 10.041 lineto +262.193 12.432 lineto +258.912 12.432 lineto +258.912 16.307 lineto +258.912 16.885 258.92 17.314 QUADTO +258.928 17.744 259.068 18.088 QUADTO +259.209 18.432 259.553 18.635 QUADTO +259.896 18.838 260.568 18.838 QUADTO +260.834 18.838 261.279 18.721 QUADTO +261.725 18.604 261.912 18.494 QUADTO +262.193 18.494 lineto +262.193 20.916 lineto +261.646 21.057 261.021 21.143 QUADTO +260.396 21.229 259.49 21.229 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +1 setgray +newpath +71.356 376.516 M +328.644 376.516 lineto +328.644 399 lineto +71.356 399 lineto +71.356 376.516 lineto +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +2 setlinecap +10.0 setmiterlimit +newpath +71.856 377.016 M +328.144 377.016 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +2 setlinecap +10.0 setmiterlimit +newpath +71.856 398.5 M +328.144 398.5 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +2 setlinecap +10.0 setmiterlimit +newpath +71.856 398.5 M +71.856 377.016 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +2 setlinecap +10.0 setmiterlimit +newpath +328.144 398.5 M +328.144 377.016 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +2 setlinecap +2 setlinejoin +2 setlinejoin +newpath +75.356 387.758 M +89.356 387.758 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +newpath +79.356 384.758 M +85.356 384.758 lineto +85.356 390.758 lineto +79.356 390.758 lineto +79.356 384.758 lineto +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +79.356 384.758 M +85.356 384.758 lineto +85.356 390.758 lineto +79.356 390.758 lineto +79.356 384.758 lineto +closepath +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +99.747 386.428 M +99.747 386.99 99.544 387.498 QUADTO +99.341 388.006 98.981 388.365 QUADTO +98.544 388.818 97.935 389.045 QUADTO +97.325 389.271 96.388 389.271 QUADTO +95.403 389.271 lineto +95.403 392.521 lineto +94.247 392.521 lineto +94.247 383.803 lineto +96.435 383.803 lineto +97.2 383.803 97.747 383.928 QUADTO +98.294 384.053 98.716 384.334 QUADTO +99.2 384.678 99.474 385.186 QUADTO +99.747 385.693 99.747 386.428 QUADTO +closepath +98.544 386.459 M +98.544 386.006 98.388 385.67 QUADTO +98.231 385.334 97.919 385.131 QUADTO +97.638 384.943 97.271 384.865 QUADTO +96.903 384.787 96.356 384.787 QUADTO +95.403 384.787 lineto +95.403 388.271 lineto +96.2 388.271 lineto +96.903 388.271 97.333 388.162 QUADTO +97.763 388.053 98.044 387.756 QUADTO +98.31 387.49 98.427 387.178 QUADTO +98.544 386.865 98.544 386.459 QUADTO +closepath +99.909 392.521 M +102.862 383.803 lineto +104.284 383.803 lineto +107.222 392.521 lineto +106.003 392.521 lineto +105.206 390.084 lineto +101.878 390.084 lineto +101.081 392.521 lineto +99.909 392.521 lineto +closepath +104.894 389.1 M +103.534 384.975 lineto +102.206 389.1 lineto +104.894 389.1 lineto +closepath +110.479 392.678 M +109.542 392.678 108.87 392.482 QUADTO +108.198 392.287 107.62 392.006 QUADTO +107.62 390.553 lineto +107.698 390.553 lineto +108.276 391.084 109.003 391.381 QUADTO +109.729 391.678 110.401 391.678 QUADTO +111.37 391.678 111.862 391.279 QUADTO +112.354 390.881 112.354 390.225 QUADTO +112.354 389.693 112.097 389.365 QUADTO +111.839 389.037 111.276 388.881 QUADTO +110.87 388.771 110.534 388.693 QUADTO +110.198 388.615 109.698 388.49 QUADTO +109.245 388.365 108.886 388.186 QUADTO +108.526 388.006 108.245 387.709 QUADTO +107.979 387.428 107.831 387.037 QUADTO +107.683 386.646 107.683 386.131 QUADTO +107.683 385.068 108.511 384.35 QUADTO +109.339 383.631 110.62 383.631 QUADTO +111.354 383.631 112.003 383.779 QUADTO +112.651 383.928 113.214 384.193 QUADTO +113.214 385.584 lineto +113.12 385.584 lineto +112.714 385.209 112.065 384.928 QUADTO +111.417 384.646 110.698 384.646 QUADTO +109.886 384.646 109.386 385.021 QUADTO +108.886 385.396 108.886 386.006 QUADTO +108.886 386.568 109.183 386.896 QUADTO +109.479 387.225 110.026 387.381 QUADTO +110.401 387.475 110.933 387.6 QUADTO +111.464 387.725 111.823 387.834 QUADTO +112.729 388.115 113.144 388.67 QUADTO +113.558 389.225 113.558 390.037 QUADTO +113.558 390.553 113.347 391.045 QUADTO +113.136 391.537 112.761 391.881 QUADTO +112.339 392.271 111.808 392.475 QUADTO +111.276 392.678 110.479 392.678 QUADTO +closepath +116.774 392.521 M +116.774 384.834 lineto +113.853 384.834 lineto +113.853 383.803 lineto +120.868 383.803 lineto +120.868 384.834 lineto +117.931 384.834 lineto +117.931 392.521 lineto +116.774 392.521 lineto +closepath +131.735 392.521 M +130.438 392.521 lineto +126.579 384.99 lineto +126.579 392.521 lineto +125.501 392.521 lineto +125.501 383.803 lineto +127.142 383.803 lineto +130.657 390.709 lineto +130.657 383.803 lineto +131.735 383.803 lineto +131.735 392.521 lineto +closepath +140.229 387.943 M +133.745 387.943 lineto +133.745 387.006 lineto +140.229 387.006 lineto +140.229 387.943 lineto +closepath +140.229 390.396 M +133.745 390.396 lineto +133.745 389.459 lineto +140.229 389.459 lineto +140.229 390.396 lineto +closepath +146.929 392.521 M +142.679 392.521 lineto +142.679 391.631 lineto +144.257 391.631 lineto +144.257 385.787 lineto +142.679 385.787 lineto +142.679 384.99 lineto +143.569 384.99 143.999 384.732 QUADTO +144.429 384.475 144.476 383.771 QUADTO +145.382 383.771 lineto +145.382 391.631 lineto +146.929 391.631 lineto +146.929 392.521 lineto +closepath +154.151 389.521 M +153.12 389.521 lineto +153.12 392.521 lineto +151.995 392.521 lineto +151.995 389.521 lineto +148.136 389.521 lineto +148.136 388.287 lineto +152.042 383.803 lineto +153.12 383.803 lineto +153.12 388.584 lineto +154.151 388.584 lineto +154.151 389.521 lineto +closepath +151.995 388.584 M +151.995 385.178 lineto +149.058 388.584 lineto +151.995 388.584 lineto +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +2 setlinecap +2 setlinejoin +2 setlinejoin +newpath +160.452 387.758 M +174.452 387.758 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +newpath +170.452 387.758 M +170.452 389.415 169.109 390.758 167.452 390.758 curveto +165.795 390.758 164.452 389.415 164.452 387.758 curveto +164.452 386.101 165.795 384.758 167.452 384.758 curveto +169.109 384.758 170.452 386.101 170.452 387.758 curveto +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +170.452 387.758 M +170.452 389.415 169.109 390.758 167.452 390.758 curveto +165.795 390.758 164.452 389.415 164.452 387.758 curveto +164.452 386.101 165.795 384.758 167.452 384.758 curveto +169.109 384.758 170.452 386.101 170.452 387.758 curveto +closepath +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +184.843 386.428 M +184.843 386.99 184.64 387.498 QUADTO +184.437 388.006 184.077 388.365 QUADTO +183.64 388.818 183.03 389.045 QUADTO +182.421 389.271 181.483 389.271 QUADTO +180.499 389.271 lineto +180.499 392.521 lineto +179.343 392.521 lineto +179.343 383.803 lineto +181.53 383.803 lineto +182.296 383.803 182.843 383.928 QUADTO +183.39 384.053 183.812 384.334 QUADTO +184.296 384.678 184.569 385.186 QUADTO +184.843 385.693 184.843 386.428 QUADTO +closepath +183.64 386.459 M +183.64 386.006 183.483 385.67 QUADTO +183.327 385.334 183.015 385.131 QUADTO +182.733 384.943 182.366 384.865 QUADTO +181.999 384.787 181.452 384.787 QUADTO +180.499 384.787 lineto +180.499 388.271 lineto +181.296 388.271 lineto +181.999 388.271 182.429 388.162 QUADTO +182.858 388.053 183.14 387.756 QUADTO +183.405 387.49 183.522 387.178 QUADTO +183.64 386.865 183.64 386.459 QUADTO +closepath +185.005 392.521 M +187.958 383.803 lineto +189.38 383.803 lineto +192.317 392.521 lineto +191.099 392.521 lineto +190.302 390.084 lineto +186.974 390.084 lineto +186.177 392.521 lineto +185.005 392.521 lineto +closepath +189.989 389.1 M +188.63 384.975 lineto +187.302 389.1 lineto +189.989 389.1 lineto +closepath +195.575 392.678 M +194.638 392.678 193.966 392.482 QUADTO +193.294 392.287 192.716 392.006 QUADTO +192.716 390.553 lineto +192.794 390.553 lineto +193.372 391.084 194.099 391.381 QUADTO +194.825 391.678 195.497 391.678 QUADTO +196.466 391.678 196.958 391.279 QUADTO +197.45 390.881 197.45 390.225 QUADTO +197.45 389.693 197.192 389.365 QUADTO +196.935 389.037 196.372 388.881 QUADTO +195.966 388.771 195.63 388.693 QUADTO +195.294 388.615 194.794 388.49 QUADTO +194.341 388.365 193.981 388.186 QUADTO +193.622 388.006 193.341 387.709 QUADTO +193.075 387.428 192.927 387.037 QUADTO +192.778 386.646 192.778 386.131 QUADTO +192.778 385.068 193.606 384.35 QUADTO +194.435 383.631 195.716 383.631 QUADTO +196.45 383.631 197.099 383.779 QUADTO +197.747 383.928 198.31 384.193 QUADTO +198.31 385.584 lineto +198.216 385.584 lineto +197.81 385.209 197.161 384.928 QUADTO +196.513 384.646 195.794 384.646 QUADTO +194.981 384.646 194.481 385.021 QUADTO +193.981 385.396 193.981 386.006 QUADTO +193.981 386.568 194.278 386.896 QUADTO +194.575 387.225 195.122 387.381 QUADTO +195.497 387.475 196.028 387.6 QUADTO +196.56 387.725 196.919 387.834 QUADTO +197.825 388.115 198.239 388.67 QUADTO +198.653 389.225 198.653 390.037 QUADTO +198.653 390.553 198.442 391.045 QUADTO +198.231 391.537 197.856 391.881 QUADTO +197.435 392.271 196.903 392.475 QUADTO +196.372 392.678 195.575 392.678 QUADTO +closepath +201.87 392.521 M +201.87 384.834 lineto +198.948 384.834 lineto +198.948 383.803 lineto +205.964 383.803 lineto +205.964 384.834 lineto +203.026 384.834 lineto +203.026 392.521 lineto +201.87 392.521 lineto +closepath +216.831 392.521 M +215.534 392.521 lineto +211.675 384.99 lineto +211.675 392.521 lineto +210.597 392.521 lineto +210.597 383.803 lineto +212.237 383.803 lineto +215.753 390.709 lineto +215.753 383.803 lineto +216.831 383.803 lineto +216.831 392.521 lineto +closepath +225.325 387.943 M +218.841 387.943 lineto +218.841 387.006 lineto +225.325 387.006 lineto +225.325 387.943 lineto +closepath +225.325 390.396 M +218.841 390.396 lineto +218.841 389.459 lineto +225.325 389.459 lineto +225.325 390.396 lineto +closepath +232.024 392.521 M +227.774 392.521 lineto +227.774 391.631 lineto +229.353 391.631 lineto +229.353 385.787 lineto +227.774 385.787 lineto +227.774 384.99 lineto +228.665 384.99 229.095 384.732 QUADTO +229.524 384.475 229.571 383.771 QUADTO +230.478 383.771 lineto +230.478 391.631 lineto +232.024 391.631 lineto +232.024 392.521 lineto +closepath +239.153 389.693 M +239.153 390.365 238.935 390.936 QUADTO +238.716 391.506 238.341 391.881 QUADTO +237.95 392.287 237.458 392.498 QUADTO +236.966 392.709 236.388 392.709 QUADTO +235.794 392.709 235.31 392.506 QUADTO +234.825 392.303 234.45 391.912 QUADTO +234.013 391.443 233.755 390.646 QUADTO +233.497 389.85 233.497 388.693 QUADTO +233.497 387.553 233.731 386.639 QUADTO +233.966 385.725 234.45 385.037 QUADTO +234.919 384.381 235.669 384.006 QUADTO +236.419 383.631 237.403 383.631 QUADTO +237.7 383.631 237.95 383.662 QUADTO +238.2 383.693 238.403 383.756 QUADTO +238.403 384.881 lineto +238.341 384.881 lineto +238.185 384.787 237.888 384.717 QUADTO +237.591 384.646 237.247 384.646 QUADTO +236.169 384.646 235.497 385.381 QUADTO +234.825 386.115 234.7 387.553 QUADTO +235.106 387.271 235.552 387.107 QUADTO +235.997 386.943 236.513 386.943 QUADTO +237.028 386.943 237.411 387.045 QUADTO +237.794 387.146 238.2 387.459 QUADTO +238.685 387.834 238.919 388.389 QUADTO +239.153 388.943 239.153 389.693 QUADTO +closepath +237.966 389.74 M +237.966 389.193 237.825 388.834 QUADTO +237.685 388.475 237.372 388.209 QUADTO +237.138 388.021 236.849 387.967 QUADTO +236.56 387.912 236.263 387.912 QUADTO +235.841 387.912 235.458 388.037 QUADTO +235.075 388.162 234.7 388.396 QUADTO +234.685 388.521 234.677 388.639 QUADTO +234.669 388.756 234.669 388.943 QUADTO +234.669 389.85 234.849 390.404 QUADTO +235.028 390.959 235.294 391.256 QUADTO +235.544 391.521 235.81 391.631 QUADTO +236.075 391.74 236.388 391.74 QUADTO +237.091 391.74 237.528 391.24 QUADTO +237.966 390.74 237.966 389.74 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +2 setlinecap +2 setlinejoin +2 setlinejoin +newpath +245.548 387.758 M +259.548 387.758 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +newpath +252.548 384.758 M +255.548 390.758 lineto +249.548 390.758 lineto +closepath +eofill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +252.548 384.758 M +255.548 390.758 lineto +249.548 390.758 lineto +closepath +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +269.938 386.428 M +269.938 386.99 269.735 387.498 QUADTO +269.532 388.006 269.173 388.365 QUADTO +268.735 388.818 268.126 389.045 QUADTO +267.517 389.271 266.579 389.271 QUADTO +265.595 389.271 lineto +265.595 392.521 lineto +264.438 392.521 lineto +264.438 383.803 lineto +266.626 383.803 lineto +267.392 383.803 267.938 383.928 QUADTO +268.485 384.053 268.907 384.334 QUADTO +269.392 384.678 269.665 385.186 QUADTO +269.938 385.693 269.938 386.428 QUADTO +closepath +268.735 386.459 M +268.735 386.006 268.579 385.67 QUADTO +268.423 385.334 268.11 385.131 QUADTO +267.829 384.943 267.462 384.865 QUADTO +267.095 384.787 266.548 384.787 QUADTO +265.595 384.787 lineto +265.595 388.271 lineto +266.392 388.271 lineto +267.095 388.271 267.524 388.162 QUADTO +267.954 388.053 268.235 387.756 QUADTO +268.501 387.49 268.618 387.178 QUADTO +268.735 386.865 268.735 386.459 QUADTO +closepath +270.101 392.521 M +273.054 383.803 lineto +274.476 383.803 lineto +277.413 392.521 lineto +276.194 392.521 lineto +275.397 390.084 lineto +272.069 390.084 lineto +271.272 392.521 lineto +270.101 392.521 lineto +closepath +275.085 389.1 M +273.726 384.975 lineto +272.397 389.1 lineto +275.085 389.1 lineto +closepath +280.671 392.678 M +279.733 392.678 279.062 392.482 QUADTO +278.39 392.287 277.812 392.006 QUADTO +277.812 390.553 lineto +277.89 390.553 lineto +278.468 391.084 279.194 391.381 QUADTO +279.921 391.678 280.593 391.678 QUADTO +281.562 391.678 282.054 391.279 QUADTO +282.546 390.881 282.546 390.225 QUADTO +282.546 389.693 282.288 389.365 QUADTO +282.03 389.037 281.468 388.881 QUADTO +281.062 388.771 280.726 388.693 QUADTO +280.39 388.615 279.89 388.49 QUADTO +279.437 388.365 279.077 388.186 QUADTO +278.718 388.006 278.437 387.709 QUADTO +278.171 387.428 278.022 387.037 QUADTO +277.874 386.646 277.874 386.131 QUADTO +277.874 385.068 278.702 384.35 QUADTO +279.53 383.631 280.812 383.631 QUADTO +281.546 383.631 282.194 383.779 QUADTO +282.843 383.928 283.405 384.193 QUADTO +283.405 385.584 lineto +283.312 385.584 lineto +282.905 385.209 282.257 384.928 QUADTO +281.608 384.646 280.89 384.646 QUADTO +280.077 384.646 279.577 385.021 QUADTO +279.077 385.396 279.077 386.006 QUADTO +279.077 386.568 279.374 386.896 QUADTO +279.671 387.225 280.218 387.381 QUADTO +280.593 387.475 281.124 387.6 QUADTO +281.655 387.725 282.015 387.834 QUADTO +282.921 388.115 283.335 388.67 QUADTO +283.749 389.225 283.749 390.037 QUADTO +283.749 390.553 283.538 391.045 QUADTO +283.327 391.537 282.952 391.881 QUADTO +282.53 392.271 281.999 392.475 QUADTO +281.468 392.678 280.671 392.678 QUADTO +closepath +286.966 392.521 M +286.966 384.834 lineto +284.044 384.834 lineto +284.044 383.803 lineto +291.06 383.803 lineto +291.06 384.834 lineto +288.122 384.834 lineto +288.122 392.521 lineto +286.966 392.521 lineto +closepath +301.927 392.521 M +300.63 392.521 lineto +296.771 384.99 lineto +296.771 392.521 lineto +295.692 392.521 lineto +295.692 383.803 lineto +297.333 383.803 lineto +300.849 390.709 lineto +300.849 383.803 lineto +301.927 383.803 lineto +301.927 392.521 lineto +closepath +310.421 387.943 M +303.937 387.943 lineto +303.937 387.006 lineto +310.421 387.006 lineto +310.421 387.943 lineto +closepath +310.421 390.396 M +303.937 390.396 lineto +303.937 389.459 lineto +310.421 389.459 lineto +310.421 390.396 lineto +closepath +317.12 392.521 M +312.87 392.521 lineto +312.87 391.631 lineto +314.448 391.631 lineto +314.448 385.787 lineto +312.87 385.787 lineto +312.87 384.99 lineto +313.761 384.99 314.19 384.732 QUADTO +314.62 384.475 314.667 383.771 QUADTO +315.573 383.771 lineto +315.573 391.631 lineto +317.12 391.631 lineto +317.12 392.521 lineto +closepath +322.843 385.834 M +322.843 385.24 322.452 384.85 QUADTO +322.062 384.459 321.358 384.459 QUADTO +320.718 384.459 320.312 384.811 QUADTO +319.905 385.162 319.905 385.756 QUADTO +319.905 386.193 320.116 386.49 QUADTO +320.327 386.787 320.749 387.037 QUADTO +320.921 387.131 321.296 387.303 QUADTO +321.671 387.475 321.968 387.584 QUADTO +322.483 387.193 322.663 386.779 QUADTO +322.843 386.365 322.843 385.834 QUADTO +closepath +323.015 390.193 M +323.015 389.662 322.804 389.342 QUADTO +322.593 389.021 321.983 388.693 QUADTO +321.78 388.584 321.452 388.459 QUADTO +321.124 388.334 320.702 388.162 QUADTO +320.218 388.475 319.96 388.951 QUADTO +319.702 389.428 319.702 390.021 QUADTO +319.702 390.818 320.179 391.334 QUADTO +320.655 391.85 321.374 391.85 QUADTO +322.124 391.85 322.569 391.404 QUADTO +323.015 390.959 323.015 390.193 QUADTO +closepath +321.374 392.725 M +320.718 392.725 320.187 392.529 QUADTO +319.655 392.334 319.28 391.975 QUADTO +318.905 391.615 318.718 391.131 QUADTO +318.53 390.646 318.53 390.115 QUADTO +318.53 389.396 318.905 388.803 QUADTO +319.28 388.209 319.999 387.865 QUADTO +319.999 387.834 lineto +319.358 387.459 319.046 387.014 QUADTO +318.733 386.568 318.733 385.896 QUADTO +318.733 384.912 319.483 384.256 QUADTO +320.233 383.6 321.374 383.6 QUADTO +322.562 383.6 323.28 384.225 QUADTO +323.999 384.85 323.999 385.818 QUADTO +323.999 386.396 323.655 386.975 QUADTO +323.312 387.553 322.671 387.881 QUADTO +322.671 387.912 lineto +323.421 388.271 323.819 388.779 QUADTO +324.218 389.287 324.218 390.084 QUADTO +324.218 391.209 323.405 391.967 QUADTO +322.593 392.725 321.374 392.725 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +1 setgray +newpath +98.938 34.141 M +388 34.141 lineto +388 326.133 lineto +98.938 326.133 lineto +98.938 34.141 lineto +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +98.938 330.133 M +388 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +107.829 346.139 M +103.579 346.139 lineto +103.579 345.248 lineto +105.157 345.248 lineto +105.157 339.404 lineto +103.579 339.404 lineto +103.579 338.607 lineto +104.469 338.607 104.899 338.35 QUADTO +105.329 338.092 105.376 337.389 QUADTO +106.282 337.389 lineto +106.282 345.248 lineto +107.829 345.248 lineto +107.829 346.139 lineto +closepath +114.723 343.357 M +114.723 343.998 114.52 344.537 QUADTO +114.317 345.076 113.958 345.467 QUADTO +113.583 345.857 113.036 346.092 QUADTO +112.489 346.326 111.817 346.326 QUADTO +111.192 346.326 110.583 346.186 QUADTO +109.973 346.045 109.536 345.826 QUADTO +109.536 344.592 lineto +109.614 344.592 lineto +109.739 344.686 109.997 344.811 QUADTO +110.254 344.936 110.551 345.045 QUADTO +110.895 345.17 111.184 345.24 QUADTO +111.473 345.311 111.833 345.311 QUADTO +112.176 345.311 112.512 345.186 QUADTO +112.848 345.061 113.098 344.779 QUADTO +113.301 344.514 113.419 344.17 QUADTO +113.536 343.826 113.536 343.357 QUADTO +113.536 342.889 113.411 342.592 QUADTO +113.286 342.295 113.051 342.107 QUADTO +112.786 341.873 112.426 341.787 QUADTO +112.067 341.701 111.614 341.701 QUADTO +111.129 341.701 110.653 341.771 QUADTO +110.176 341.842 109.879 341.904 QUADTO +109.879 337.42 lineto +114.676 337.42 lineto +114.676 338.436 lineto +111.02 338.436 lineto +111.02 340.764 lineto +111.223 340.732 111.434 340.725 QUADTO +111.645 340.717 111.801 340.717 QUADTO +112.473 340.717 112.934 340.826 QUADTO +113.395 340.936 113.833 341.264 QUADTO +114.27 341.592 114.497 342.084 QUADTO +114.723 342.576 114.723 343.357 QUADTO +closepath +121.415 341.779 M +121.415 344.123 120.735 345.225 QUADTO +120.055 346.326 118.633 346.326 QUADTO +117.18 346.326 116.508 345.201 QUADTO +115.836 344.076 115.836 341.779 QUADTO +115.836 339.42 116.524 338.326 QUADTO +117.211 337.232 118.633 337.232 QUADTO +120.071 337.232 120.743 338.365 QUADTO +121.415 339.498 121.415 341.779 QUADTO +closepath +119.915 344.436 M +120.086 343.998 120.157 343.334 QUADTO +120.227 342.67 120.227 341.779 QUADTO +120.227 340.873 120.157 340.209 QUADTO +120.086 339.545 119.915 339.107 QUADTO +119.743 338.67 119.43 338.436 QUADTO +119.118 338.201 118.633 338.201 QUADTO +118.149 338.201 117.829 338.436 QUADTO +117.508 338.67 117.336 339.123 QUADTO +117.165 339.576 117.094 340.256 QUADTO +117.024 340.936 117.024 341.779 QUADTO +117.024 342.701 117.079 343.318 QUADTO +117.133 343.936 117.321 344.42 QUADTO +117.493 344.873 117.805 345.115 QUADTO +118.118 345.357 118.633 345.357 QUADTO +119.118 345.357 119.43 345.123 QUADTO +119.743 344.889 119.915 344.436 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +112.077 332.133 M +112.077 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +137.48 346.139 M +132.089 346.139 lineto +132.089 344.92 lineto +132.668 344.357 133.183 343.857 QUADTO +133.699 343.357 134.199 342.811 QUADTO +135.183 341.717 135.55 341.053 QUADTO +135.918 340.389 135.918 339.686 QUADTO +135.918 339.326 135.8 339.053 QUADTO +135.683 338.779 135.48 338.592 QUADTO +135.277 338.42 135.011 338.334 QUADTO +134.746 338.248 134.418 338.248 QUADTO +134.105 338.248 133.785 338.326 QUADTO +133.464 338.404 133.183 338.514 QUADTO +132.933 338.623 132.707 338.764 QUADTO +132.48 338.904 132.339 338.998 QUADTO +132.277 338.998 lineto +132.277 337.764 lineto +132.636 337.576 133.285 337.404 QUADTO +133.933 337.232 134.511 337.232 QUADTO +135.73 337.232 136.418 337.881 QUADTO +137.105 338.529 137.105 339.639 QUADTO +137.105 340.139 136.988 340.576 QUADTO +136.871 341.014 136.652 341.373 QUADTO +136.449 341.764 136.144 342.139 QUADTO +135.839 342.514 135.496 342.873 QUADTO +134.949 343.467 134.269 344.115 QUADTO +133.589 344.764 133.183 345.139 QUADTO +137.48 345.139 lineto +137.48 346.139 lineto +closepath +144.062 341.779 M +144.062 344.123 143.382 345.225 QUADTO +142.703 346.326 141.281 346.326 QUADTO +139.828 346.326 139.156 345.201 QUADTO +138.484 344.076 138.484 341.779 QUADTO +138.484 339.42 139.171 338.326 QUADTO +139.859 337.232 141.281 337.232 QUADTO +142.718 337.232 143.39 338.365 QUADTO +144.062 339.498 144.062 341.779 QUADTO +closepath +142.562 344.436 M +142.734 343.998 142.804 343.334 QUADTO +142.875 342.67 142.875 341.779 QUADTO +142.875 340.873 142.804 340.209 QUADTO +142.734 339.545 142.562 339.107 QUADTO +142.39 338.67 142.078 338.436 QUADTO +141.765 338.201 141.281 338.201 QUADTO +140.796 338.201 140.476 338.436 QUADTO +140.156 338.67 139.984 339.123 QUADTO +139.812 339.576 139.742 340.256 QUADTO +139.671 340.936 139.671 341.779 QUADTO +139.671 342.701 139.726 343.318 QUADTO +139.781 343.936 139.968 344.42 QUADTO +140.14 344.873 140.453 345.115 QUADTO +140.765 345.357 141.281 345.357 QUADTO +141.765 345.357 142.078 345.123 QUADTO +142.39 344.889 142.562 344.436 QUADTO +closepath +150.613 341.779 M +150.613 344.123 149.933 345.225 QUADTO +149.253 346.326 147.832 346.326 QUADTO +146.378 346.326 145.707 345.201 QUADTO +145.035 344.076 145.035 341.779 QUADTO +145.035 339.42 145.722 338.326 QUADTO +146.41 337.232 147.832 337.232 QUADTO +149.269 337.232 149.941 338.365 QUADTO +150.613 339.498 150.613 341.779 QUADTO +closepath +149.113 344.436 M +149.285 343.998 149.355 343.334 QUADTO +149.425 342.67 149.425 341.779 QUADTO +149.425 340.873 149.355 340.209 QUADTO +149.285 339.545 149.113 339.107 QUADTO +148.941 338.67 148.628 338.436 QUADTO +148.316 338.201 147.832 338.201 QUADTO +147.347 338.201 147.027 338.436 QUADTO +146.707 338.67 146.535 339.123 QUADTO +146.363 339.576 146.293 340.256 QUADTO +146.222 340.936 146.222 341.779 QUADTO +146.222 342.701 146.277 343.318 QUADTO +146.332 343.936 146.519 344.42 QUADTO +146.691 344.873 147.003 345.115 QUADTO +147.316 345.357 147.832 345.357 QUADTO +148.316 345.357 148.628 345.123 QUADTO +148.941 344.889 149.113 344.436 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +141.275 332.133 M +141.275 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +166.678 346.139 M +161.288 346.139 lineto +161.288 344.92 lineto +161.866 344.357 162.381 343.857 QUADTO +162.897 343.357 163.397 342.811 QUADTO +164.381 341.717 164.749 341.053 QUADTO +165.116 340.389 165.116 339.686 QUADTO +165.116 339.326 164.999 339.053 QUADTO +164.881 338.779 164.678 338.592 QUADTO +164.475 338.42 164.21 338.334 QUADTO +163.944 338.248 163.616 338.248 QUADTO +163.303 338.248 162.983 338.326 QUADTO +162.663 338.404 162.381 338.514 QUADTO +162.131 338.623 161.905 338.764 QUADTO +161.678 338.904 161.538 338.998 QUADTO +161.475 338.998 lineto +161.475 337.764 lineto +161.835 337.576 162.483 337.404 QUADTO +163.131 337.232 163.71 337.232 QUADTO +164.928 337.232 165.616 337.881 QUADTO +166.303 338.529 166.303 339.639 QUADTO +166.303 340.139 166.186 340.576 QUADTO +166.069 341.014 165.85 341.373 QUADTO +165.647 341.764 165.342 342.139 QUADTO +165.038 342.514 164.694 342.873 QUADTO +164.147 343.467 163.467 344.115 QUADTO +162.788 344.764 162.381 345.139 QUADTO +166.678 345.139 lineto +166.678 346.139 lineto +closepath +173.12 343.357 M +173.12 343.998 172.917 344.537 QUADTO +172.713 345.076 172.354 345.467 QUADTO +171.979 345.857 171.432 346.092 QUADTO +170.885 346.326 170.213 346.326 QUADTO +169.588 346.326 168.979 346.186 QUADTO +168.37 346.045 167.932 345.826 QUADTO +167.932 344.592 lineto +168.01 344.592 lineto +168.135 344.686 168.393 344.811 QUADTO +168.651 344.936 168.948 345.045 QUADTO +169.292 345.17 169.581 345.24 QUADTO +169.87 345.311 170.229 345.311 QUADTO +170.573 345.311 170.909 345.186 QUADTO +171.245 345.061 171.495 344.779 QUADTO +171.698 344.514 171.815 344.17 QUADTO +171.932 343.826 171.932 343.357 QUADTO +171.932 342.889 171.807 342.592 QUADTO +171.682 342.295 171.448 342.107 QUADTO +171.182 341.873 170.823 341.787 QUADTO +170.463 341.701 170.01 341.701 QUADTO +169.526 341.701 169.049 341.771 QUADTO +168.573 341.842 168.276 341.904 QUADTO +168.276 337.42 lineto +173.073 337.42 lineto +173.073 338.436 lineto +169.417 338.436 lineto +169.417 340.764 lineto +169.62 340.732 169.831 340.725 QUADTO +170.042 340.717 170.198 340.717 QUADTO +170.87 340.717 171.331 340.826 QUADTO +171.792 340.936 172.229 341.264 QUADTO +172.667 341.592 172.893 342.084 QUADTO +173.12 342.576 173.12 343.357 QUADTO +closepath +179.811 341.779 M +179.811 344.123 179.131 345.225 QUADTO +178.452 346.326 177.03 346.326 QUADTO +175.577 346.326 174.905 345.201 QUADTO +174.233 344.076 174.233 341.779 QUADTO +174.233 339.42 174.92 338.326 QUADTO +175.608 337.232 177.03 337.232 QUADTO +178.467 337.232 179.139 338.365 QUADTO +179.811 339.498 179.811 341.779 QUADTO +closepath +178.311 344.436 M +178.483 343.998 178.553 343.334 QUADTO +178.624 342.67 178.624 341.779 QUADTO +178.624 340.873 178.553 340.209 QUADTO +178.483 339.545 178.311 339.107 QUADTO +178.139 338.67 177.827 338.436 QUADTO +177.514 338.201 177.03 338.201 QUADTO +176.545 338.201 176.225 338.436 QUADTO +175.905 338.67 175.733 339.123 QUADTO +175.561 339.576 175.491 340.256 QUADTO +175.42 340.936 175.42 341.779 QUADTO +175.42 342.701 175.475 343.318 QUADTO +175.53 343.936 175.717 344.42 QUADTO +175.889 344.873 176.202 345.115 QUADTO +176.514 345.357 177.03 345.357 QUADTO +177.514 345.357 177.827 345.123 QUADTO +178.139 344.889 178.311 344.436 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +170.473 332.133 M +170.473 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +192.845 338.248 M +192.517 338.248 192.205 338.326 QUADTO +191.892 338.404 191.58 338.514 QUADTO +191.298 338.639 191.095 338.764 QUADTO +190.892 338.889 190.72 339.014 QUADTO +190.658 339.014 lineto +190.658 337.764 lineto +191.033 337.561 191.689 337.396 QUADTO +192.345 337.232 192.939 337.232 QUADTO +193.533 337.232 193.97 337.35 QUADTO +194.408 337.467 194.767 337.717 QUADTO +195.158 337.998 195.353 338.396 QUADTO +195.548 338.795 195.548 339.342 QUADTO +195.548 340.061 195.072 340.6 QUADTO +194.595 341.139 193.939 341.295 QUADTO +193.939 341.373 lineto +194.205 341.42 194.525 341.553 QUADTO +194.845 341.686 195.111 341.936 QUADTO +195.376 342.186 195.548 342.584 QUADTO +195.72 342.982 195.72 343.545 QUADTO +195.72 344.139 195.517 344.631 QUADTO +195.314 345.123 194.939 345.514 QUADTO +194.564 345.904 194.025 346.115 QUADTO +193.486 346.326 192.798 346.326 QUADTO +192.142 346.326 191.501 346.162 QUADTO +190.861 345.998 190.408 345.779 QUADTO +190.408 344.545 lineto +190.501 344.545 lineto +190.861 344.811 191.486 345.061 QUADTO +192.111 345.311 192.736 345.311 QUADTO +193.08 345.311 193.447 345.193 QUADTO +193.814 345.076 194.048 344.826 QUADTO +194.283 344.561 194.408 344.248 QUADTO +194.533 343.936 194.533 343.467 QUADTO +194.533 342.998 194.392 342.693 QUADTO +194.251 342.389 194.001 342.201 QUADTO +193.767 342.029 193.423 341.959 QUADTO +193.08 341.889 192.689 341.889 QUADTO +192.189 341.889 lineto +192.189 340.92 lineto +192.58 340.92 lineto +193.376 340.92 193.869 340.545 QUADTO +194.361 340.17 194.361 339.451 QUADTO +194.361 339.123 194.236 338.889 QUADTO +194.111 338.654 193.908 338.514 QUADTO +193.689 338.357 193.416 338.303 QUADTO +193.142 338.248 192.845 338.248 QUADTO +closepath +202.459 341.779 M +202.459 344.123 201.779 345.225 QUADTO +201.099 346.326 199.677 346.326 QUADTO +198.224 346.326 197.552 345.201 QUADTO +196.88 344.076 196.88 341.779 QUADTO +196.88 339.42 197.568 338.326 QUADTO +198.255 337.232 199.677 337.232 QUADTO +201.115 337.232 201.787 338.365 QUADTO +202.459 339.498 202.459 341.779 QUADTO +closepath +200.959 344.436 M +201.13 343.998 201.201 343.334 QUADTO +201.271 342.67 201.271 341.779 QUADTO +201.271 340.873 201.201 340.209 QUADTO +201.13 339.545 200.959 339.107 QUADTO +200.787 338.67 200.474 338.436 QUADTO +200.162 338.201 199.677 338.201 QUADTO +199.193 338.201 198.873 338.436 QUADTO +198.552 338.67 198.38 339.123 QUADTO +198.209 339.576 198.138 340.256 QUADTO +198.068 340.936 198.068 341.779 QUADTO +198.068 342.701 198.123 343.318 QUADTO +198.177 343.936 198.365 344.42 QUADTO +198.537 344.873 198.849 345.115 QUADTO +199.162 345.357 199.677 345.357 QUADTO +200.162 345.357 200.474 345.123 QUADTO +200.787 344.889 200.959 344.436 QUADTO +closepath +209.009 341.779 M +209.009 344.123 208.33 345.225 QUADTO +207.65 346.326 206.228 346.326 QUADTO +204.775 346.326 204.103 345.201 QUADTO +203.431 344.076 203.431 341.779 QUADTO +203.431 339.42 204.119 338.326 QUADTO +204.806 337.232 206.228 337.232 QUADTO +207.666 337.232 208.337 338.365 QUADTO +209.009 339.498 209.009 341.779 QUADTO +closepath +207.509 344.436 M +207.681 343.998 207.751 343.334 QUADTO +207.822 342.67 207.822 341.779 QUADTO +207.822 340.873 207.751 340.209 QUADTO +207.681 339.545 207.509 339.107 QUADTO +207.337 338.67 207.025 338.436 QUADTO +206.712 338.201 206.228 338.201 QUADTO +205.744 338.201 205.423 338.436 QUADTO +205.103 338.67 204.931 339.123 QUADTO +204.759 339.576 204.689 340.256 QUADTO +204.619 340.936 204.619 341.779 QUADTO +204.619 342.701 204.673 343.318 QUADTO +204.728 343.936 204.916 344.42 QUADTO +205.087 344.873 205.4 345.115 QUADTO +205.712 345.357 206.228 345.357 QUADTO +206.712 345.357 207.025 345.123 QUADTO +207.337 344.889 207.509 344.436 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +199.671 332.133 M +199.671 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +222.043 338.248 M +221.715 338.248 221.403 338.326 QUADTO +221.09 338.404 220.778 338.514 QUADTO +220.497 338.639 220.293 338.764 QUADTO +220.09 338.889 219.918 339.014 QUADTO +219.856 339.014 lineto +219.856 337.764 lineto +220.231 337.561 220.887 337.396 QUADTO +221.543 337.232 222.137 337.232 QUADTO +222.731 337.232 223.168 337.35 QUADTO +223.606 337.467 223.965 337.717 QUADTO +224.356 337.998 224.551 338.396 QUADTO +224.747 338.795 224.747 339.342 QUADTO +224.747 340.061 224.27 340.6 QUADTO +223.793 341.139 223.137 341.295 QUADTO +223.137 341.373 lineto +223.403 341.42 223.723 341.553 QUADTO +224.043 341.686 224.309 341.936 QUADTO +224.575 342.186 224.747 342.584 QUADTO +224.918 342.982 224.918 343.545 QUADTO +224.918 344.139 224.715 344.631 QUADTO +224.512 345.123 224.137 345.514 QUADTO +223.762 345.904 223.223 346.115 QUADTO +222.684 346.326 221.997 346.326 QUADTO +221.34 346.326 220.7 346.162 QUADTO +220.059 345.998 219.606 345.779 QUADTO +219.606 344.545 lineto +219.7 344.545 lineto +220.059 344.811 220.684 345.061 QUADTO +221.309 345.311 221.934 345.311 QUADTO +222.278 345.311 222.645 345.193 QUADTO +223.012 345.076 223.247 344.826 QUADTO +223.481 344.561 223.606 344.248 QUADTO +223.731 343.936 223.731 343.467 QUADTO +223.731 342.998 223.59 342.693 QUADTO +223.45 342.389 223.2 342.201 QUADTO +222.965 342.029 222.622 341.959 QUADTO +222.278 341.889 221.887 341.889 QUADTO +221.387 341.889 lineto +221.387 340.92 lineto +221.778 340.92 lineto +222.575 340.92 223.067 340.545 QUADTO +223.559 340.17 223.559 339.451 QUADTO +223.559 339.123 223.434 338.889 QUADTO +223.309 338.654 223.106 338.514 QUADTO +222.887 338.357 222.614 338.303 QUADTO +222.34 338.248 222.043 338.248 QUADTO +closepath +231.516 343.357 M +231.516 343.998 231.313 344.537 QUADTO +231.11 345.076 230.75 345.467 QUADTO +230.375 345.857 229.829 346.092 QUADTO +229.282 346.326 228.61 346.326 QUADTO +227.985 346.326 227.375 346.186 QUADTO +226.766 346.045 226.329 345.826 QUADTO +226.329 344.592 lineto +226.407 344.592 lineto +226.532 344.686 226.79 344.811 QUADTO +227.047 344.936 227.344 345.045 QUADTO +227.688 345.17 227.977 345.24 QUADTO +228.266 345.311 228.625 345.311 QUADTO +228.969 345.311 229.305 345.186 QUADTO +229.641 345.061 229.891 344.779 QUADTO +230.094 344.514 230.211 344.17 QUADTO +230.329 343.826 230.329 343.357 QUADTO +230.329 342.889 230.204 342.592 QUADTO +230.079 342.295 229.844 342.107 QUADTO +229.579 341.873 229.219 341.787 QUADTO +228.86 341.701 228.407 341.701 QUADTO +227.922 341.701 227.446 341.771 QUADTO +226.969 341.842 226.672 341.904 QUADTO +226.672 337.42 lineto +231.469 337.42 lineto +231.469 338.436 lineto +227.813 338.436 lineto +227.813 340.764 lineto +228.016 340.732 228.227 340.725 QUADTO +228.438 340.717 228.594 340.717 QUADTO +229.266 340.717 229.727 340.826 QUADTO +230.188 340.936 230.625 341.264 QUADTO +231.063 341.592 231.29 342.084 QUADTO +231.516 342.576 231.516 343.357 QUADTO +closepath +238.208 341.779 M +238.208 344.123 237.528 345.225 QUADTO +236.848 346.326 235.426 346.326 QUADTO +233.973 346.326 233.301 345.201 QUADTO +232.629 344.076 232.629 341.779 QUADTO +232.629 339.42 233.317 338.326 QUADTO +234.004 337.232 235.426 337.232 QUADTO +236.864 337.232 237.536 338.365 QUADTO +238.208 339.498 238.208 341.779 QUADTO +closepath +236.708 344.436 M +236.879 343.998 236.95 343.334 QUADTO +237.02 342.67 237.02 341.779 QUADTO +237.02 340.873 236.95 340.209 QUADTO +236.879 339.545 236.708 339.107 QUADTO +236.536 338.67 236.223 338.436 QUADTO +235.911 338.201 235.426 338.201 QUADTO +234.942 338.201 234.622 338.436 QUADTO +234.301 338.67 234.129 339.123 QUADTO +233.958 339.576 233.887 340.256 QUADTO +233.817 340.936 233.817 341.779 QUADTO +233.817 342.701 233.872 343.318 QUADTO +233.926 343.936 234.114 344.42 QUADTO +234.286 344.873 234.598 345.115 QUADTO +234.911 345.357 235.426 345.357 QUADTO +235.911 345.357 236.223 345.123 QUADTO +236.536 344.889 236.708 344.436 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +228.87 332.133 M +228.87 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +254.492 343.139 M +253.46 343.139 lineto +253.46 346.139 lineto +252.335 346.139 lineto +252.335 343.139 lineto +248.476 343.139 lineto +248.476 341.904 lineto +252.382 337.42 lineto +253.46 337.42 lineto +253.46 342.201 lineto +254.492 342.201 lineto +254.492 343.139 lineto +closepath +252.335 342.201 M +252.335 338.795 lineto +249.398 342.201 lineto +252.335 342.201 lineto +closepath +260.855 341.779 M +260.855 344.123 260.175 345.225 QUADTO +259.496 346.326 258.074 346.326 QUADTO +256.621 346.326 255.949 345.201 QUADTO +255.277 344.076 255.277 341.779 QUADTO +255.277 339.42 255.964 338.326 QUADTO +256.652 337.232 258.074 337.232 QUADTO +259.511 337.232 260.183 338.365 QUADTO +260.855 339.498 260.855 341.779 QUADTO +closepath +259.355 344.436 M +259.527 343.998 259.597 343.334 QUADTO +259.667 342.67 259.667 341.779 QUADTO +259.667 340.873 259.597 340.209 QUADTO +259.527 339.545 259.355 339.107 QUADTO +259.183 338.67 258.871 338.436 QUADTO +258.558 338.201 258.074 338.201 QUADTO +257.589 338.201 257.269 338.436 QUADTO +256.949 338.67 256.777 339.123 QUADTO +256.605 339.576 256.535 340.256 QUADTO +256.464 340.936 256.464 341.779 QUADTO +256.464 342.701 256.519 343.318 QUADTO +256.574 343.936 256.761 344.42 QUADTO +256.933 344.873 257.246 345.115 QUADTO +257.558 345.357 258.074 345.357 QUADTO +258.558 345.357 258.871 345.123 QUADTO +259.183 344.889 259.355 344.436 QUADTO +closepath +267.406 341.779 M +267.406 344.123 266.726 345.225 QUADTO +266.046 346.326 264.625 346.326 QUADTO +263.171 346.326 262.5 345.201 QUADTO +261.828 344.076 261.828 341.779 QUADTO +261.828 339.42 262.515 338.326 QUADTO +263.203 337.232 264.625 337.232 QUADTO +266.062 337.232 266.734 338.365 QUADTO +267.406 339.498 267.406 341.779 QUADTO +closepath +265.906 344.436 M +266.078 343.998 266.148 343.334 QUADTO +266.218 342.67 266.218 341.779 QUADTO +266.218 340.873 266.148 340.209 QUADTO +266.078 339.545 265.906 339.107 QUADTO +265.734 338.67 265.421 338.436 QUADTO +265.109 338.201 264.625 338.201 QUADTO +264.14 338.201 263.82 338.436 QUADTO +263.5 338.67 263.328 339.123 QUADTO +263.156 339.576 263.085 340.256 QUADTO +263.015 340.936 263.015 341.779 QUADTO +263.015 342.701 263.07 343.318 QUADTO +263.125 343.936 263.312 344.42 QUADTO +263.484 344.873 263.796 345.115 QUADTO +264.109 345.357 264.625 345.357 QUADTO +265.109 345.357 265.421 345.123 QUADTO +265.734 344.889 265.906 344.436 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +258.068 332.133 M +258.068 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +283.69 343.139 M +282.659 343.139 lineto +282.659 346.139 lineto +281.534 346.139 lineto +281.534 343.139 lineto +277.674 343.139 lineto +277.674 341.904 lineto +281.581 337.42 lineto +282.659 337.42 lineto +282.659 342.201 lineto +283.69 342.201 lineto +283.69 343.139 lineto +closepath +281.534 342.201 M +281.534 338.795 lineto +278.596 342.201 lineto +281.534 342.201 lineto +closepath +289.913 343.357 M +289.913 343.998 289.709 344.537 QUADTO +289.506 345.076 289.147 345.467 QUADTO +288.772 345.857 288.225 346.092 QUADTO +287.678 346.326 287.006 346.326 QUADTO +286.381 346.326 285.772 346.186 QUADTO +285.163 346.045 284.725 345.826 QUADTO +284.725 344.592 lineto +284.803 344.592 lineto +284.928 344.686 285.186 344.811 QUADTO +285.444 344.936 285.741 345.045 QUADTO +286.084 345.17 286.374 345.24 QUADTO +286.663 345.311 287.022 345.311 QUADTO +287.366 345.311 287.702 345.186 QUADTO +288.038 345.061 288.288 344.779 QUADTO +288.491 344.514 288.608 344.17 QUADTO +288.725 343.826 288.725 343.357 QUADTO +288.725 342.889 288.6 342.592 QUADTO +288.475 342.295 288.241 342.107 QUADTO +287.975 341.873 287.616 341.787 QUADTO +287.256 341.701 286.803 341.701 QUADTO +286.319 341.701 285.842 341.771 QUADTO +285.366 341.842 285.069 341.904 QUADTO +285.069 337.42 lineto +289.866 337.42 lineto +289.866 338.436 lineto +286.209 338.436 lineto +286.209 340.764 lineto +286.413 340.732 286.624 340.725 QUADTO +286.834 340.717 286.991 340.717 QUADTO +287.663 340.717 288.124 340.826 QUADTO +288.584 340.936 289.022 341.264 QUADTO +289.459 341.592 289.686 342.084 QUADTO +289.913 342.576 289.913 343.357 QUADTO +closepath +296.604 341.779 M +296.604 344.123 295.924 345.225 QUADTO +295.245 346.326 293.823 346.326 QUADTO +292.37 346.326 291.698 345.201 QUADTO +291.026 344.076 291.026 341.779 QUADTO +291.026 339.42 291.713 338.326 QUADTO +292.401 337.232 293.823 337.232 QUADTO +295.26 337.232 295.932 338.365 QUADTO +296.604 339.498 296.604 341.779 QUADTO +closepath +295.104 344.436 M +295.276 343.998 295.346 343.334 QUADTO +295.417 342.67 295.417 341.779 QUADTO +295.417 340.873 295.346 340.209 QUADTO +295.276 339.545 295.104 339.107 QUADTO +294.932 338.67 294.62 338.436 QUADTO +294.307 338.201 293.823 338.201 QUADTO +293.338 338.201 293.018 338.436 QUADTO +292.698 338.67 292.526 339.123 QUADTO +292.354 339.576 292.284 340.256 QUADTO +292.213 340.936 292.213 341.779 QUADTO +292.213 342.701 292.268 343.318 QUADTO +292.323 343.936 292.51 344.42 QUADTO +292.682 344.873 292.995 345.115 QUADTO +293.307 345.357 293.823 345.357 QUADTO +294.307 345.357 294.62 345.123 QUADTO +294.932 344.889 295.104 344.436 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +287.266 332.133 M +287.266 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +312.56 343.357 M +312.56 343.998 312.357 344.537 QUADTO +312.154 345.076 311.794 345.467 QUADTO +311.419 345.857 310.873 346.092 QUADTO +310.326 346.326 309.654 346.326 QUADTO +309.029 346.326 308.419 346.186 QUADTO +307.81 346.045 307.373 345.826 QUADTO +307.373 344.592 lineto +307.451 344.592 lineto +307.576 344.686 307.833 344.811 QUADTO +308.091 344.936 308.388 345.045 QUADTO +308.732 345.17 309.021 345.24 QUADTO +309.31 345.311 309.669 345.311 QUADTO +310.013 345.311 310.349 345.186 QUADTO +310.685 345.061 310.935 344.779 QUADTO +311.138 344.514 311.255 344.17 QUADTO +311.373 343.826 311.373 343.357 QUADTO +311.373 342.889 311.248 342.592 QUADTO +311.123 342.295 310.888 342.107 QUADTO +310.623 341.873 310.263 341.787 QUADTO +309.904 341.701 309.451 341.701 QUADTO +308.966 341.701 308.49 341.771 QUADTO +308.013 341.842 307.716 341.904 QUADTO +307.716 337.42 lineto +312.513 337.42 lineto +312.513 338.436 lineto +308.857 338.436 lineto +308.857 340.764 lineto +309.06 340.732 309.271 340.725 QUADTO +309.482 340.717 309.638 340.717 QUADTO +310.31 340.717 310.771 340.826 QUADTO +311.232 340.936 311.669 341.264 QUADTO +312.107 341.592 312.333 342.084 QUADTO +312.56 342.576 312.56 343.357 QUADTO +closepath +319.251 341.779 M +319.251 344.123 318.572 345.225 QUADTO +317.892 346.326 316.47 346.326 QUADTO +315.017 346.326 314.345 345.201 QUADTO +313.673 344.076 313.673 341.779 QUADTO +313.673 339.42 314.361 338.326 QUADTO +315.048 337.232 316.47 337.232 QUADTO +317.908 337.232 318.58 338.365 QUADTO +319.251 339.498 319.251 341.779 QUADTO +closepath +317.751 344.436 M +317.923 343.998 317.994 343.334 QUADTO +318.064 342.67 318.064 341.779 QUADTO +318.064 340.873 317.994 340.209 QUADTO +317.923 339.545 317.751 339.107 QUADTO +317.58 338.67 317.267 338.436 QUADTO +316.955 338.201 316.47 338.201 QUADTO +315.986 338.201 315.665 338.436 QUADTO +315.345 338.67 315.173 339.123 QUADTO +315.001 339.576 314.931 340.256 QUADTO +314.861 340.936 314.861 341.779 QUADTO +314.861 342.701 314.915 343.318 QUADTO +314.97 343.936 315.158 344.42 QUADTO +315.33 344.873 315.642 345.115 QUADTO +315.955 345.357 316.47 345.357 QUADTO +316.955 345.357 317.267 345.123 QUADTO +317.58 344.889 317.751 344.436 QUADTO +closepath +325.802 341.779 M +325.802 344.123 325.123 345.225 QUADTO +324.443 346.326 323.021 346.326 QUADTO +321.568 346.326 320.896 345.201 QUADTO +320.224 344.076 320.224 341.779 QUADTO +320.224 339.42 320.912 338.326 QUADTO +321.599 337.232 323.021 337.232 QUADTO +324.458 337.232 325.13 338.365 QUADTO +325.802 339.498 325.802 341.779 QUADTO +closepath +324.302 344.436 M +324.474 343.998 324.544 343.334 QUADTO +324.615 342.67 324.615 341.779 QUADTO +324.615 340.873 324.544 340.209 QUADTO +324.474 339.545 324.302 339.107 QUADTO +324.13 338.67 323.818 338.436 QUADTO +323.505 338.201 323.021 338.201 QUADTO +322.537 338.201 322.216 338.436 QUADTO +321.896 338.67 321.724 339.123 QUADTO +321.552 339.576 321.482 340.256 QUADTO +321.412 340.936 321.412 341.779 QUADTO +321.412 342.701 321.466 343.318 QUADTO +321.521 343.936 321.708 344.42 QUADTO +321.88 344.873 322.193 345.115 QUADTO +322.505 345.357 323.021 345.357 QUADTO +323.505 345.357 323.818 345.123 QUADTO +324.13 344.889 324.302 344.436 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +316.464 332.133 M +316.464 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +341.758 343.357 M +341.758 343.998 341.555 344.537 QUADTO +341.352 345.076 340.993 345.467 QUADTO +340.618 345.857 340.071 346.092 QUADTO +339.524 346.326 338.852 346.326 QUADTO +338.227 346.326 337.618 346.186 QUADTO +337.008 346.045 336.571 345.826 QUADTO +336.571 344.592 lineto +336.649 344.592 lineto +336.774 344.686 337.032 344.811 QUADTO +337.29 344.936 337.586 345.045 QUADTO +337.93 345.17 338.219 345.24 QUADTO +338.508 345.311 338.868 345.311 QUADTO +339.211 345.311 339.547 345.186 QUADTO +339.883 345.061 340.133 344.779 QUADTO +340.336 344.514 340.454 344.17 QUADTO +340.571 343.826 340.571 343.357 QUADTO +340.571 342.889 340.446 342.592 QUADTO +340.321 342.295 340.086 342.107 QUADTO +339.821 341.873 339.461 341.787 QUADTO +339.102 341.701 338.649 341.701 QUADTO +338.165 341.701 337.688 341.771 QUADTO +337.211 341.842 336.915 341.904 QUADTO +336.915 337.42 lineto +341.711 337.42 lineto +341.711 338.436 lineto +338.055 338.436 lineto +338.055 340.764 lineto +338.258 340.732 338.469 340.725 QUADTO +338.68 340.717 338.836 340.717 QUADTO +339.508 340.717 339.969 340.826 QUADTO +340.43 340.936 340.868 341.264 QUADTO +341.305 341.592 341.532 342.084 QUADTO +341.758 342.576 341.758 343.357 QUADTO +closepath +348.309 343.357 M +348.309 343.998 348.106 344.537 QUADTO +347.903 345.076 347.543 345.467 QUADTO +347.168 345.857 346.622 346.092 QUADTO +346.075 346.326 345.403 346.326 QUADTO +344.778 346.326 344.168 346.186 QUADTO +343.559 346.045 343.122 345.826 QUADTO +343.122 344.592 lineto +343.2 344.592 lineto +343.325 344.686 343.582 344.811 QUADTO +343.84 344.936 344.137 345.045 QUADTO +344.481 345.17 344.77 345.24 QUADTO +345.059 345.311 345.418 345.311 QUADTO +345.762 345.311 346.098 345.186 QUADTO +346.434 345.061 346.684 344.779 QUADTO +346.887 344.514 347.004 344.17 QUADTO +347.122 343.826 347.122 343.357 QUADTO +347.122 342.889 346.997 342.592 QUADTO +346.872 342.295 346.637 342.107 QUADTO +346.372 341.873 346.012 341.787 QUADTO +345.653 341.701 345.2 341.701 QUADTO +344.715 341.701 344.239 341.771 QUADTO +343.762 341.842 343.465 341.904 QUADTO +343.465 337.42 lineto +348.262 337.42 lineto +348.262 338.436 lineto +344.606 338.436 lineto +344.606 340.764 lineto +344.809 340.732 345.02 340.725 QUADTO +345.231 340.717 345.387 340.717 QUADTO +346.059 340.717 346.52 340.826 QUADTO +346.981 340.936 347.418 341.264 QUADTO +347.856 341.592 348.082 342.084 QUADTO +348.309 342.576 348.309 343.357 QUADTO +closepath +355 341.779 M +355 344.123 354.321 345.225 QUADTO +353.641 346.326 352.219 346.326 QUADTO +350.766 346.326 350.094 345.201 QUADTO +349.422 344.076 349.422 341.779 QUADTO +349.422 339.42 350.11 338.326 QUADTO +350.797 337.232 352.219 337.232 QUADTO +353.657 337.232 354.329 338.365 QUADTO +355 339.498 355 341.779 QUADTO +closepath +353.5 344.436 M +353.672 343.998 353.743 343.334 QUADTO +353.813 342.67 353.813 341.779 QUADTO +353.813 340.873 353.743 340.209 QUADTO +353.672 339.545 353.5 339.107 QUADTO +353.329 338.67 353.016 338.436 QUADTO +352.704 338.201 352.219 338.201 QUADTO +351.735 338.201 351.415 338.436 QUADTO +351.094 338.67 350.922 339.123 QUADTO +350.75 339.576 350.68 340.256 QUADTO +350.61 340.936 350.61 341.779 QUADTO +350.61 342.701 350.665 343.318 QUADTO +350.719 343.936 350.907 344.42 QUADTO +351.079 344.873 351.391 345.115 QUADTO +351.704 345.357 352.219 345.357 QUADTO +352.704 345.357 353.016 345.123 QUADTO +353.329 344.889 353.5 344.436 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +345.663 332.133 M +345.663 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +371.191 343.311 M +371.191 343.982 370.972 344.553 QUADTO +370.753 345.123 370.378 345.498 QUADTO +369.988 345.904 369.496 346.115 QUADTO +369.003 346.326 368.425 346.326 QUADTO +367.832 346.326 367.347 346.123 QUADTO +366.863 345.92 366.488 345.529 QUADTO +366.05 345.061 365.792 344.264 QUADTO +365.535 343.467 365.535 342.311 QUADTO +365.535 341.17 365.769 340.256 QUADTO +366.003 339.342 366.488 338.654 QUADTO +366.957 337.998 367.707 337.623 QUADTO +368.457 337.248 369.441 337.248 QUADTO +369.738 337.248 369.988 337.279 QUADTO +370.238 337.311 370.441 337.373 QUADTO +370.441 338.498 lineto +370.378 338.498 lineto +370.222 338.404 369.925 338.334 QUADTO +369.628 338.264 369.285 338.264 QUADTO +368.207 338.264 367.535 338.998 QUADTO +366.863 339.732 366.738 341.17 QUADTO +367.144 340.889 367.589 340.725 QUADTO +368.035 340.561 368.55 340.561 QUADTO +369.066 340.561 369.449 340.662 QUADTO +369.832 340.764 370.238 341.076 QUADTO +370.722 341.451 370.957 342.006 QUADTO +371.191 342.561 371.191 343.311 QUADTO +closepath +370.003 343.357 M +370.003 342.811 369.863 342.451 QUADTO +369.722 342.092 369.41 341.826 QUADTO +369.175 341.639 368.886 341.584 QUADTO +368.597 341.529 368.3 341.529 QUADTO +367.878 341.529 367.496 341.654 QUADTO +367.113 341.779 366.738 342.014 QUADTO +366.722 342.139 366.714 342.256 QUADTO +366.707 342.373 366.707 342.561 QUADTO +366.707 343.467 366.886 344.021 QUADTO +367.066 344.576 367.332 344.873 QUADTO +367.582 345.139 367.847 345.248 QUADTO +368.113 345.357 368.425 345.357 QUADTO +369.128 345.357 369.566 344.857 QUADTO +370.003 344.357 370.003 343.357 QUADTO +closepath +377.648 341.779 M +377.648 344.123 376.968 345.225 QUADTO +376.289 346.326 374.867 346.326 QUADTO +373.414 346.326 372.742 345.201 QUADTO +372.07 344.076 372.07 341.779 QUADTO +372.07 339.42 372.757 338.326 QUADTO +373.445 337.232 374.867 337.232 QUADTO +376.304 337.232 376.976 338.365 QUADTO +377.648 339.498 377.648 341.779 QUADTO +closepath +376.148 344.436 M +376.32 343.998 376.39 343.334 QUADTO +376.46 342.67 376.46 341.779 QUADTO +376.46 340.873 376.39 340.209 QUADTO +376.32 339.545 376.148 339.107 QUADTO +375.976 338.67 375.664 338.436 QUADTO +375.351 338.201 374.867 338.201 QUADTO +374.382 338.201 374.062 338.436 QUADTO +373.742 338.67 373.57 339.123 QUADTO +373.398 339.576 373.328 340.256 QUADTO +373.257 340.936 373.257 341.779 QUADTO +373.257 342.701 373.312 343.318 QUADTO +373.367 343.936 373.554 344.42 QUADTO +373.726 344.873 374.039 345.115 QUADTO +374.351 345.357 374.867 345.357 QUADTO +375.351 345.357 375.664 345.123 QUADTO +375.976 344.889 376.148 344.436 QUADTO +closepath +384.199 341.779 M +384.199 344.123 383.519 345.225 QUADTO +382.839 346.326 381.417 346.326 QUADTO +379.964 346.326 379.292 345.201 QUADTO +378.621 344.076 378.621 341.779 QUADTO +378.621 339.42 379.308 338.326 QUADTO +379.996 337.232 381.417 337.232 QUADTO +382.855 337.232 383.527 338.365 QUADTO +384.199 339.498 384.199 341.779 QUADTO +closepath +382.699 344.436 M +382.871 343.998 382.941 343.334 QUADTO +383.011 342.67 383.011 341.779 QUADTO +383.011 340.873 382.941 340.209 QUADTO +382.871 339.545 382.699 339.107 QUADTO +382.527 338.67 382.214 338.436 QUADTO +381.902 338.201 381.417 338.201 QUADTO +380.933 338.201 380.613 338.436 QUADTO +380.292 338.67 380.121 339.123 QUADTO +379.949 339.576 379.878 340.256 QUADTO +379.808 340.936 379.808 341.779 QUADTO +379.808 342.701 379.863 343.318 QUADTO +379.917 343.936 380.105 344.42 QUADTO +380.277 344.873 380.589 345.115 QUADTO +380.902 345.357 381.417 345.357 QUADTO +381.902 345.357 382.214 345.123 QUADTO +382.527 344.889 382.699 344.436 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +374.861 332.133 M +374.861 330.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +236.031 365.624 M +233.484 365.624 lineto +233.484 358.874 lineto +231.64 363.233 lineto +229.89 363.233 lineto +228.031 358.874 lineto +228.031 365.624 lineto +225.625 365.624 lineto +225.625 355.452 lineto +228.578 355.452 lineto +230.828 360.468 lineto +233.078 355.452 lineto +236.031 355.452 lineto +236.031 365.624 lineto +closepath +240.096 365.624 M +240.096 357.39 lineto +237.081 357.39 lineto +237.081 355.452 lineto +245.659 355.452 lineto +245.659 357.39 lineto +242.643 357.39 lineto +242.643 365.624 lineto +240.096 365.624 lineto +closepath +248.668 365.624 M +248.668 357.39 lineto +245.653 357.39 lineto +245.653 355.452 lineto +254.231 355.452 lineto +254.231 357.39 lineto +251.215 357.39 lineto +251.215 365.624 lineto +248.668 365.624 lineto +closepath +262.1 357.39 M +257.819 357.39 lineto +257.819 359.312 lineto +261.756 359.312 lineto +261.756 361.249 lineto +257.819 361.249 lineto +257.819 365.624 lineto +255.272 365.624 lineto +255.272 355.452 lineto +262.1 355.452 lineto +262.1 357.39 lineto +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +94.938 34.141 M +94.938 326.133 lineto +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.251 setgray +newpath +38.961 180.541 M +38.961 182.885 38.281 183.986 QUADTO +37.602 185.088 36.18 185.088 QUADTO +34.727 185.088 34.055 183.963 QUADTO +33.383 182.838 33.383 180.541 QUADTO +33.383 178.182 34.07 177.088 QUADTO +34.758 175.994 36.18 175.994 QUADTO +37.617 175.994 38.289 177.127 QUADTO +38.961 178.26 38.961 180.541 QUADTO +closepath +37.461 183.197 M +37.633 182.76 37.703 182.096 QUADTO +37.773 181.432 37.773 180.541 QUADTO +37.773 179.635 37.703 178.971 QUADTO +37.633 178.307 37.461 177.869 QUADTO +37.289 177.432 36.977 177.197 QUADTO +36.664 176.963 36.18 176.963 QUADTO +35.695 176.963 35.375 177.197 QUADTO +35.055 177.432 34.883 177.885 QUADTO +34.711 178.338 34.641 179.018 QUADTO +34.57 179.697 34.57 180.541 QUADTO +34.57 181.463 34.625 182.08 QUADTO +34.68 182.697 34.867 183.182 QUADTO +35.039 183.635 35.352 183.877 QUADTO +35.664 184.119 36.18 184.119 QUADTO +36.664 184.119 36.977 183.885 QUADTO +37.289 183.65 37.461 183.197 QUADTO +closepath +41.965 184.9 M +40.559 184.9 lineto +40.559 183.229 lineto +41.965 183.229 lineto +41.965 184.9 lineto +closepath +49.145 180.541 M +49.145 182.885 48.465 183.986 QUADTO +47.785 185.088 46.363 185.088 QUADTO +44.91 185.088 44.238 183.963 QUADTO +43.566 182.838 43.566 180.541 QUADTO +43.566 178.182 44.254 177.088 QUADTO +44.941 175.994 46.363 175.994 QUADTO +47.801 175.994 48.473 177.127 QUADTO +49.145 178.26 49.145 180.541 QUADTO +closepath +47.645 183.197 M +47.816 182.76 47.887 182.096 QUADTO +47.957 181.432 47.957 180.541 QUADTO +47.957 179.635 47.887 178.971 QUADTO +47.816 178.307 47.645 177.869 QUADTO +47.473 177.432 47.16 177.197 QUADTO +46.848 176.963 46.363 176.963 QUADTO +45.879 176.963 45.559 177.197 QUADTO +45.238 177.432 45.066 177.885 QUADTO +44.895 178.338 44.824 179.018 QUADTO +44.754 179.697 44.754 180.541 QUADTO +44.754 181.463 44.809 182.08 QUADTO +44.863 182.697 45.051 183.182 QUADTO +45.223 183.635 45.535 183.877 QUADTO +45.848 184.119 46.363 184.119 QUADTO +46.848 184.119 47.16 183.885 QUADTO +47.473 183.65 47.645 183.197 QUADTO +closepath +55.695 180.541 M +55.695 182.885 55.016 183.986 QUADTO +54.336 185.088 52.914 185.088 QUADTO +51.461 185.088 50.789 183.963 QUADTO +50.117 182.838 50.117 180.541 QUADTO +50.117 178.182 50.805 177.088 QUADTO +51.492 175.994 52.914 175.994 QUADTO +54.352 175.994 55.023 177.127 QUADTO +55.695 178.26 55.695 180.541 QUADTO +closepath +54.195 183.197 M +54.367 182.76 54.438 182.096 QUADTO +54.508 181.432 54.508 180.541 QUADTO +54.508 179.635 54.438 178.971 QUADTO +54.367 178.307 54.195 177.869 QUADTO +54.023 177.432 53.711 177.197 QUADTO +53.398 176.963 52.914 176.963 QUADTO +52.43 176.963 52.109 177.197 QUADTO +51.789 177.432 51.617 177.885 QUADTO +51.445 178.338 51.375 179.018 QUADTO +51.305 179.697 51.305 180.541 QUADTO +51.305 181.463 51.359 182.08 QUADTO +51.414 182.697 51.602 183.182 QUADTO +51.773 183.635 52.086 183.877 QUADTO +52.398 184.119 52.914 184.119 QUADTO +53.398 184.119 53.711 183.885 QUADTO +54.023 183.65 54.195 183.197 QUADTO +closepath +62.246 180.541 M +62.246 182.885 61.566 183.986 QUADTO +60.887 185.088 59.465 185.088 QUADTO +58.012 185.088 57.34 183.963 QUADTO +56.668 182.838 56.668 180.541 QUADTO +56.668 178.182 57.355 177.088 QUADTO +58.043 175.994 59.465 175.994 QUADTO +60.902 175.994 61.574 177.127 QUADTO +62.246 178.26 62.246 180.541 QUADTO +closepath +60.746 183.197 M +60.918 182.76 60.988 182.096 QUADTO +61.059 181.432 61.059 180.541 QUADTO +61.059 179.635 60.988 178.971 QUADTO +60.918 178.307 60.746 177.869 QUADTO +60.574 177.432 60.262 177.197 QUADTO +59.949 176.963 59.465 176.963 QUADTO +58.98 176.963 58.66 177.197 QUADTO +58.34 177.432 58.168 177.885 QUADTO +57.996 178.338 57.926 179.018 QUADTO +57.855 179.697 57.855 180.541 QUADTO +57.855 181.463 57.91 182.08 QUADTO +57.965 182.697 58.152 183.182 QUADTO +58.324 183.635 58.637 183.877 QUADTO +58.949 184.119 59.465 184.119 QUADTO +59.949 184.119 60.262 183.885 QUADTO +60.574 183.65 60.746 183.197 QUADTO +closepath +68.797 180.541 M +68.797 182.885 68.117 183.986 QUADTO +67.438 185.088 66.016 185.088 QUADTO +64.562 185.088 63.891 183.963 QUADTO +63.219 182.838 63.219 180.541 QUADTO +63.219 178.182 63.906 177.088 QUADTO +64.594 175.994 66.016 175.994 QUADTO +67.453 175.994 68.125 177.127 QUADTO +68.797 178.26 68.797 180.541 QUADTO +closepath +67.297 183.197 M +67.469 182.76 67.539 182.096 QUADTO +67.609 181.432 67.609 180.541 QUADTO +67.609 179.635 67.539 178.971 QUADTO +67.469 178.307 67.297 177.869 QUADTO +67.125 177.432 66.812 177.197 QUADTO +66.5 176.963 66.016 176.963 QUADTO +65.531 176.963 65.211 177.197 QUADTO +64.891 177.432 64.719 177.885 QUADTO +64.547 178.338 64.477 179.018 QUADTO +64.406 179.697 64.406 180.541 QUADTO +64.406 181.463 64.461 182.08 QUADTO +64.516 182.697 64.703 183.182 QUADTO +64.875 183.635 65.188 183.877 QUADTO +65.5 184.119 66.016 184.119 QUADTO +66.5 184.119 66.812 183.885 QUADTO +67.125 183.65 67.297 183.197 QUADTO +closepath +75.348 180.541 M +75.348 182.885 74.668 183.986 QUADTO +73.988 185.088 72.566 185.088 QUADTO +71.113 185.088 70.441 183.963 QUADTO +69.77 182.838 69.77 180.541 QUADTO +69.77 178.182 70.457 177.088 QUADTO +71.145 175.994 72.566 175.994 QUADTO +74.004 175.994 74.676 177.127 QUADTO +75.348 178.26 75.348 180.541 QUADTO +closepath +73.848 183.197 M +74.02 182.76 74.09 182.096 QUADTO +74.16 181.432 74.16 180.541 QUADTO +74.16 179.635 74.09 178.971 QUADTO +74.02 178.307 73.848 177.869 QUADTO +73.676 177.432 73.363 177.197 QUADTO +73.051 176.963 72.566 176.963 QUADTO +72.082 176.963 71.762 177.197 QUADTO +71.441 177.432 71.27 177.885 QUADTO +71.098 178.338 71.027 179.018 QUADTO +70.957 179.697 70.957 180.541 QUADTO +70.957 181.463 71.012 182.08 QUADTO +71.066 182.697 71.254 183.182 QUADTO +71.426 183.635 71.738 183.877 QUADTO +72.051 184.119 72.566 184.119 QUADTO +73.051 184.119 73.363 183.885 QUADTO +73.676 183.65 73.848 183.197 QUADTO +closepath +81.898 180.541 M +81.898 182.885 81.219 183.986 QUADTO +80.539 185.088 79.117 185.088 QUADTO +77.664 185.088 76.992 183.963 QUADTO +76.32 182.838 76.32 180.541 QUADTO +76.32 178.182 77.008 177.088 QUADTO +77.695 175.994 79.117 175.994 QUADTO +80.555 175.994 81.227 177.127 QUADTO +81.898 178.26 81.898 180.541 QUADTO +closepath +80.398 183.197 M +80.57 182.76 80.641 182.096 QUADTO +80.711 181.432 80.711 180.541 QUADTO +80.711 179.635 80.641 178.971 QUADTO +80.57 178.307 80.398 177.869 QUADTO +80.227 177.432 79.914 177.197 QUADTO +79.602 176.963 79.117 176.963 QUADTO +78.633 176.963 78.312 177.197 QUADTO +77.992 177.432 77.82 177.885 QUADTO +77.648 178.338 77.578 179.018 QUADTO +77.508 179.697 77.508 180.541 QUADTO +77.508 181.463 77.562 182.08 QUADTO +77.617 182.697 77.805 183.182 QUADTO +77.977 183.635 78.289 183.877 QUADTO +78.602 184.119 79.117 184.119 QUADTO +79.602 184.119 79.914 183.885 QUADTO +80.227 183.65 80.398 183.197 QUADTO +closepath +88.449 180.541 M +88.449 182.885 87.77 183.986 QUADTO +87.09 185.088 85.668 185.088 QUADTO +84.215 185.088 83.543 183.963 QUADTO +82.871 182.838 82.871 180.541 QUADTO +82.871 178.182 83.559 177.088 QUADTO +84.246 175.994 85.668 175.994 QUADTO +87.105 175.994 87.777 177.127 QUADTO +88.449 178.26 88.449 180.541 QUADTO +closepath +86.949 183.197 M +87.121 182.76 87.191 182.096 QUADTO +87.262 181.432 87.262 180.541 QUADTO +87.262 179.635 87.191 178.971 QUADTO +87.121 178.307 86.949 177.869 QUADTO +86.777 177.432 86.465 177.197 QUADTO +86.152 176.963 85.668 176.963 QUADTO +85.184 176.963 84.863 177.197 QUADTO +84.543 177.432 84.371 177.885 QUADTO +84.199 178.338 84.129 179.018 QUADTO +84.059 179.697 84.059 180.541 QUADTO +84.059 181.463 84.113 182.08 QUADTO +84.168 182.697 84.355 183.182 QUADTO +84.527 183.635 84.84 183.877 QUADTO +85.152 184.119 85.668 184.119 QUADTO +86.152 184.119 86.465 183.885 QUADTO +86.777 183.65 86.949 183.197 QUADTO +closepath +fill +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +0.502 setgray +2 setlinecap +10.0 setmiterlimit +newpath +92.938 180.137 M +94.938 180.137 lineto +stroke +grestore +gsave +[0 -1 1 0 -160.6875 199.58594] concat +newpath +199.586 160.688 M +199.586 560.688 lineto +-200.414 560.688 lineto +-200.414 160.688 lineto +199.586 160.688 lineto +closepath +clip +0.251 setgray +newpath +2.089 181.866 M +2.089 183.757 1.058 184.843 QUADTO +0.026 185.929 -1.864 185.929 QUADTO +-3.755 185.929 -4.794 184.843 QUADTO +-5.833 183.757 -5.833 181.866 QUADTO +-5.833 179.944 -4.794 178.858 QUADTO +-3.755 177.772 -1.864 177.772 QUADTO +0.011 177.772 1.05 178.866 QUADTO +2.089 179.96 2.089 181.866 QUADTO +closepath +-0.364 181.866 M +-0.364 181.194 -0.474 180.733 QUADTO +-0.583 180.272 -0.786 180.007 QUADTO +-0.989 179.726 -1.263 179.608 QUADTO +-1.536 179.491 -1.864 179.491 QUADTO +-2.192 179.491 -2.45 179.593 QUADTO +-2.708 179.694 -2.927 179.976 QUADTO +-3.13 180.241 -3.255 180.702 QUADTO +-3.38 181.163 -3.38 181.866 QUADTO +-3.38 182.569 -3.263 183.007 QUADTO +-3.146 183.444 -2.958 183.71 QUADTO +-2.771 183.976 -2.481 184.093 QUADTO +-2.192 184.21 -1.849 184.21 QUADTO +-1.567 184.21 -1.271 184.093 QUADTO +-0.974 183.976 -0.786 183.726 QUADTO +-0.583 183.429 -0.474 183.015 QUADTO +-0.364 182.601 -0.364 181.866 QUADTO +closepath +8.448 181.804 M +8.448 180.647 8.097 180.155 QUADTO +7.745 179.663 7.026 179.663 QUADTO +6.698 179.663 6.378 179.765 QUADTO +6.058 179.866 5.761 180.069 QUADTO +5.761 183.929 lineto +5.995 184.022 6.237 184.062 QUADTO +6.479 184.101 6.714 184.101 QUADTO +7.589 184.101 8.019 183.522 QUADTO +8.448 182.944 8.448 181.804 QUADTO +closepath +10.886 181.757 M +10.886 183.585 9.956 184.718 QUADTO +9.026 185.851 7.651 185.851 QUADTO +7.089 185.851 6.659 185.718 QUADTO +6.229 185.585 5.761 185.304 QUADTO +5.651 185.694 lineto +3.37 185.694 lineto +3.37 175.054 lineto +5.761 175.054 lineto +5.761 178.819 lineto +6.276 178.366 6.823 178.085 QUADTO +7.37 177.804 8.089 177.804 QUADTO +9.433 177.804 10.159 178.851 QUADTO +10.886 179.897 10.886 181.757 QUADTO +closepath +15.45 185.835 M +15.45 187.132 14.685 187.858 QUADTO +13.919 188.585 12.685 188.585 QUADTO +12.028 188.585 11.497 188.53 QUADTO +10.966 188.476 10.716 188.413 QUADTO +10.716 186.647 lineto +10.903 186.647 lineto +11.138 186.726 11.419 186.765 QUADTO +11.7 186.804 11.872 186.804 QUADTO +12.56 186.804 12.81 186.382 QUADTO +13.06 185.96 13.06 184.991 QUADTO +13.06 179.679 lineto +11.685 179.679 lineto +11.685 178.022 lineto +15.45 178.022 lineto +15.45 185.835 lineto +closepath +15.45 176.929 M +12.903 176.929 lineto +12.903 175.054 lineto +15.45 175.054 lineto +15.45 176.929 lineto +closepath +21.217 185.897 M +19.045 185.897 17.889 184.858 QUADTO +16.732 183.819 16.732 181.897 QUADTO +16.732 180.022 17.811 178.897 QUADTO +18.889 177.772 20.826 177.772 QUADTO +22.592 177.772 23.482 178.726 QUADTO +24.373 179.679 24.373 181.444 QUADTO +24.373 182.304 lineto +19.123 182.304 lineto +19.17 182.851 19.357 183.202 QUADTO +19.545 183.554 19.857 183.772 QUADTO +20.17 183.991 20.576 184.077 QUADTO +20.982 184.163 21.467 184.163 QUADTO +21.889 184.163 22.295 184.077 QUADTO +22.701 183.991 23.029 183.835 QUADTO +23.326 183.71 23.576 183.554 QUADTO +23.826 183.397 23.982 183.288 QUADTO +24.232 183.288 lineto +24.232 185.257 lineto +23.936 185.382 23.693 185.483 QUADTO +23.451 185.585 23.029 185.679 QUADTO +22.639 185.788 22.225 185.843 QUADTO +21.811 185.897 21.217 185.897 QUADTO +closepath +22.014 180.866 M +21.998 180.116 21.662 179.718 QUADTO +21.326 179.319 20.639 179.319 QUADTO +19.951 179.319 19.561 179.733 QUADTO +19.17 180.147 19.123 180.866 QUADTO +22.014 180.866 lineto +closepath +29.373 185.897 M +28.42 185.897 27.631 185.655 QUADTO +26.842 185.413 26.264 184.929 QUADTO +25.686 184.429 25.365 183.671 QUADTO +25.045 182.913 25.045 181.882 QUADTO +25.045 180.788 25.389 180.015 QUADTO +25.732 179.241 26.326 178.741 QUADTO +26.904 178.257 27.686 178.03 QUADTO +28.467 177.804 29.295 177.804 QUADTO +29.936 177.804 30.568 177.952 QUADTO +31.201 178.101 31.857 178.413 QUADTO +31.857 180.476 lineto +31.545 180.476 lineto +31.404 180.335 31.193 180.171 QUADTO +30.982 180.007 30.732 179.866 QUADTO +30.482 179.726 30.154 179.624 QUADTO +29.826 179.522 29.42 179.522 QUADTO +28.498 179.522 27.998 180.163 QUADTO +27.498 180.804 27.498 181.882 QUADTO +27.498 183.038 28.037 183.608 QUADTO +28.576 184.179 29.451 184.179 QUADTO +29.904 184.179 30.248 184.077 QUADTO +30.592 183.976 30.842 183.835 QUADTO +31.076 183.679 31.248 183.53 QUADTO +31.42 183.382 31.545 183.257 QUADTO +31.857 183.257 lineto +31.857 185.304 lineto +31.639 185.397 31.373 185.515 QUADTO +31.107 185.632 30.826 185.694 QUADTO +30.482 185.804 30.178 185.851 QUADTO +29.873 185.897 29.373 185.897 QUADTO +closepath +35.928 185.851 M +34.459 185.851 33.795 185.249 QUADTO +33.131 184.647 33.131 183.272 QUADTO +33.131 179.679 lineto +32.209 179.679 lineto +32.209 178.022 lineto +33.131 178.022 lineto +33.131 175.819 lineto +35.521 175.819 lineto +35.521 178.022 lineto +37.818 178.022 lineto +37.818 179.679 lineto +35.521 179.679 lineto +35.521 182.413 lineto +35.521 182.804 35.529 183.108 QUADTO +35.537 183.413 35.631 183.647 QUADTO +35.74 183.882 35.982 184.022 QUADTO +36.225 184.163 36.678 184.163 QUADTO +36.881 184.163 37.186 184.085 QUADTO +37.49 184.007 37.615 183.944 QUADTO +37.818 183.944 lineto +37.818 185.632 lineto +37.428 185.726 36.99 185.788 QUADTO +36.553 185.851 35.928 185.851 QUADTO +closepath +41.136 185.913 M +40.292 185.913 39.534 185.726 QUADTO +38.776 185.538 38.292 185.288 QUADTO +38.292 183.272 lineto +38.495 183.272 lineto +38.651 183.397 38.87 183.554 QUADTO +39.089 183.71 39.479 183.882 QUADTO +39.823 184.038 40.245 184.155 QUADTO +40.667 184.272 41.151 184.272 QUADTO +41.667 184.272 42.05 184.108 QUADTO +42.433 183.944 42.433 183.616 QUADTO +42.433 183.335 42.253 183.202 QUADTO +42.073 183.069 41.558 182.944 QUADTO +41.292 182.882 40.878 182.804 QUADTO +40.464 182.726 40.136 182.616 QUADTO +39.229 182.335 38.776 181.788 QUADTO +38.323 181.241 38.323 180.335 QUADTO +38.323 179.835 38.558 179.374 QUADTO +38.792 178.913 39.261 178.569 QUADTO +39.714 178.21 40.386 178.007 QUADTO +41.058 177.804 41.901 177.804 QUADTO +42.714 177.804 43.394 177.968 QUADTO +44.073 178.132 44.526 178.351 QUADTO +44.526 180.288 lineto +44.339 180.288 lineto +44.214 180.194 43.948 180.038 QUADTO +43.683 179.882 43.433 179.757 QUADTO +43.136 179.616 42.761 179.53 QUADTO +42.386 179.444 41.995 179.444 QUADTO +41.479 179.444 41.12 179.624 QUADTO +40.761 179.804 40.761 180.101 QUADTO +40.761 180.351 40.94 180.507 QUADTO +41.12 180.663 41.714 180.804 QUADTO +42.011 180.882 42.409 180.952 QUADTO +42.808 181.022 43.183 181.147 QUADTO +44.011 181.413 44.44 181.929 QUADTO +44.87 182.444 44.87 183.288 QUADTO +44.87 183.835 44.612 184.327 QUADTO +44.354 184.819 43.901 185.147 QUADTO +43.401 185.507 42.737 185.71 QUADTO +42.073 185.913 41.136 185.913 QUADTO +closepath +fill +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +112.077 34.141 M +112.077 326.133 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +141.275 34.141 M +141.275 326.133 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +170.473 34.141 M +170.473 326.133 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +199.671 34.141 M +199.671 326.133 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +228.87 34.141 M +228.87 326.133 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +258.068 34.141 M +258.068 326.133 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +287.266 34.141 M +287.266 326.133 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +316.464 34.141 M +316.464 326.133 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +345.663 34.141 M +345.663 326.133 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +374.861 34.141 M +374.861 326.133 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 setgray +[2 2] 0 setdash +2 setlinejoin +2 setlinejoin +0.5 setlinewidth +newpath +98.938 180.137 M +388 180.137 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +2 setlinecap +2 setlinejoin +2 setlinejoin +newpath +112.077 180.137 M +199.671 180.137 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +2 setlinecap +2 setlinejoin +2 setlinejoin +newpath +199.671 180.137 M +374.861 180.137 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +2 setlinecap +2 setlinejoin +2 setlinejoin +newpath +112.077 180.137 M +199.671 180.137 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +2 setlinecap +2 setlinejoin +2 setlinejoin +newpath +199.671 180.137 M +374.861 180.137 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +2 setlinecap +2 setlinejoin +2 setlinejoin +newpath +112.077 180.137 M +199.671 180.137 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +2 setlinecap +2 setlinejoin +2 setlinejoin +newpath +199.671 180.137 M +374.861 180.137 lineto +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +newpath +112.077 177.137 M +115.077 183.137 lineto +109.077 183.137 lineto +closepath +eofill +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +112.077 177.137 M +115.077 183.137 lineto +109.077 183.137 lineto +closepath +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +newpath +199.671 177.137 M +202.671 183.137 lineto +196.671 183.137 lineto +closepath +eofill +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +199.671 177.137 M +202.671 183.137 lineto +196.671 183.137 lineto +closepath +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +newpath +374.861 177.137 M +377.861 183.137 lineto +371.861 183.137 lineto +closepath +eofill +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 1 0.333 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +374.861 177.137 M +377.861 183.137 lineto +371.861 183.137 lineto +closepath +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +newpath +115.077 180.137 M +115.077 181.794 113.734 183.137 112.077 183.137 curveto +110.42 183.137 109.077 181.794 109.077 180.137 curveto +109.077 178.48 110.42 177.137 112.077 177.137 curveto +113.734 177.137 115.077 178.48 115.077 180.137 curveto +closepath +fill +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +115.077 180.137 M +115.077 181.794 113.734 183.137 112.077 183.137 curveto +110.42 183.137 109.077 181.794 109.077 180.137 curveto +109.077 178.48 110.42 177.137 112.077 177.137 curveto +113.734 177.137 115.077 178.48 115.077 180.137 curveto +closepath +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +newpath +202.671 180.137 M +202.671 181.794 201.328 183.137 199.671 183.137 curveto +198.015 183.137 196.671 181.794 196.671 180.137 curveto +196.671 178.48 198.015 177.137 199.671 177.137 curveto +201.328 177.137 202.671 178.48 202.671 180.137 curveto +closepath +fill +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +202.671 180.137 M +202.671 181.794 201.328 183.137 199.671 183.137 curveto +198.015 183.137 196.671 181.794 196.671 180.137 curveto +196.671 178.48 198.015 177.137 199.671 177.137 curveto +201.328 177.137 202.671 178.48 202.671 180.137 curveto +closepath +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +newpath +377.861 180.137 M +377.861 181.794 376.518 183.137 374.861 183.137 curveto +373.204 183.137 371.861 181.794 371.861 180.137 curveto +371.861 178.48 373.204 177.137 374.861 177.137 curveto +376.518 177.137 377.861 178.48 377.861 180.137 curveto +closepath +fill +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +0.333 0.333 1 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +377.861 180.137 M +377.861 181.794 376.518 183.137 374.861 183.137 curveto +373.204 183.137 371.861 181.794 371.861 180.137 curveto +371.861 178.48 373.204 177.137 374.861 177.137 curveto +376.518 177.137 377.861 178.48 377.861 180.137 curveto +closepath +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +newpath +109.077 177.137 M +115.077 177.137 lineto +115.077 183.137 lineto +109.077 183.137 lineto +109.077 177.137 lineto +closepath +fill +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +109.077 177.137 M +115.077 177.137 lineto +115.077 183.137 lineto +109.077 183.137 lineto +109.077 177.137 lineto +closepath +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +newpath +196.671 177.137 M +202.671 177.137 lineto +202.671 183.137 lineto +196.671 183.137 lineto +196.671 177.137 lineto +closepath +fill +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +196.671 177.137 M +202.671 177.137 lineto +202.671 183.137 lineto +196.671 183.137 lineto +196.671 177.137 lineto +closepath +stroke +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +newpath +371.861 177.137 M +377.861 177.137 lineto +377.861 183.137 lineto +371.861 183.137 lineto +371.861 177.137 lineto +closepath +fill +grestore +gsave +newpath +98.938 34.141 M +98.938 326.133 lineto +388 326.133 lineto +388 34.141 lineto +closepath +clip +1 0.333 0.333 setrgbcolor +2 setlinecap +10.0 setmiterlimit +newpath +371.861 177.137 M +377.861 177.137 lineto +377.861 183.137 lineto +371.861 183.137 lineto +371.861 177.137 lineto +closepath +stroke +grestore +gsave +newpath +0 0 M +400 0 lineto +400 400 lineto +0 400 lineto +0 0 lineto +closepath +clip +2 setlinecap +10.0 setmiterlimit +0.5 setlinewidth +newpath +98.938 34.141 M +388 34.141 lineto +388 326.133 lineto +98.938 326.133 lineto +98.938 34.141 lineto +closepath +stroke +grestore +%%Trailer +%%Pages: 1 +%%EOF hooks/post-receive -- Grayskull Repository

1 0

Grayskull Repository branch, master, updated. git-migration-255-gc8fbc01
by noreply＠mcs.anl.gov 07 Dec '09

07 Dec '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via c8fbc01d3ba901fb39fbb042e74ca51aad5847b3 (commit) via f64cc36343032188fca815f34815f5a4b466000c (commit) via 6857d49b90bd1b51853251efb1588cb24ed2d672 (commit) from 4ab72050c4ccf3b80a554d7da0b1eefdaae2872f (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit c8fbc01d3ba901fb39fbb042e74ca51aad5847b3 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Mon Dec 7 13:27:55 2009 -0500 fix 64 bit cast warning in cosd prototype commit f64cc36343032188fca815f34815f5a4b466000c Author: Phil Carns <carns(a)mcs.anl.gov> Date: Mon Dec 7 13:27:14 2009 -0500 fix 64bit cast warning in opcache init macro commit 6857d49b90bd1b51853251efb1588cb24ed2d672 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Mon Dec 7 13:23:11 2009 -0500 fix warnings related to printing 64 bit integers ----------------------------------------------------------------------- Summary of changes: code/src/gsl/include/gs-opcache.h | 2 +- .../gsl/resources/cosd-prototype/cosd-prototype.c | 34 ++++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) Diff of changes: diff --git a/code/src/gsl/include/gs-opcache.h b/code/src/gsl/include/gs-opcache.h index 84adece..f5b2ce5 100644 --- a/code/src/gsl/include/gs-opcache.h +++ b/code/src/gsl/include/gs-opcache.h @@ -8,7 +8,7 @@ typedef struct gs_opcache *gs_opcache_t; #define GS_OPCACHE_INIT(_optype, _member, _size, _name) \ - gs_opcache_init(sizeof(_optype), (int)(&((_optype *)0)->_member), _size, (_name)) + gs_opcache_init(sizeof(_optype), (unsigned long)(&((_optype *)0)->_member), _size, (_name)) /* complete an operation by returning it to opcache and invoking callback */ /* note that __error_code is copied on purpose for safety in case it is a diff --git a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c index 4c5431d..8e9b9cd 100644 --- a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c +++ b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c @@ -13,6 +13,7 @@ #include <limits.h> #include <db.h> #include <stdlib.h> +#include "gs-config.h" /**************************************************************/ /* TODO: general stuff: @@ -26,6 +27,21 @@ */ /**************************************************************/ +/* TODO: this needs to be in a header somewhere */ +#ifndef llu +#if SIZEOF_LONG_INT == 4 +# define llu(x) (x) +# define lld(x) (x) +# define SCANF_lld "%lld" +#elif SIZEOF_LONG_INT == 8 +# define llu(x) (unsigned long long)(x) +# define lld(x) (long long)(x) +# define SCANF_lld "%ld" +#else +# error Unexpected sizeof(long int) +#endif +#endif + /* TODO: figure out how to toggle something like this at run time; we don't * need locking if initialized in polling mode */ @@ -645,7 +661,7 @@ static int write_op_worker(struct gs_op* op) { /* open log file */ sprintf(log_name, "%s/%llu.dat", cosd_log_path, - c_op->u.write.oid); + llu(c_op->u.write.oid)); ret = open(log_name, O_RDWR|O_DIRECT|O_EXCL|O_NOATIME, S_IRUSR|S_IWUSR); if(ret < 0) @@ -752,7 +768,7 @@ static int write_op_worker(struct gs_op* op) global_log_offset += aligned_size; gs_mutex_unlock(&global_log_mutex); - if(((int)mem_ptr) % DIRECT_ALIGN || tmp_update->size != aligned_size) + if(((unsigned long)mem_ptr) % DIRECT_ALIGN || tmp_update->size != aligned_size) { /* mem buffer is not aligned */ /* allocate a scratch buffer to copy to */ @@ -867,16 +883,16 @@ static int dump_op_worker(struct gs_op* op) /* loop through version db, one entry per object */ while ((ret = ver_dbcp->c_get(ver_dbcp, &ver_key, &ver_value, DB_NEXT)) == 0) { - printf("OID: %llu\n", vk); + printf("OID: %llu\n", llu(vk)); printf("=============================================\n"); - printf(" version: %llu\n", vv); + printf(" version: %llu\n", llu(vv)); /* next log offset entry should match, no need to search */ ret = log_offset_dbcp->c_get(log_offset_dbcp, &log_offset_key, &log_offset_value, DB_NEXT); assert(ret == 0); /* TODO: err handling */ assert(lok == vk); - printf(" next log offset: %lld\n", lov); + printf(" next log offset: %lld\n", lld(lov)); /* search for any missing versions */ printf(" missing versions: "); @@ -887,7 +903,7 @@ static int dump_op_worker(struct gs_op* op) &missing_ver_key, &missing_ver_value, c_get_flag)) == 0) { c_get_flag = DB_NEXT; - printf("%llu ", mvk.version); + printf("%llu ", llu(mvk.version)); } printf("\n"); assert(ret == DB_NOTFOUND); /* TODO: err handling */ @@ -902,8 +918,8 @@ static int dump_op_worker(struct gs_op* op) &log_map_key, &log_map_value, c_get_flag)) == 0) { printf("logical_off: %lld, logical_end: %lld, size: %lld, log_off: %lld, ver: %llu\n", - lmv.logical_offset, lmv.logical_offset_end, lmv.size, - lmv.log_offset, lmv.version); + lld(lmv.logical_offset), lld(lmv.logical_offset_end), + lld(lmv.size), lld(lmv.log_offset), llu(lmv.version)); c_get_flag = DB_NEXT; } assert(ret == DB_NOTFOUND); /* TODO: err handling */ @@ -951,7 +967,7 @@ static int create_op_worker(struct gs_op* op) /* create a log file */ sprintf(log_name, "%s/%llu.dat", cosd_log_path, - c_op->u.create.requested_oid); + llu(c_op->u.create.requested_oid)); ret = open(log_name, O_RDWR|O_CREAT|O_DIRECT|O_EXCL|O_NOATIME, S_IRUSR|S_IWUSR); if(ret < 0) hooks/post-receive -- Grayskull Repository

1 0