Triton Repository branch, master, updated. 2f58f0721fd0eb023ac697593338f8a82e0b76e3
This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via 2f58f0721fd0eb023ac697593338f8a82e0b76e3 (commit) via d7f7e2e56b4f6ba2367e717923a28892b67f3fc4 (commit) via 0e7377233578416aa5a9db670cc25cf9ccc35468 (commit) via 57e98b8a976275f8379810bdfcec4fa0615f3211 (commit) via 1da14f4624c76aa64b0eb4eae886c0328d36e5b8 (commit) via 16e546a721ab457b75e8ab75148fda28a22da3a7 (commit) from 7c8131a56d34d2e3924aa15f83493f25afc9f022 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 2f58f0721fd0eb023ac697593338f8a82e0b76e3 Author: Justin Wozniak <[email protected]> Date: Wed Mar 10 17:38:37 2010 -0600 EOL fixes. commit d7f7e2e56b4f6ba2367e717923a28892b67f3fc4 Author: Justin Wozniak <[email protected]> Date: Wed Mar 10 17:37:20 2010 -0600 New and modified input configurations for DSN draft. commit 0e7377233578416aa5a9db670cc25cf9ccc35468 Author: Justin Wozniak <[email protected]> Date: Wed Mar 10 17:36:34 2010 -0600 Moved time arithmetic function. commit 57e98b8a976275f8379810bdfcec4fa0615f3211 Author: Justin Wozniak <[email protected]> Date: Wed Mar 10 17:35:38 2010 -0600 Minor modifications to plots and sweeps for DSN draft. commit 1da14f4624c76aa64b0eb4eae886c0328d36e5b8 Author: Justin Wozniak <[email protected]> Date: Wed Mar 10 17:33:57 2010 -0600 Add some comments and profiling output. commit 16e546a721ab457b75e8ab75148fda28a22da3a7 Author: Justin Wozniak <[email protected]> Date: Wed Mar 10 17:32:48 2010 -0600 Fix erroneous rebuild time calculation. ----------------------------------------------------------------------- Summary of changes: sim/gobs/src/gobs/plot/AbstractPlot.java | 30 ++++++- sim/gobs/src/gobs/plot/MultiLogXPlot.java | 12 ++- sim/gobs/src/gobs/plot/MultiplePlot.java | 10 +-- sim/gobs/src/gobs/plot/Plot.java | 2 +- sim/gobs/src/gobs/sim/Rebuild.java | 14 ++- sim/gobs/src/gobs/sim/Simulator.java | 23 +++++- sim/gobs/src/gobs/sweep/AbstractSweep.java | 60 ++++++++++++- sim/gobs/src/gobs/sweep/Helpers.java | 5 +- sim/gobs/src/gobs/sweep/Sweep.java | 11 ++- sim/gobs/src/gobs/sweep/x/DataLoss01.java | 112 +++++++++++------------- sim/gobs/src/gobs/sweep/x/DataLoss02.java | 4 +- sim/gobs/src/gobs/sweep/x/Decluster03.java | 13 +--- sim/gobs/src/gobs/sweep/x/Rebuild01.java | 52 +++++++---- sim/gobs/src/gobs/sweep/x/Rebuild02.java | 126 ++++++++++++-------------- sim/gobs/src/gobs/util/Time.java | 28 ++++++- sim/gobs/src/gobs/util/Tools.java | 14 +--- sim/gobs/sweeps/dc03kpv01.cfg | 6 +- sim/gobs/sweeps/dc03kpv03.cfg | 4 +- sim/gobs/sweeps/dc03kpv05.cfg | 5 +- sim/gobs/sweeps/dc04kpv01.cfg | 5 +- sim/gobs/sweeps/dc04kpv03.cfg | 6 +- sim/gobs/sweeps/dc04kpv05.cfg | 6 +- sim/gobs/sweeps/dc_output.cfg | 11 +-- sim/gobs/sweeps/decluster03.zsh | 2 +- sim/gobs/sweeps/disks-full-zaps-only.cfg | 19 ++++ sim/gobs/sweeps/disks-full.cfg | 19 +--- sim/gobs/sweeps/disks-long.cfg | 18 ++++ sim/gobs/sweeps/disks-medium-zaps-only.cfg | 38 ++++++++ sim/gobs/sweeps/disks-medium.cfg | 15 +--- sim/gobs/sweeps/disks-small.cfg | 9 -- sim/gobs/sweeps/dl01-full.cfg | 33 +++++++ sim/gobs/sweeps/dl01-full.zsh | 9 ++ sim/gobs/sweeps/dl01-medium.cfg | 20 ++--- sim/gobs/sweeps/dl01-small.cfg | 18 ++--- sim/gobs/sweeps/dl01.cfg | 2 +- sim/gobs/sweeps/dl_output.cfg | 8 +- sim/gobs/sweeps/faults_medium.cfg | 10 ++ sim/gobs/sweeps/faults_medium_range_dl01.cfg | 18 ++++ sim/gobs/sweeps/faults_real.cfg | 10 ++ sim/gobs/sweeps/faults_real_range_dl01.cfg | 20 ++++ sim/gobs/sweeps/faults_real_zaps_only.cfg | 16 ++++ sim/gobs/sweeps/faults_small.cfg | 7 ++ sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg | 11 +++ sim/gobs/sweeps/fs_pdsi_medium.cfg | 2 +- sim/gobs/sweeps/helpers.zsh | 18 ++++ sim/gobs/sweeps/kpv05.cfg | 2 - sim/gobs/sweeps/rb01-long.cfg | 41 +++++++++ sim/gobs/sweeps/rb01-long.zsh | 11 +++ sim/gobs/sweeps/rb01-medium.cfg | 9 ++- sim/gobs/sweeps/rb02-full.cfg | 32 +++++++ sim/gobs/sweeps/rb02-full.zsh | 9 ++ sim/gobs/sweeps/rb02-medium.cfg | 31 +++++++ sim/gobs/sweeps/rb02-medium.zsh | 9 ++ sim/gobs/sweeps/rb02-small.cfg | 2 +- sim/gobs/sweeps/rb_output.cfg | 6 +- sim/gobs/sweeps/sweep.zsh | 5 +- sim/gobs/sweeps/system_100,200.cfg | 24 +++++ sim/gobs/sweeps/system_100,600.cfg | 16 ---- sim/gobs/sweeps/system_100-600.cfg | 16 ---- sim/gobs/sweeps/system_100-700.cfg | 23 +++++ sim/gobs/sweeps/system_600.cfg | 6 +- 61 files changed, 801 insertions(+), 322 deletions(-) create mode 100644 sim/gobs/sweeps/disks-full-zaps-only.cfg create mode 100644 sim/gobs/sweeps/disks-long.cfg create mode 100644 sim/gobs/sweeps/disks-medium-zaps-only.cfg create mode 100644 sim/gobs/sweeps/dl01-full.cfg create mode 100755 sim/gobs/sweeps/dl01-full.zsh create mode 100644 sim/gobs/sweeps/faults_medium.cfg create mode 100644 sim/gobs/sweeps/faults_medium_range_dl01.cfg create mode 100644 sim/gobs/sweeps/faults_real.cfg create mode 100644 sim/gobs/sweeps/faults_real_range_dl01.cfg create mode 100644 sim/gobs/sweeps/faults_real_zaps_only.cfg create mode 100644 sim/gobs/sweeps/faults_small.cfg create mode 100644 sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg create mode 100644 sim/gobs/sweeps/helpers.zsh create mode 100644 sim/gobs/sweeps/rb01-long.cfg create mode 100755 sim/gobs/sweeps/rb01-long.zsh create mode 100644 sim/gobs/sweeps/rb02-full.cfg create mode 100755 sim/gobs/sweeps/rb02-full.zsh create mode 100644 sim/gobs/sweeps/rb02-medium.cfg create mode 100755 sim/gobs/sweeps/rb02-medium.zsh create mode 100644 sim/gobs/sweeps/system_100,200.cfg create mode 100644 sim/gobs/sweeps/system_100-700.cfg Diff of changes: diff --git a/sim/gobs/src/gobs/plot/AbstractPlot.java b/sim/gobs/src/gobs/plot/AbstractPlot.java index be62a5f..4906fba 100755 --- a/sim/gobs/src/gobs/plot/AbstractPlot.java +++ b/sim/gobs/src/gobs/plot/AbstractPlot.java @@ -2,13 +2,37 @@ package gobs.plot; import java.io.IOException; import java.io.OutputStream; +import java.util.Properties; import org.apache.xmlgraphics.java2d.ps.EPSDocumentGraphics2D; +import gobs.util.Tools; + public class AbstractPlot { - static int w = 400; - static int h = 400; + static int width = 400; + static int height = 400; + + public static boolean bw = false; + public static boolean forceDomainUpper = false; + public static double forceDomainUpperValue = 0; + + public static void configure(Properties properties) + { + if (Tools.has(properties, "plot.force.domain.upper")) + forceDomainUpper + (Tools.decimal(properties, "plot.force.domain.upper")); + if (Tools.enabled(properties, "plot.blackandwhite")) + bw = true; + if (Tools.has(properties, "plot.width")) + width = Tools.integer(properties, "plot.width"); + } + + static void forceDomainUpper(double v) + { + forceDomainUpper = true; + forceDomainUpperValue = v; + } /** Get a writable EPS file at output. @@ -22,7 +46,7 @@ public class AbstractPlot g2d.setGraphicContext (new org.apache.xmlgraphics.java2d.GraphicContext()); - g2d.setupDocument(output, w, h); + g2d.setupDocument(output, width, height); return g2d; } diff --git a/sim/gobs/src/gobs/plot/MultiLogXPlot.java b/sim/gobs/src/gobs/plot/MultiLogXPlot.java index d275a0a..68dcf4a 100644 --- a/sim/gobs/src/gobs/plot/MultiLogXPlot.java +++ b/sim/gobs/src/gobs/plot/MultiLogXPlot.java @@ -19,6 +19,7 @@ import org.jfree.chart.axis.*; import org.jfree.chart.plot.PlotOrientation; import org.jfree.chart.plot.XYPlot; import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer; +import org.jfree.data.Range; import org.jfree.data.xy.XYSeries; import org.jfree.data.xy.XYSeriesCollection; @@ -29,9 +30,8 @@ import gobs.util.*; * */ public class MultiLogXPlot + extends AbstractPlot { - public static boolean bw = false; - /** Generate simple plot. @param collection The x,y data @@ -100,6 +100,14 @@ public class MultiLogXPlot LogarithmicAxis domainAxis = new LogarithmicAxis(xlabel); domainAxis.setLabelFont(labelFont); domainAxis.setTickLabelFont(tickFont); + if (forceDomainUpper) + { + Range range = plot.getDomainAxis().getRange(); + double lower = range.getLowerBound(); + Range adjusted = new Range(lower, forceDomainUpperValue); + domainAxis.setRange(adjusted); + System.out.println("range: " + domainAxis.getRange()); + } plot.setDomainAxis(domainAxis); chart.draw(g2d, rectangle); diff --git a/sim/gobs/src/gobs/plot/MultiplePlot.java b/sim/gobs/src/gobs/plot/MultiplePlot.java index 1c7521d..084b50c 100644 --- a/sim/gobs/src/gobs/plot/MultiplePlot.java +++ b/sim/gobs/src/gobs/plot/MultiplePlot.java @@ -27,9 +27,8 @@ import gobs.util.Bits; * */ public class MultiplePlot + extends AbstractPlot { - public static boolean bw = false; - /** Generate simple plot. @param collection The x,y data. @@ -46,9 +45,6 @@ public class MultiplePlot Rectangle2D.Double rectangle = null; OutputStream out = null; - int w = 400; - int h = 400; - try { out = new FileOutputStream(output); @@ -58,11 +54,11 @@ public class MultiplePlot g2d.setGraphicContext (new org.apache.xmlgraphics.java2d.GraphicContext()); - rectangle = new Rectangle2D.Double(0, 0, w, h); + rectangle = new Rectangle2D.Double(0, 0, width, height); g2d.setGraphicContext (new org.apache.xmlgraphics.java2d.GraphicContext()); - g2d.setupDocument(out, w, h); + g2d.setupDocument(out, width, height); } catch (IOException e) { diff --git a/sim/gobs/src/gobs/plot/Plot.java b/sim/gobs/src/gobs/plot/Plot.java index 685db0f..695c7f6 100644 --- a/sim/gobs/src/gobs/plot/Plot.java +++ b/sim/gobs/src/gobs/plot/Plot.java @@ -30,7 +30,7 @@ public class Plot EPSDocumentGraphics2D g2d = null; OutputStream out = null; - Rectangle2D.Double rectangle = new Rectangle2D.Double(0, 0, w, h); + Rectangle2D.Double rectangle = new Rectangle2D.Double(0, 0, width, height); try { diff --git a/sim/gobs/src/gobs/sim/Rebuild.java b/sim/gobs/src/gobs/sim/Rebuild.java index 0cda51c..b1518e1 100644 --- a/sim/gobs/src/gobs/sim/Rebuild.java +++ b/sim/gobs/src/gobs/sim/Rebuild.java @@ -446,11 +446,17 @@ public class Rebuild { double result = 0.0; for (Copy copy : entries) - if (copy.finish > result) - result = copy.finish; + { + double t = copy.finish-start; + if (t > result) + result = t; + } for (Copy copy : repair.entries) - if (copy.finish > result) - result = copy.finish; + { + double t = copy.finish-start; + if (t > result) + result = t; + } return result; } diff --git a/sim/gobs/src/gobs/sim/Simulator.java b/sim/gobs/src/gobs/sim/Simulator.java index 703d1e9..ed6f2ac 100644 --- a/sim/gobs/src/gobs/sim/Simulator.java +++ b/sim/gobs/src/gobs/sim/Simulator.java @@ -19,9 +19,19 @@ import static gobs.util.Tools.bail; public class Simulator { + /** + Bit-length of addresses. + */ int B; + + /** + Number of server nodes. + */ int N; + /** + 2^B + */ public BigInteger M; int fileCount; @@ -374,7 +384,7 @@ public class Simulator // MTTC = span/mttfLoss = # loss events, so: dataLossFraction = mttc/events; - print("mttf: " + df.format(mttf)); + print("mttf: " + Time.toString(mttf)); print("dataLossFraction: " + df.format(dataLossFraction)); DRand engine = new DRand(Bits.nextInt()); @@ -529,9 +539,11 @@ public class Simulator void place() { + Date start = new Date(); print("allocating: " + objects.size()); placer.place(objects, nodes); - + Date stop = new Date(); + print("took: " + Time.diff(start, stop)); origCount = countObjects(nodes); // print("OBJECT_COUNT: " + objects.size()); @@ -717,6 +729,7 @@ public class Simulator while (! complete()) { fault = next; + Date start = new Date(); System.out.println("fault: " + faultCount); Node failed = induceFault(fault); Collection<Gob> damaged = failed.collection(); @@ -726,6 +739,8 @@ public class Simulator Collection<Copy> fixes = failCopies(failed); todo.addAll(fixes); next = rebuild(fault, failed, damaged); + Date stop = new Date(); + print("took: " + Time.diff(start, stop)); } finish(fault); } @@ -1163,10 +1178,12 @@ public class Simulator */ void computeTraffic() { + Date start = new Date(); if (! has("plot.traffic.interval")) return; System.out.println("TRAFFIC: "); + System.out.println("REBUILDS:" + rebuilds.size()); double interval = decimal("plot.traffic.interval"); List<Double> intervals = new ArrayList<Double>(); @@ -1186,6 +1203,8 @@ public class Simulator stats.results.setProperty("traffic-x", intervals.toString()); stats.results.setProperty("traffic-y", sample.toString()); + Date finish = new Date(); + System.out.println("took: " + Time.diff(start, finish)); } /** diff --git a/sim/gobs/src/gobs/sweep/AbstractSweep.java b/sim/gobs/src/gobs/sweep/AbstractSweep.java index bb4c1c5..682b4e1 100644 --- a/sim/gobs/src/gobs/sweep/AbstractSweep.java +++ b/sim/gobs/src/gobs/sweep/AbstractSweep.java @@ -205,7 +205,7 @@ public abstract class AbstractSweep } /** - Sweep over all integer values between min and max + Sweep over all values between min and max stepping over the interval argument. @param arg The argument, e.g., "interval:4". @return The list, e.g., min, min+4, min+8, max. @@ -259,6 +259,64 @@ public abstract class AbstractSweep return result; } + public static List<String> values(String step, + double min, double max) + { + /* + if (step.equals("all")) + return all(min,max); + else if (step.equals("tens")) + return tens(min,max); + else if (step.equals("exps")) + return exps(min,max); + else if (step.equals("mids")) + return mids(min,max); + else */ + if (step.startsWith("interval")) + return interval(step, min, max); + else if (step.startsWith("literal")) + return literal(step); + else + Tools.bail("Sweep.values(): Bad step: " + step); + return null; + } + + /** + Sweep over all values between min and max + stepping over the interval argument. + @param arg The argument, e.g., "interval:4". + @return The list, e.g., min, min+4, min+8, max. + */ + static List<String> interval(String arg, double min, double max) + { + String[] tokens = arg.split(":"); + + double interval = 0; + try + { + interval = Double.parseDouble(tokens[1]); + } + catch (NumberFormatException e) + { + Tools.bail("AbstractSweep: interval must be an double!"); + } + + Tools.check(interval > 0, + "AbstractSweep: interval must be a positive double!"); + + List<String> result = new ArrayList<String>(); + result.add(""+min); + int i = 1; + while (min + interval*i < max) + { + result.add("" + (min+interval*i)); + i++; + } + if (!result.contains(""+max)) + result.add(""+max); + return result; + } + /** Set properties from memory. */ diff --git a/sim/gobs/src/gobs/sweep/Helpers.java b/sim/gobs/src/gobs/sweep/Helpers.java index 4ec29e2..418d975 100755 --- a/sim/gobs/src/gobs/sweep/Helpers.java +++ b/sim/gobs/src/gobs/sweep/Helpers.java @@ -14,8 +14,8 @@ public class Helpers { public static List<String> valuesMTTF(Properties properties) { - int minMTTF = Tools.integer(properties, "disk.mttf.min"); - int maxMTTF = Tools.integer(properties, "disk.mttf.max"); + double minMTTF = Tools.decimal(properties, "disk.mttf.min"); + double maxMTTF = Tools.decimal(properties, "disk.mttf.max"); String stepMTTF = Tools.string(properties, "disk.mttf.step"); List<String> valuesMTTF = Sweep.values(stepMTTF, minMTTF, maxMTTF); @@ -38,5 +38,4 @@ public class Helpers properties.setProperty("replicas", ""+replicas); return raid; } - } diff --git a/sim/gobs/src/gobs/sweep/Sweep.java b/sim/gobs/src/gobs/sweep/Sweep.java index ce2263e..a3eb10b 100644 --- a/sim/gobs/src/gobs/sweep/Sweep.java +++ b/sim/gobs/src/gobs/sweep/Sweep.java @@ -17,6 +17,11 @@ import gobs.util.*; public class Sweep extends AbstractSweep { + static + { + System.out.println("SWEEP_START: " + Time.sdf(new Date())); + } + public Map<String,Simulator> sims = new Hashtable<String,Simulator>(); @@ -56,12 +61,14 @@ public class Sweep { for (int i = 0; i < repeats; i++) { - System.out.println("\nEXECUTE:"); + String tag = xname+"="+v+";unique="+i; + System.out.println("\nEXECUTE: " + tag); properties.setProperty("unique", new Integer(i).toString()); properties.setProperty(xname, v); + properties.setProperty("tag", tag); Simulator simulator = new Simulator(properties); simulator.execute(); - sims.put(xname+"="+v+";unique="+i, simulator); + sims.put(tag, simulator); } } } diff --git a/sim/gobs/src/gobs/sweep/x/DataLoss01.java b/sim/gobs/src/gobs/sweep/x/DataLoss01.java index 58b3f88..043061b 100644 --- a/sim/gobs/src/gobs/sweep/x/DataLoss01.java +++ b/sim/gobs/src/gobs/sweep/x/DataLoss01.java @@ -30,57 +30,52 @@ class DataLoss01 String output = Tools.string(properties, "output"); - int minNodes = Tools.integer(properties, "nodes.min"); - int maxNodes = Tools.integer(properties, "nodes.max"); - String stepNodes = Tools.string(properties, "nodes.step"); - List<String> valuesNodes = - Sweep.values(stepNodes, minNodes, maxNodes); - - int minMTTF = Tools.integer(properties, "mttf.min"); - int maxMTTF = Tools.integer(properties, "mttf.max"); - String stepMTTF = Tools.string(properties, "mttf.step"); - List<String> valuesMTTF = - Sweep.values(stepMTTF, minMTTF, maxMTTF); - - System.out.println("DataLoss01: nodes: " + valuesNodes + "\n" + - " mttf: " + valuesMTTF); - - final XYSeriesCollection collection = new XYSeriesCollection(); + List<String> systems = Tools.list(properties, "systems"); - properties.setProperty("impl.placement", "Nearest"); - Sweep pSweep = new Sweep(properties); - pSweep.repeats = Tools.integer(properties, "repeats"); - pSweep.execute("nodes", valuesNodes, "mttf", valuesMTTF); + String timeunit = "seconds"; + double scale = 1; + if (Tools.has(properties, "plot.timeunit")) + { + timeunit = Tools.string(properties, "plot.timeunit"); + if (timeunit.equals("minutes")) + scale = 60; + else if (timeunit.equals("hours")) + scale = 3600; + else if (timeunit.equals("days")) + scale = 86400; + } - List<XYSeries> allSeries = new ArrayList<XYSeries>(); + List<String> valuesMTTF = Helpers.valuesMTTF(properties); - List<XYSeries> pSeries = plots(pSweep, "PAST", - valuesNodes, valuesMTTF); - allSeries.addAll(pSeries); + System.out.println("DataLoss01: systems: " + systems + "\n" + + " mttf: " + valuesMTTF); - /* - properties.setProperty("impl.placement", "Kademlia"); - ArraySweep kSweep = new ArraySweep(properties); - kSweep.repeats = Tools.integer(properties, "repeats"); - kSweep.execute("Kademlia", "nodes", values, "distribution"); - allSeries.addAll(kSweep.collection); + final XYSeriesCollection collection = new XYSeriesCollection(); + List<XYSeries> allSeries = new ArrayList<XYSeries>(); - properties.setProperty("impl.placement", "VertexGroups"); - ArraySweep vgSweep = new ArraySweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); - allSeries.addAll(vgSweep.collection); - */ + properties.setProperty("impl.placement", "Nearest"); + for (String system : systems) + { + properties.setProperty("system", system); + Helpers.setupRAID(properties, system); + String mttc = Tools.string(properties, system+".mttc"); + properties.setProperty("mttc", mttc); + Sweep pSweep = new Sweep(properties); + pSweep.repeats = Tools.integer(properties, "repeats"); + pSweep.execute("disk.mttf", valuesMTTF); + + List<XYSeries> pSeries = plots(pSweep, "NEAREST-"+system, + valuesMTTF); + allSeries.addAll(pSeries); + } for (XYSeries series : allSeries) collection.addSeries(series); + MultiplePlot.configure(properties); MultiplePlot.plot(collection, "", "Mean Time to Disk Failure", - "Mean Time to Object Loss", output); - - // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); + "Objects Lost per Year", output); } /** @@ -88,33 +83,28 @@ class DataLoss01 @param label E.g., "PAST" */ static List<XYSeries> plots(Sweep sweep, String label, - List<String> valuesNodes, List<String> valuesMTTF) { List<XYSeries> results = new ArrayList<XYSeries>(); - for (String nodes : valuesNodes) + XYSeries series = new XYSeries(label); + for (String mttf : valuesMTTF) { - XYSeries series = new XYSeries(label + " N=" + nodes); - for (String mttf : valuesMTTF) - { - double time = 0.0; - int lost = 0; - double total = 0.0; - for (Simulator simulator : sweep.sims.values()) - if (simulator.string("nodes").equals(nodes) && - simulator.string("mttf").equals(mttf)) - { - System.out.println(simulator.string("tag")); - time += Tools.decimal(simulator.stats.results, "time"); - lost += Tools.integer(simulator.stats.results, "lost"); - total++; - } - if (lost > 0) - series.add(Integer.parseInt(mttf), time/lost); - } - results.add(series); + double time = 0.0; + int lost = 0; + double total = 0.0; + for (Simulator simulator : sweep.sims.values()) + if (simulator.string("disk.mttf").equals(mttf)) + { + System.out.println(simulator.string("tag")); + time += Tools.decimal(simulator.stats.results, "time"); + lost += Tools.integer(simulator.stats.results, "lost"); + total++; + } + series.add(Long.parseLong(mttf), lost); } + results.add(series); + return results; } diff --git a/sim/gobs/src/gobs/sweep/x/DataLoss02.java b/sim/gobs/src/gobs/sweep/x/DataLoss02.java index 61973e0..3faf206 100755 --- a/sim/gobs/src/gobs/sweep/x/DataLoss02.java +++ b/sim/gobs/src/gobs/sweep/x/DataLoss02.java @@ -31,8 +31,8 @@ public class DataLoss02 String output = Tools.string(properties, "output"); - int minMTTF = Tools.integer(properties, "disk.mttf.min"); - int maxMTTF = Tools.integer(properties, "disk.mttf.max"); + double minMTTF = Tools.decimal(properties, "disk.mttf.min"); + double maxMTTF = Tools.decimal(properties, "disk.mttf.max"); String stepMTTF = Tools.string(properties, "disk.mttf.step"); List<String> valuesMTTF = Sweep.values(stepMTTF, minMTTF, maxMTTF); diff --git a/sim/gobs/src/gobs/sweep/x/Decluster03.java b/sim/gobs/src/gobs/sweep/x/Decluster03.java index ace67ab..2977916 100644 --- a/sim/gobs/src/gobs/sweep/x/Decluster03.java +++ b/sim/gobs/src/gobs/sweep/x/Decluster03.java @@ -58,17 +58,8 @@ class Decluster03 values, "maxTotal"); collection.addSeries(series); } - /* - properties.setProperty("impl.placement", "VertexGroups"); - Sweep vgSweep = new Sweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - series = vgSweep.perform("Vertex Groups", "node.count", - values, "maxTotal"); - collection.addSeries(series); - */ - - if (Tools.enabled(properties, "plot.blackandwhite")) - MultiLogXPlot.bw = true; + + MultiLogXPlot.configure(properties); MultiLogXPlot.plot(collection, "", "node.count", "load fraction(%)", output); } diff --git a/sim/gobs/src/gobs/sweep/x/Rebuild01.java b/sim/gobs/src/gobs/sweep/x/Rebuild01.java index 7152eb6..dbc4e3b 100644 --- a/sim/gobs/src/gobs/sweep/x/Rebuild01.java +++ b/sim/gobs/src/gobs/sweep/x/Rebuild01.java @@ -30,6 +30,19 @@ class Rebuild01 String output = Tools.string(properties, "output"); + String timeunit = "seconds"; + double scale = 1; + if (Tools.has(properties, "plot.timeunit")) + { + timeunit = Tools.string(properties, "plot.timeunit"); + if (timeunit.equals("minutes")) + scale = 60; + else if (timeunit.equals("hours")) + scale = 3600; + else if (timeunit.equals("days")) + scale = 86400; + } + List<String> valuesMTTF = Helpers.valuesMTTF(properties); System.out.println("Rebuild01: " + valuesMTTF); @@ -50,30 +63,33 @@ class Rebuild01 List<XYSeries> series = pSweep.execute("NEAREST-"+system+" MTTF=", "disk.mttf", valuesMTTF, "traffic-x", "traffic-y"); + for (XYSeries s : series) + s.setKey("NEAREST-target"); allSeries.addAll(series); } - /* - properties.setProperty("impl.placement", "Kademlia"); - ArraySweep kSweep = new ArraySweep(properties); - kSweep.repeats = Tools.integer(properties, "repeats"); - kSweep.execute("Kademlia", "nodes", values, "distribution"); - allSeries.addAll(kSweep.collection); - - properties.setProperty("impl.placement", "VertexGroups"); - ArraySweep vgSweep = new ArraySweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); - allSeries.addAll(vgSweep.collection); - */ - for (XYSeries series : allSeries) - collection.addSeries(series); + { + XYSeries scaled = scaleX(series, scale); + collection.addSeries(scaled); + } - if (Tools.enabled(properties, "plot.blackandwhite")) - MultiplePlot.bw = true; + MultiplePlot.configure(properties); MultiplePlot.plot(collection, "", - "time", "copies in flight", + "time ("+timeunit+")", "copies in flight", output); } + + static XYSeries scaleX(XYSeries series, double scale) + { + XYSeries result = new XYSeries(series.getKey()); + for (int i = 0; i < series.getItemCount(); i++) + { + Number x = series.getX(i); + Number y = series.getY(i); + double dx = x.doubleValue()/scale; + result.add(dx,y); + } + return result; + } } diff --git a/sim/gobs/src/gobs/sweep/x/Rebuild02.java b/sim/gobs/src/gobs/sweep/x/Rebuild02.java index fc4b7fd..a4fd2c7 100644 --- a/sim/gobs/src/gobs/sweep/x/Rebuild02.java +++ b/sim/gobs/src/gobs/sweep/x/Rebuild02.java @@ -34,12 +34,22 @@ class Rebuild02 String output = Tools.string(properties, "output"); double interval = Tools.decimal(properties, "plot.traffic.interval"); + String timeunit = "seconds"; + double scale = 1; + if (Tools.has(properties, "plot.timeunit")) + { + timeunit = Tools.string(properties, "plot.timeunit"); + if (timeunit.equals("minutes")) + scale = 60; + else if (timeunit.equals("hours")) + scale = 3600; + } double diskSize = Tools.decimal(properties, "disk.size"); double diskSpeed = Tools.decimal(properties, "disk.speed"); - int minMTTF = Tools.integer(properties, "disk.mttf.min"); - int maxMTTF = Tools.integer(properties, "disk.mttf.max"); + double minMTTF = Tools.decimal(properties, "disk.mttf.min"); + double maxMTTF = Tools.decimal(properties, "disk.mttf.max"); String stepMTTF = Tools.string(properties, "disk.mttf.step"); List<String> valuesMTTF = Sweep.values(stepMTTF, minMTTF, maxMTTF); @@ -50,72 +60,49 @@ class Rebuild02 List<XYSeries> allSeries = new ArrayList<XYSeries>(); - properties.setProperty("impl.placement", "Nearest"); - List<String> systems = Tools.list(properties, "systems"); - List<Simulator> sims = new ArrayList<Simulator>(); - for (String system : systems) + for (String skip : new String[] { "false", "true" }) { - int nodeDevices = Tools.integer(properties, - system+".node.osds"); - String spec = Tools.string(properties, system+".raid"); - RAID raid = new RAID(spec, diskSize, diskSpeed); - int nodes = raid.estimateNodes(properties, nodeDevices); - properties.setProperty("raid", spec); - properties.setProperty("node.count", ""+nodes); - properties.setProperty("node.osds", ""+nodeDevices); - properties.setProperty("regimen.skip", "false"); - properties.setProperty("system", system); - - Sweep pySweep = new Sweep(properties); - pySweep.repeats = Tools.integer(properties, "repeats"); - - pySweep.execute("disk.mttf", valuesMTTF); - sims.addAll(pySweep.sims.values()); - } + properties.setProperty("impl.placement", "Nearest"); + List<String> systems = Tools.list(properties, "systems"); + List<Simulator> sims = new ArrayList<Simulator>(); + for (String system : systems) + { + int nodeDevices = Tools.integer(properties, + system+".node.osds"); + String spec = Tools.string(properties, system+".raid"); + RAID raid = new RAID(spec, diskSize, diskSpeed); + int nodes = raid.estimateNodes(properties, nodeDevices); + String mttc = Tools.string(properties, system+".mttc"); + properties.setProperty("mttc", mttc); + properties.setProperty("raid", spec); + properties.setProperty("node.count", ""+nodes); + properties.setProperty("node.osds", ""+nodeDevices); + properties.setProperty("regimen.skip", skip); + properties.setProperty("system", system); + + Sweep pySweep = new Sweep(properties); + pySweep.repeats = Tools.integer(properties, "repeats"); + + pySweep.execute("disk.mttf", valuesMTTF); + sims.addAll(pySweep.sims.values()); + } - List<XYSeries> pySeries = plots(sims, "PAST-Y", - "system", systems, - "disk.mttf", valuesMTTF, - interval); - allSeries.addAll(pySeries); - - /* - properties.setProperty("impl.placement", "Nearest"); - Sweep pnSweep = new Sweep(properties); - pnSweep.repeats = Tools.integer(properties, "repeats"); - pnSweep.execute("nodes", values); - List<XYSeries> pnSeries = plots(pnSweep, "PAST-N", "nodes", - values, interval); - allSeries.addAll(pnSeries); - */ - /* - properties.setProperty("impl.placement", "Kademlia"); - Sweep kSweep = new Sweep(properties); - kSweep.repeats = Tools.integer(properties, "repeats"); - kSweep.execute("nodes", values); - - List<XYSeries> kSeries = plots(kSweep, "Kademlia", "nodes", - values, interval); - allSeries.addAll(kSeries); - */ - - /* - properties.setProperty("impl.placement", "VertexGroups"); - ArraySweep vgSweep = new ArraySweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); - allSeries.addAll(vgSweep.collection); - */ + String t = skip.equals("true") ? "latent" : "active"; + List<XYSeries> pySeries = plots(sims, "NEAREST-"+t, + "system", systems, + "disk.mttf", valuesMTTF, + interval, scale); + allSeries.addAll(pySeries); + } for (XYSeries series : allSeries) collection.addSeries(series); - MultiplePlot.plot(collection, "Rebuild Traffic", - "time", "copies in flight", - output); - // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); + MultiplePlot.configure(properties); + MultiplePlot.plot(collection, "", + "time ("+timeunit+")", "copies in flight", + output); Sweep.reportRuntime(); } @@ -123,7 +110,7 @@ class Rebuild02 static List<XYSeries> plots(List<Simulator> sims, String label, String x1, List<String> values1, String x2, List<String> values2, - double interval) + double interval, double scale) { List<XYSeries> results = new ArrayList<XYSeries>(); @@ -132,13 +119,14 @@ class Rebuild02 for (String v2 : values2) { XYSeries series = - new XYSeries(label + " SYSTEM=" + v1 + ":" + v2); + // new XYSeries(label + " " + v1 + ":" + v2); + new XYSeries(label + " " + v1 + " "); List<Rebuild> rebuilds = new ArrayList<Rebuild>(); for (Simulator simulator : sims) if (Tools.string(simulator.properties, x1).equals(v1) && Tools.string(simulator.properties, x2).equals(v2)) rebuilds.addAll(simulator.rebuilds); - loadsToSeries(rebuilds, interval, series); + loadsToSeries(rebuilds, interval, scale, series); results.add(series); } } @@ -150,24 +138,28 @@ class Rebuild02 Store in series. */ static void loadsToSeries(List<Rebuild> rebuilds, double interval, - XYSeries series) + double scale, XYSeries series) { + System.out.println("loadsToSeries():"); int count; double longest = longestRebuild(rebuilds); + System.out.println("longest: " + longest); double d = 0.0; while (d <= longest) { + System.out.println("sample: " + d); + System.out.println("rebuilds: " + rebuilds.size()); double total = 0.0; for (Rebuild rebuild : rebuilds) { // rebuild.printOffset(d, Cause.ANY); if (Tools.bool(properties, "regimen.skip")) - total += rebuild.loadOffset(d, Cause.ANY); + total += rebuild.loadOffset(d, Cause.REPAIR); else total += rebuild.loadOffset(d, Cause.FAULT); } double sample = total/rebuilds.size(); - series.add(d, sample); + series.add(d/scale, sample); d += interval; } series.add(d, 0.0); diff --git a/sim/gobs/src/gobs/util/Time.java b/sim/gobs/src/gobs/util/Time.java index 260ee02..8b5a00b 100755 --- a/sim/gobs/src/gobs/util/Time.java +++ b/sim/gobs/src/gobs/util/Time.java @@ -1,6 +1,7 @@ package gobs.util; -import java.text.DecimalFormat; +import java.text.*; +import java.util.Date; import static java.lang.Math.floor; @@ -12,19 +13,33 @@ public class Time public static final double hour = 60*minute; public static final double day = 24*hour; + static DecimalFormat zf = new DecimalFormat("0"); static DecimalFormat zz = new DecimalFormat("00"); + static DateFormat sdf = new SimpleDateFormat("MM/dd/yyyy hh:mm a"); public String toString() { return toString(value); } + /** + Get d2-d1 as mm:ss + */ + public static String diff(Date d1, Date d2) + { + long t1 = d1.getTime()/1000; + long t2 = d2.getTime()/1000; + double t = t2 - t1; + + return toString(t); + } + public static String toString(double time) { StringBuilder sb = new StringBuilder(32); if (time > day) { - sb.append(floor(time/day)).append(" "); + sb.append(zf.format(floor(time/day))).append(" "); time %= day; } if (time > hour || sb.length() > 0) @@ -42,6 +57,15 @@ public class Time } /** + Primarily for real times. + sdf = SimpleDateFormat + */ + public static String sdf(Date date) + { + return sdf.format(date); + } + + /** Debugging only. */ public static void main(String args[]) diff --git a/sim/gobs/src/gobs/util/Tools.java b/sim/gobs/src/gobs/util/Tools.java index fa0221f..acacab8 100644 --- a/sim/gobs/src/gobs/util/Tools.java +++ b/sim/gobs/src/gobs/util/Tools.java @@ -3,8 +3,8 @@ package gobs.util; import java.io.IOException; import java.math.BigInteger; +import java.text.DecimalFormat; import java.util.Arrays; -import java.util.Date; import java.util.List; import java.util.Properties; @@ -14,6 +14,8 @@ import java.util.Properties; public class Tools { + static DecimalFormat df = new DecimalFormat("00"); + public static int integer(Properties properties, String variable) { int result = 0; @@ -284,14 +286,4 @@ public class Tools return result; } - - public static String diff(Date d1, Date d2) - { - long t1 = d1.getTime(); - long t2 = d2.getTime(); - long t = t2-t1; - long s = t%60; - long m = t/60; - return "" + m + ":" + s; - } } diff --git a/sim/gobs/sweeps/dc03kpv01.cfg b/sim/gobs/sweeps/dc03kpv01.cfg index a51bc26..10c086a 100644 --- a/sim/gobs/sweeps/dc03kpv01.cfg +++ b/sim/gobs/sweeps/dc03kpv01.cfg @@ -1,3 +1,7 @@ -include = kpv01.cfg system_20-1000.cfg dc_output.cfg user00.cfg +include = kpv01.cfg system_100-700.cfg fs_pdsi_1EB.cfg \ + dc_output.cfg user00.cfg +plot.force.domain.upper = 1100 + +fault.count = 1 diff --git a/sim/gobs/sweeps/dc03kpv03.cfg b/sim/gobs/sweeps/dc03kpv03.cfg index 24e03f5..c5d72e3 100644 --- a/sim/gobs/sweeps/dc03kpv03.cfg +++ b/sim/gobs/sweeps/dc03kpv03.cfg @@ -1,3 +1,5 @@ -include = kpv03.cfg system_20-1000.cfg dc_output.cfg user00.cfg +include = kpv03.cfg fs_pdsi_1EB.cfg system_100-700.cfg \ + dc_output.cfg user00.cfg +plot.force.domain.upper = 1100 diff --git a/sim/gobs/sweeps/dc03kpv05.cfg b/sim/gobs/sweeps/dc03kpv05.cfg index aab6a3f..abdfb49 100644 --- a/sim/gobs/sweeps/dc03kpv05.cfg +++ b/sim/gobs/sweeps/dc03kpv05.cfg @@ -1,2 +1,5 @@ -include = kpv05.cfg system_20-1000.cfg dc_output.cfg user00.cfg +include = kpv05.cfg system_100-700.cfg fs_pdsi_1EB.cfg \ + dc_output.cfg user00.cfg + +plot.force.domain.upper = 1100 diff --git a/sim/gobs/sweeps/dc04kpv01.cfg b/sim/gobs/sweeps/dc04kpv01.cfg index 9db537c..1b6f9bd 100644 --- a/sim/gobs/sweeps/dc04kpv01.cfg +++ b/sim/gobs/sweeps/dc04kpv01.cfg @@ -1,2 +1,5 @@ -include = kpv01.cfg system_600.cfg dc_output.cfg user00.cfg +include = kpv01.cfg system_600.cfg fs_pdsi_1EB.cfg \ + dc_output.cfg user00.cfg + +fault.count = 1 diff --git a/sim/gobs/sweeps/dc04kpv03.cfg b/sim/gobs/sweeps/dc04kpv03.cfg index 9db537c..c19a005 100644 --- a/sim/gobs/sweeps/dc04kpv03.cfg +++ b/sim/gobs/sweeps/dc04kpv03.cfg @@ -1,2 +1,6 @@ -include = kpv01.cfg system_600.cfg dc_output.cfg user00.cfg +include = kpv03.cfg system_600.cfg fs_pdsi_1EB.cfg \ + dc_output.cfg user00.cfg + +fault.count = 1 +plot.blackandwhite = true diff --git a/sim/gobs/sweeps/dc04kpv05.cfg b/sim/gobs/sweeps/dc04kpv05.cfg index 9db537c..4962c40 100644 --- a/sim/gobs/sweeps/dc04kpv05.cfg +++ b/sim/gobs/sweeps/dc04kpv05.cfg @@ -1,2 +1,6 @@ -include = kpv01.cfg system_600.cfg dc_output.cfg user00.cfg +include = kpv05.cfg system_600.cfg fs_pdsi_1EB.cfg \ + dc_output.cfg user00.cfg + +fault.count = 1 +plot.blackandwhite = true diff --git a/sim/gobs/sweeps/dc_output.cfg b/sim/gobs/sweeps/dc_output.cfg index b3b7d43..f26b687 100644 --- a/sim/gobs/sweeps/dc_output.cfg +++ b/sim/gobs/sweeps/dc_output.cfg @@ -4,17 +4,16 @@ # Print all properties at simulation start print.properties = true # Print nodes and objects at startup (true/false) -print.initialnodes = true +print.initialnodes = false # Print nodes and objects after each rebuild (true/false) -print.nodes = true +print.nodes = false # Print object tranfers during rebuilds (true/false) -print.rebuilds = true +print.rebuilds = false # Print file information (true/false) -print.files = true -print.filesobjects = true +print.files = false +print.filesobjects = false # Print load change statistics (true/false) print.loaddiffs = true # Print user accesses print.accesses = false -plot.blackandwhite = true diff --git a/sim/gobs/sweeps/decluster03.zsh b/sim/gobs/sweeps/decluster03.zsh index 4b9cc9a..d3a5ff1 100755 --- a/sim/gobs/sweeps/decluster03.zsh +++ b/sim/gobs/sweeps/decluster03.zsh @@ -6,7 +6,7 @@ NAME=$1 NAME="dc03${NAME}" -# SEED="seed=821" +SEED="seed=822" CLASS=gobs.sweep.x.Decluster03 SWEEPDIR=$( dirname $0 ) diff --git a/sim/gobs/sweeps/disks-full-zaps-only.cfg b/sim/gobs/sweeps/disks-full-zaps-only.cfg new file mode 100644 index 0000000..87e9e0c --- /dev/null +++ b/sim/gobs/sweeps/disks-full-zaps-only.cfg @@ -0,0 +1,19 @@ + +# 30 TB T G M K +disk.size = 30000000000000 +disk.speed = 400000000 + +# RAID: None +cluster.node.osds = 1 +cluster.raid = 1+0 +cluster.replicas = 4 + +# RAID: 4 @ 4+2 +san.node.osds = 8 +san.raid = 8+2 +san.replicas = 2 + +# RAID: 2 @ 3+1 +target.node.osds = 4 +target.raid = 4+1 +target.replicas = 3 diff --git a/sim/gobs/sweeps/disks-full.cfg b/sim/gobs/sweeps/disks-full.cfg index 93f56f4..fe2f5ce 100644 --- a/sim/gobs/sweeps/disks-full.cfg +++ b/sim/gobs/sweeps/disks-full.cfg @@ -1,31 +1,22 @@ -mttr.reboot = 100 -mttr.disk = 500 - # 30 TB T G M K disk.size = 30000000000000 disk.speed = 400000000 -disk.mttf.min = 20000000 -disk.mttf.max = 30000000 -disk.mttf.step = interval:10000000 - -systems = san;target +# MTTC: 3 months # RAID: None cluster.node.osds = 1 cluster.raid = 1+0 cluster.replicas = 4 -cluster.mttc = 20000 -# RAID: 4 @ 4+2 +# RAID: 8 @ 8+2 san.node.osds = 8 san.raid = 8+2 san.replicas = 2 -san.mttc = 20000 -# RAID: 2 @ 3+1 +# RAID: 4 @ 4+1 target.node.osds = 4 -target.raid = 3+1 +target.raid = 4+1 target.replicas = 3 -target.mttc = 60000 + diff --git a/sim/gobs/sweeps/disks-long.cfg b/sim/gobs/sweeps/disks-long.cfg new file mode 100644 index 0000000..132a5dc --- /dev/null +++ b/sim/gobs/sweeps/disks-long.cfg @@ -0,0 +1,18 @@ + +mttr.reboot = 10 +mttr.disk = 50 + +disk.size = 500000 +disk.speed = 3000 + +disk.mttf.min = 10000 +disk.mttf.max = 10000 +disk.mttf.step = literal:10000 + +systems = target + +# RAID: 2 @ 3+1 +target.node.osds = 2 +target.raid = 3+1 +target.replicas = 3 +target.mttc = 40000 diff --git a/sim/gobs/sweeps/disks-medium-zaps-only.cfg b/sim/gobs/sweeps/disks-medium-zaps-only.cfg new file mode 100644 index 0000000..6922b8a --- /dev/null +++ b/sim/gobs/sweeps/disks-medium-zaps-only.cfg @@ -0,0 +1,38 @@ + +# 1 day +mttr.reboot = 3600 +mttr.disk = 86400 + +# 5GB +disk.size = 5000000000 +# 100MB/s +disk.speed = 100000000 + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 315360000 +disk.mttf.max = 315360000 +disk.mttf.step = interval:10000000 + +systems = san;target + +# 3 months: 7776000 + +# RAID: None +cluster.node.osds = 1 +cluster.raid = 1+0 +cluster.replicas = 4 +cluster.mttc = 30000000000 + +# RAID: 4 @ 4+2 +san.node.osds = 8 +san.raid = 4+2 +san.replicas = 2 +san.mttc = 30000000000 + +# RAID: 2 @ 3+1 +target.node.osds = 2 +target.raid = 3+1 +target.replicas = 3 +target.mttc = 30000000000 diff --git a/sim/gobs/sweeps/disks-medium.cfg b/sim/gobs/sweeps/disks-medium.cfg index 0be9e02..e794faf 100644 --- a/sim/gobs/sweeps/disks-medium.cfg +++ b/sim/gobs/sweeps/disks-medium.cfg @@ -1,32 +1,25 @@ -mttr.reboot = 100 -mttr.disk = 500 - # 5GB disk.size = 5000000000 # 100MB/s disk.speed = 100000000 -disk.mttf.min = 20000 -disk.mttf.max = 30000 -disk.mttf.step = interval:10000 - -systems = san;target +# 3 months: 7776000 # RAID: None cluster.node.osds = 1 cluster.raid = 1+0 cluster.replicas = 4 -cluster.mttc = 20000 +cluster.mttc = 7776000 # RAID: 4 @ 4+2 san.node.osds = 8 san.raid = 4+2 san.replicas = 2 -san.mttc = 20000 +san.mttc = 7776000 # RAID: 2 @ 3+1 target.node.osds = 2 target.raid = 3+1 target.replicas = 3 -target.mttc = 60000 +target.mttc = 7776000 diff --git a/sim/gobs/sweeps/disks-small.cfg b/sim/gobs/sweeps/disks-small.cfg index 5d47470..56b17c0 100644 --- a/sim/gobs/sweeps/disks-small.cfg +++ b/sim/gobs/sweeps/disks-small.cfg @@ -1,16 +1,7 @@ -mttr.reboot = 10 -mttr.disk = 50 - disk.size = 500000 disk.speed = 3000 -disk.mttf.min = 6000 -disk.mttf.max = 10000 -disk.mttf.step = interval:4000 - -systems = san;target - # RAID: None cluster.node.osds = 1 cluster.raid = 1+0 diff --git a/sim/gobs/sweeps/dl01-full.cfg b/sim/gobs/sweeps/dl01-full.cfg new file mode 100644 index 0000000..5e3fad1 --- /dev/null +++ b/sim/gobs/sweeps/dl01-full.cfg @@ -0,0 +1,33 @@ + +# Simulator configuration + +# Basics: + +include = disks-full.cfg fs_pdsi_1EB_chunky.cfg faults_real_range_dl01.cfg \ + user00.cfg dl_output.cfg + +# Bit-length of addresses: +B = 32 + +systems = san;target + +# 1 year: 31536000 +simulator.duration = 3153600 + +plot.blackandwhite = true + +# Clear some memory after each simulator run +sweep.free = true + +# Do a fsck +fsck = true +# Check for stray objects +strays = true + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/dl01-full.zsh b/sim/gobs/sweeps/dl01-full.zsh new file mode 100755 index 0000000..ffbae45 --- /dev/null +++ b/sim/gobs/sweeps/dl01-full.zsh @@ -0,0 +1,9 @@ +#!/bin/zsh + +NAME="dl01-full" +SEED="seed=821" +CLASS=gobs.sweep.x.DataLoss01 + +SWEEPDIR=$( dirname $0 ) + +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/dl01-medium.cfg b/sim/gobs/sweeps/dl01-medium.cfg index 820d382..9980cdb 100644 --- a/sim/gobs/sweeps/dl01-medium.cfg +++ b/sim/gobs/sweeps/dl01-medium.cfg @@ -3,16 +3,18 @@ # Basics: -include = system_30-100.cfg faults-range02.cfg user00.cfg \ - fs_pdsi_medium.cfg dl_output.cfg +include = disks-medium.cfg fs_pdsi_medium.cfg faults_medium_range_dl01.cfg \ + user00.cfg dl_output.cfg # Bit-length of addresses: B = 32 -fault.count = 30 -fault.dataloss = 0.5 +systems = san;target -node.diskspernode = 1 +# fault.count = 100 +simulator.duration = 31536000 + +plot.blackandwhite = true # Clear some memory after each simulator run sweep.free = true @@ -22,18 +24,10 @@ fsck = true # Check for stray objects strays = true -### Choose a PlacementScheme implementation: - -#impl.placement = Kademlia -impl.placement = Nearest -#impl.placement = VertexGroups - vertexgroups.prefix = 6 # Replica selection: See Replicated.locateSource() replica.source = primary ### Choose a NodeScheme implementation: -# impl.nodefactory = RandomNodes impl.nodefactory = SimpleNodes - diff --git a/sim/gobs/sweeps/dl01-small.cfg b/sim/gobs/sweeps/dl01-small.cfg index 0afc246..c40d2a5 100644 --- a/sim/gobs/sweeps/dl01-small.cfg +++ b/sim/gobs/sweeps/dl01-small.cfg @@ -3,16 +3,17 @@ # Basics: -include = system_10-20.cfg faults-range01.cfg user00.cfg \ - fs_pdsi_mini.cfg dl_output.cfg +include = disks-small.cfg fs_pdsi_mini.cfg faults_small.cfg \ + user00.cfg dl_output.cfg # Bit-length of addresses: B = 16 -fault.count = 10 -fault.dataloss = 0.5 +systems = san;target -node.diskspernode = 1 +fault.count = 100 + +plot.blackandwhite = true # Clear some memory after each simulator run sweep.free = true @@ -22,18 +23,11 @@ fsck = true # Check for stray objects strays = true -### Choose a PlacementScheme implementation: - -#impl.placement = Kademlia -impl.placement = Nearest -#impl.placement = VertexGroups - vertexgroups.prefix = 6 # Replica selection: See Replicated.locateSource() replica.source = primary ### Choose a NodeScheme implementation: -# impl.nodefactory = RandomNodes impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/dl01.cfg b/sim/gobs/sweeps/dl01.cfg index 5ab20d2..6691f99 100644 --- a/sim/gobs/sweeps/dl01.cfg +++ b/sim/gobs/sweeps/dl01.cfg @@ -11,7 +11,7 @@ include = dl_output.cfg sweep.free = true # Rebuild traffic over time analysis: -plot.traffic.interval = 10 +# plot.traffic.interval = 10 ### Parameters for GOBS Simulator run diff --git a/sim/gobs/sweeps/dl_output.cfg b/sim/gobs/sweeps/dl_output.cfg index c9372d3..f26b687 100644 --- a/sim/gobs/sweeps/dl_output.cfg +++ b/sim/gobs/sweeps/dl_output.cfg @@ -4,13 +4,13 @@ # Print all properties at simulation start print.properties = true # Print nodes and objects at startup (true/false) -print.initialnodes = true +print.initialnodes = false # Print nodes and objects after each rebuild (true/false) -print.nodes = true +print.nodes = false # Print object tranfers during rebuilds (true/false) -print.rebuilds = true +print.rebuilds = false # Print file information (true/false) -print.files = true +print.files = false print.filesobjects = false # Print load change statistics (true/false) print.loaddiffs = true diff --git a/sim/gobs/sweeps/faults_medium.cfg b/sim/gobs/sweeps/faults_medium.cfg new file mode 100644 index 0000000..7c01ad7 --- /dev/null +++ b/sim/gobs/sweeps/faults_medium.cfg @@ -0,0 +1,10 @@ + +mttr.reboot = 3600 +mttr.disk = 86400 + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 315360000 +disk.mttf.max = 315360000 +disk.mttf.step = interval:10000 diff --git a/sim/gobs/sweeps/faults_medium_range_dl01.cfg b/sim/gobs/sweeps/faults_medium_range_dl01.cfg new file mode 100644 index 0000000..9fbae40 --- /dev/null +++ b/sim/gobs/sweeps/faults_medium_range_dl01.cfg @@ -0,0 +1,18 @@ + +mttr.reboot = 3600 +mttr.disk = 86400 + +# Disk MTTF: 1-100 years. + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +#disk.mttf.min = 31536000 +#disk.mttf.max = 31536000000 +#disk.mttf.step = literal:31536000,31536000000 +#exps + +# Experimental short values +disk.mttf.min = 3153 +disk.mttf.max = 3153600 +disk.mttf.step = literal:31536000,31536000000 diff --git a/sim/gobs/sweeps/faults_real.cfg b/sim/gobs/sweeps/faults_real.cfg new file mode 100644 index 0000000..d2892bb --- /dev/null +++ b/sim/gobs/sweeps/faults_real.cfg @@ -0,0 +1,10 @@ + +mttr.reboot = 3600 +mttr.disk = 86400 + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 315360000 +disk.mttf.max = 315360000 +disk.mttf.step = interval:10000000 diff --git a/sim/gobs/sweeps/faults_real_range_dl01.cfg b/sim/gobs/sweeps/faults_real_range_dl01.cfg new file mode 100644 index 0000000..f0e0620 --- /dev/null +++ b/sim/gobs/sweeps/faults_real_range_dl01.cfg @@ -0,0 +1,20 @@ + +mttr.reboot = 3600 +mttr.disk = 3600 + +# Disk MTTF: 1-100 years. + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 31536000 +disk.mttf.max = 3153600000 +disk.mttf.step = literal:31536000,31536000000 + +# Artificially short +disk.mttf.step = literal:3153600,3153600000 + +# 3 months: 7776000 +cluster.mttc = 7776000 +san.mttc = 7776000 +target.mttc = 7776000 diff --git a/sim/gobs/sweeps/faults_real_zaps_only.cfg b/sim/gobs/sweeps/faults_real_zaps_only.cfg new file mode 100644 index 0000000..74eab4a --- /dev/null +++ b/sim/gobs/sweeps/faults_real_zaps_only.cfg @@ -0,0 +1,16 @@ + +mttr.reboot = 3600 +mttr.disk = 86400 + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 315360000 +disk.mttf.max = 315360000 +disk.mttf.step = interval:10000000 + +# 3 months: 7776000 +cluster.mttc = 7776000000 +san.mttc = 7776000000 +target.mttc = 7776000000 + diff --git a/sim/gobs/sweeps/faults_small.cfg b/sim/gobs/sweeps/faults_small.cfg new file mode 100644 index 0000000..6451ca3 --- /dev/null +++ b/sim/gobs/sweeps/faults_small.cfg @@ -0,0 +1,7 @@ + +mttr.reboot = 10 +mttr.disk = 50 + +disk.mttf.min = 10000 +disk.mttf.max = 20000 +disk.mttf.step = interval:4000 diff --git a/sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg b/sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg new file mode 100644 index 0000000..50ced8f --- /dev/null +++ b/sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg @@ -0,0 +1,11 @@ + +# FS PDSI 1EB - CHUNKY + +B = 32 + +# Number of files: +files = 20 +# FileFactory implementation PlainFiles +impl.filefactory = PDSIFiles +# 20% size 0, 60% size 1K-100K, 20% size 100K-10M +file.pdsi.points = 0.1,1K;0.1,4T;0.2,10P;1.0,30P diff --git a/sim/gobs/sweeps/fs_pdsi_medium.cfg b/sim/gobs/sweeps/fs_pdsi_medium.cfg index dadaef7..7cc56e9 100644 --- a/sim/gobs/sweeps/fs_pdsi_medium.cfg +++ b/sim/gobs/sweeps/fs_pdsi_medium.cfg @@ -5,4 +5,4 @@ impl.filefactory = PDSIFiles # Number of files: files = 50 # 10% size 0, 90% size 4TB-1PB -file.pdsi.points = 0.1,10G;0.6,100G;1.0,1T +file.pdsi.points = 0.1,10G;0.6,100G;1.0,2T diff --git a/sim/gobs/sweeps/helpers.zsh b/sim/gobs/sweeps/helpers.zsh new file mode 100644 index 0000000..940504d --- /dev/null +++ b/sim/gobs/sweeps/helpers.zsh @@ -0,0 +1,18 @@ + +tformat() +# Convert seconds to hh:mm:ss +{ + local -Z 2 T=$1 + local -Z 2 M + + if (( T <= 60 )) + then + print "${T}" + elif (( T <= 3600 )) + then + M=$(( T/60 )) + print "${M}:$( tformat $(( T%60 )) )" + else + print "$(( T/3600 )):$( tformat $(( T%3600 )) )" + fi +} diff --git a/sim/gobs/sweeps/kpv05.cfg b/sim/gobs/sweeps/kpv05.cfg index 9c0337c..0df9c8b 100644 --- a/sim/gobs/sweeps/kpv05.cfg +++ b/sim/gobs/sweeps/kpv05.cfg @@ -1,8 +1,6 @@ ### Kademlia vs. PAST vs. Vertex Groups 05 -include = kpv_output.cfg - ### Choose a PlacementScheme implementation: # Chosen by sweep driver. diff --git a/sim/gobs/sweeps/rb01-long.cfg b/sim/gobs/sweeps/rb01-long.cfg new file mode 100644 index 0000000..b3ab945 --- /dev/null +++ b/sim/gobs/sweeps/rb01-long.cfg @@ -0,0 +1,41 @@ + +# Simulator configuration + +# Basics: + +include = disks-full-zaps-only.cfg user00.cfg \ + fs_pdsi_1EB_chunky.cfg rb_output.cfg + +systems = target + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 86400 +plot.blackandwhite = true +plot.width = 900 +plot.timeunit = days + +### Parameters for GOBS Simulator run + +### General parameters: + +# Bit-length of addresses: +B = 20 + +# Number of faults: +# 1 month: 2592000 +# 1 month: 31536000 +# 10 years: 315360000 +# M K +simulator.duration = 31536000 +#simulator.duration = 86400 + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/rb01-long.zsh b/sim/gobs/sweeps/rb01-long.zsh new file mode 100755 index 0000000..1cf2593 --- /dev/null +++ b/sim/gobs/sweeps/rb01-long.zsh @@ -0,0 +1,11 @@ +#!/bin/zsh + +# Long run for plot along top + +NAME="rb01-long" +SEED="seed=822" +CLASS=gobs.sweep.x.Rebuild01 + +SWEEPDIR=$( dirname $0 ) + +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/rb01-medium.cfg b/sim/gobs/sweeps/rb01-medium.cfg index 1f4c671..c4e7fd8 100644 --- a/sim/gobs/sweeps/rb01-medium.cfg +++ b/sim/gobs/sweeps/rb01-medium.cfg @@ -8,9 +8,13 @@ include = disks-medium.cfg user00.cfg \ # Statistics: +systems = target + # Rebuild traffic over time analysis: -plot.traffic.interval = 10 +plot.traffic.interval = 3600 plot.blackandwhite = true +plot.width = 900 +plot.timeunit = hours ### Parameters for GOBS Simulator run @@ -20,7 +24,8 @@ plot.blackandwhite = true B = 20 # Number of faults: -simulator.duration = 1000 +# 1 day +simulator.duration = 86400 vertexgroups.prefix = 6 # Replica selection: See Replicated.locateSource() diff --git a/sim/gobs/sweeps/rb02-full.cfg b/sim/gobs/sweeps/rb02-full.cfg new file mode 100644 index 0000000..872fc11 --- /dev/null +++ b/sim/gobs/sweeps/rb02-full.cfg @@ -0,0 +1,32 @@ + +# Simulator configuration + +# Basics: + +include = disks-full.cfg fs_pdsi_1EB_chunky.cfg faults_real_zaps_only.cfg \ + user00.cfg rb_output.cfg + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 3600 +plot.blackandwhite = true +plot.timeunit = hours + +regimen.skip = false + +systems = san;target + +# Bit-length of addresses: +B = 32 + +fault.count = 10 + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/rb02-full.zsh b/sim/gobs/sweeps/rb02-full.zsh new file mode 100755 index 0000000..a4d6e5d --- /dev/null +++ b/sim/gobs/sweeps/rb02-full.zsh @@ -0,0 +1,9 @@ +#!/bin/zsh + +NAME="rb02-full" +SEED="seed=821" +CLASS=gobs.sweep.x.Rebuild02 + +SWEEPDIR=$( dirname $0 ) + +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/rb02-medium.cfg b/sim/gobs/sweeps/rb02-medium.cfg new file mode 100644 index 0000000..4182b82 --- /dev/null +++ b/sim/gobs/sweeps/rb02-medium.cfg @@ -0,0 +1,31 @@ + +# Simulator configuration + +# Basics: + +include = disks-medium-zaps-only.cfg fs_pdsi_medium.cfg \ + user00.cfg rb_output.cfg + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 3456000 +plot.blackandwhite = true +plot.timeunit = hours + +regimen.skip = false + +# Bit-length of addresses: +B = 20 + +fault.count = 10 +# simulator.duration = 86400 + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/rb02-medium.zsh b/sim/gobs/sweeps/rb02-medium.zsh new file mode 100755 index 0000000..1276c7b --- /dev/null +++ b/sim/gobs/sweeps/rb02-medium.zsh @@ -0,0 +1,9 @@ +#!/bin/zsh + +NAME="rb02-medium" +SEED="seed=821" +CLASS=gobs.sweep.x.Rebuild02 + +SWEEPDIR=$( dirname $0 ) + +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/rb02-small.cfg b/sim/gobs/sweeps/rb02-small.cfg index f9159ff..fe850d2 100644 --- a/sim/gobs/sweeps/rb02-small.cfg +++ b/sim/gobs/sweeps/rb02-small.cfg @@ -9,6 +9,7 @@ include = disks-small.cfg user00.cfg fs_pdsi_small.cfg rb_output.cfg # Rebuild traffic over time analysis: plot.traffic.interval = 50 +plot.blackandwhite = true regimen.skip = false @@ -26,4 +27,3 @@ replica.source = primary # impl.nodefactory = RandomNodes impl.nodefactory = SimpleNodes - diff --git a/sim/gobs/sweeps/rb_output.cfg b/sim/gobs/sweeps/rb_output.cfg index a997cd0..26a200a 100644 --- a/sim/gobs/sweeps/rb_output.cfg +++ b/sim/gobs/sweeps/rb_output.cfg @@ -4,11 +4,11 @@ # Print all properties at simulation start print.properties = true # Print nodes and objects at startup (true/false) -print.initialnodes = true +print.initialnodes = false # Print nodes and objects after each rebuild (true/false) -print.nodes = true +print.nodes = false # Print object tranfers during rebuilds (true/false) -print.rebuilds = true +print.rebuilds = false # Print file information (true/false) print.files = false print.filesobjects = false diff --git a/sim/gobs/sweeps/sweep.zsh b/sim/gobs/sweeps/sweep.zsh index 3629f70..d4edc90 100644 --- a/sim/gobs/sweeps/sweep.zsh +++ b/sim/gobs/sweeps/sweep.zsh @@ -5,12 +5,13 @@ GOBS=${SWEEPDIR}/.. source ${SWEEPDIR}/classpath.zsh source ${SWEEPDIR}/build.zsh +source ${SWEEPDIR}/helpers.zsh [[ ${N} == "" ]] && N=1 start=$( date +%s ) { - java -ea -Xmx1000M -cp ${CP} ${CLASS} \ + java -ea -Xmx4000M -cp ${CP} ${CLASS} \ -n ${N} -p sweeps ${NAME}.cfg ${SEED} -o ${NAME}.eps CODE=$? } >& ${NAME}.out @@ -23,4 +24,4 @@ else print "wrote: ${NAME}.out" fi -print "took: $(( stop - start ))" +print "took: $( tformat $(( stop - start )) )" diff --git a/sim/gobs/sweeps/system_100,200.cfg b/sim/gobs/sweeps/system_100,200.cfg new file mode 100644 index 0000000..02bd8cb --- /dev/null +++ b/sim/gobs/sweeps/system_100,200.cfg @@ -0,0 +1,24 @@ + +### System size: servers and files/objects +### 20-1000 servers, selected exponentially +### 3000 files * 10 objects * 3 replicas = 90,000 objects + +# Number of servers: +nodes.min = 100 +nodes.max = 200 +nodes.step = literal:100,200 + +### General parameters: + +# Bit-length of addresses: +B = 32 + +node.osds = 1 +disk.size = 500000000 +disk.speed = 1000000 +disk.mttf = 1 +mttc = 1000 +mttr.reboot = 1 +mttr.disk = 1 +raid = 1+0 + diff --git a/sim/gobs/sweeps/system_100,600.cfg b/sim/gobs/sweeps/system_100,600.cfg index e3af48b..ec6422e 100644 --- a/sim/gobs/sweeps/system_100,600.cfg +++ b/sim/gobs/sweeps/system_100,600.cfg @@ -12,22 +12,6 @@ nodes.step = literal:100,600 # Bit-length of addresses: B = 32 -# Number of files: -files = 100 - -# FileFactory implementation PlainFiles -impl.filefactory = PlainFiles -# File size: -file.size = 14 -# Strip size: -file.strip = 4 -# File/object replicas: -# file.replicas = 3 -# Number of objects per file: -file.width = 10 - -mttf = 1 -mttr = 1 node.osds = 1 disk.size = 500000000 diff --git a/sim/gobs/sweeps/system_100-600.cfg b/sim/gobs/sweeps/system_100-600.cfg index e6c2192..3d33389 100644 --- a/sim/gobs/sweeps/system_100-600.cfg +++ b/sim/gobs/sweeps/system_100-600.cfg @@ -12,22 +12,6 @@ nodes.step = exps # Bit-length of addresses: B = 32 -# Number of files: -files = 100 - -# FileFactory implementation PlainFiles -impl.filefactory = PlainFiles -# File size: -file.size = 14 -# Strip size: -file.strip = 4 -# File/object replicas: -# file.replicas = 3 -# Number of objects per file: -file.width = 10 - -mttf = 1 -mttr = 1 node.osds = 1 disk.size = 500000000 diff --git a/sim/gobs/sweeps/system_100-700.cfg b/sim/gobs/sweeps/system_100-700.cfg new file mode 100644 index 0000000..97ab861 --- /dev/null +++ b/sim/gobs/sweeps/system_100-700.cfg @@ -0,0 +1,23 @@ + +### System size: servers and files/objects +### 20-1000 servers, selected exponentially +### 3000 files * 10 objects * 3 replicas = 90,000 objects + +# Number of servers: +nodes.min = 100 +nodes.max = 700 +nodes.step = exps + +### General parameters: + +# Bit-length of addresses: +B = 32 + +node.osds = 1 +disk.size = 500000000 +disk.speed = 1000000 +disk.mttf = 1 +mttc = 1000 +mttr.reboot = 1 +mttr.disk = 1 +raid = 1+0 diff --git a/sim/gobs/sweeps/system_600.cfg b/sim/gobs/sweeps/system_600.cfg index c476879..0217462 100644 --- a/sim/gobs/sweeps/system_600.cfg +++ b/sim/gobs/sweeps/system_600.cfg @@ -4,7 +4,7 @@ ### 3000 files * 10 objects * 3 replicas = 90,000 objects # Number of servers: -nodes.min = 100 +nodes.min = 600 nodes.max = 600 nodes.step = literal:600 @@ -26,9 +26,6 @@ file.strip = 4 # Number of objects per file: file.width = 10 -mttf = 1 -mttr = 1 - node.osds = 1 disk.size = 500000000 disk.speed = 1000000 @@ -37,4 +34,3 @@ mttc = 1000 mttr.reboot = 1 mttr.disk = 1 raid = 1+0 -fault.count = 1 hooks/post-receive -- Triton Repository
participants (1)
-
noreply＠mcs.anl.gov