- Gs-commits - mailman3.cels.anl.gov

Triton Repository branch, master, updated. c9a50f0ec4e7969c726cace458bf954376e2a583
by noreply＠mcs.anl.gov 01 Apr '10

01 Apr '10

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via c9a50f0ec4e7969c726cace458bf954376e2a583 (commit) from 2f58f0721fd0eb023ac697593338f8a82e0b76e3 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit c9a50f0ec4e7969c726cace458bf954376e2a583 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Thu Apr 1 11:22:52 2010 -0500 Version of everything submitted to DSN FTXS. ----------------------------------------------------------------------- Summary of changes: sim/gobs/src/gobs/plot/MultiplePlot.java | 12 +++ sim/gobs/src/gobs/sim/Copy.java | 1 + sim/gobs/src/gobs/sim/Metric.java | 14 ++-- sim/gobs/src/gobs/sim/Node.java | 9 ++- sim/gobs/src/gobs/sim/PDSIFiles.java | 4 +- sim/gobs/src/gobs/sim/Placement.java | 2 + sim/gobs/src/gobs/sim/RAID.java | 14 +++ sim/gobs/src/gobs/sim/Rebuild.java | 45 +++++++++- sim/gobs/src/gobs/sim/Simulator.java | 63 +++++++++++--- sim/gobs/src/gobs/sweep/AbstractSweep.java | 77 ++++++++++++++++- sim/gobs/src/gobs/sweep/Helpers.java | 2 +- sim/gobs/src/gobs/sweep/x/DataLoss01.java | 11 ++- sim/gobs/src/gobs/sweep/x/DataLoss03.java | 131 ++++++++++++++++++++++++++++ sim/gobs/src/gobs/sweep/x/DataLoss04.java | 131 ++++++++++++++++++++++++++++ sim/gobs/src/gobs/sweep/x/Decluster04.java | 3 +- sim/gobs/src/gobs/sweep/x/Rebuild01.java | 15 +--- sim/gobs/src/gobs/sweep/x/Rebuild02.java | 5 +- sim/gobs/sweeps/dc03kpv01.cfg | 1 + sim/gobs/sweeps/dc03kpv03.cfg | 2 + sim/gobs/sweeps/dc03kpv05.cfg | 2 + sim/gobs/sweeps/dc04kpv01.cfg | 1 + sim/gobs/sweeps/dc_output.cfg | 2 +- sim/gobs/sweeps/disks-full.cfg | 1 - sim/gobs/sweeps/disks-small.cfg | 3 - sim/gobs/sweeps/dl01-full.cfg | 7 +- sim/gobs/sweeps/dl03-full.cfg | 39 ++++++++ sim/gobs/sweeps/dl03-full.zsh | 9 ++ sim/gobs/sweeps/dl04-full.cfg | 36 ++++++++ sim/gobs/sweeps/dl04-full.zsh | 11 +++ sim/gobs/sweeps/faults_10.cfg | 16 ++++ sim/gobs/sweeps/faults_real_range_dl01.cfg | 25 +++--- sim/gobs/sweeps/faults_real_range_dl03.cfg | 24 +++++ sim/gobs/sweeps/faults_real_zaps_only.cfg | 4 +- sim/gobs/sweeps/faults_small.cfg | 2 +- sim/gobs/sweeps/faults_small_zaps_only.cfg | 11 +++ sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg | 4 +- sim/gobs/sweeps/fs_pdsi_1EB_vchunky.cfg | 11 +++ sim/gobs/sweeps/fs_pdsi_small.cfg | 2 +- sim/gobs/sweeps/rb01-long.cfg | 6 +- sim/gobs/sweeps/rb02-full.cfg | 6 +- sim/gobs/sweeps/rb02-full.zsh | 2 +- sim/gobs/sweeps/rb02-small.cfg | 8 ++- sim/gobs/sweeps/rb02-small.zsh | 2 +- sim/gobs/sweeps/rb_output.cfg | 6 +- sim/gobs/sweeps/sweep.zsh | 2 + 45 files changed, 695 insertions(+), 89 deletions(-) create mode 100644 sim/gobs/src/gobs/sweep/x/DataLoss03.java create mode 100644 sim/gobs/src/gobs/sweep/x/DataLoss04.java create mode 100644 sim/gobs/sweeps/dl03-full.cfg create mode 100755 sim/gobs/sweeps/dl03-full.zsh create mode 100644 sim/gobs/sweeps/dl04-full.cfg create mode 100755 sim/gobs/sweeps/dl04-full.zsh create mode 100644 sim/gobs/sweeps/faults_10.cfg create mode 100644 sim/gobs/sweeps/faults_real_range_dl03.cfg create mode 100644 sim/gobs/sweeps/faults_small_zaps_only.cfg create mode 100644 sim/gobs/sweeps/fs_pdsi_1EB_vchunky.cfg Diff of changes: diff --git a/sim/gobs/src/gobs/plot/MultiplePlot.java b/sim/gobs/src/gobs/plot/MultiplePlot.java index 084b50c..bd6c7e1 100644 --- a/sim/gobs/src/gobs/plot/MultiplePlot.java +++ b/sim/gobs/src/gobs/plot/MultiplePlot.java @@ -14,9 +14,11 @@ import org.apache.xmlgraphics.java2d.ps.EPSDocumentGraphics2D; import org.jfree.chart.ChartFactory; import org.jfree.chart.JFreeChart; +import org.jfree.chart.axis.ValueAxis; import org.jfree.chart.plot.PlotOrientation; import org.jfree.chart.plot.XYPlot; import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer; +import org.jfree.data.Range; import org.jfree.data.xy.XYSeries; import org.jfree.data.xy.XYSeriesCollection; @@ -80,6 +82,16 @@ public class MultiplePlot renderer.setSeriesPaint(i, Color.BLACK); plot.setRenderer(renderer); plot.setBackgroundPaint(Color.WHITE); + if (forceDomainUpper) + { + ValueAxis axis = plot.getDomainAxis(); + Range range = axis.getRange(); + double lower = range.getLowerBound(); + Range adjusted = new Range(lower, forceDomainUpperValue); + axis.setRange(adjusted); + System.out.println("range: " + axis.getRange()); + plot.setDomainAxis(axis); + } chart.draw(g2d, rectangle); try diff --git a/sim/gobs/src/gobs/sim/Copy.java b/sim/gobs/src/gobs/sim/Copy.java index bf1b14e..d7d0688 100644 --- a/sim/gobs/src/gobs/sim/Copy.java +++ b/sim/gobs/src/gobs/sim/Copy.java @@ -67,6 +67,7 @@ class Copy { StringBuilder sb = new StringBuilder(128); sb.append("copy: ") + .append(cause).append(" ") .append(object).append(": ") .append(source).append(" -> ").append(destination) .append(" (").append(df.format(start)); diff --git a/sim/gobs/src/gobs/sim/Metric.java b/sim/gobs/src/gobs/sim/Metric.java index c834137..75bc92d 100644 --- a/sim/gobs/src/gobs/sim/Metric.java +++ b/sim/gobs/src/gobs/sim/Metric.java @@ -56,11 +56,6 @@ public abstract class Metric */ Copy regimen(SingleObject object, List<Node> system) { - if (simulator.enabled("regimen.skip")) - { - simulator.print("rebuilds", "regimen: skip"); - return null; - } simulator.print("rebuilds", "regimen: " + object); Sites sites = sites(object, system); // The node that should have the primary in the current system @@ -83,6 +78,12 @@ public abstract class Metric return null; } + if (simulator.enabled("regimen.skip")) + { + simulator.print("rebuilds", "regimen: skip"); + return null; + } + Copy result = regimen(object, primaryNode, primaryLocation, source, sites); // assert (result != null); @@ -199,9 +200,6 @@ public abstract class Metric if (copy == null) return; - copy.source.pushes++; - copy.destination.pulls++; - printRebuild(copy.object, copy.source, copy.destination); } } diff --git a/sim/gobs/src/gobs/sim/Node.java b/sim/gobs/src/gobs/sim/Node.java index 8a9b0ee..753d36b 100644 --- a/sim/gobs/src/gobs/sim/Node.java +++ b/sim/gobs/src/gobs/sim/Node.java @@ -279,12 +279,17 @@ public class Node } /** - Start using this Node as a Copy source. + Stop using this Node as a Copy source. */ void stopStreamFrom(Copy copy) { int i = lookup(copy.object.id); - assert(i >= 0); + if (i < 0) + { + System.out.println(displayObjects()); + Tools.bail("stopStreamFrom(): " + copy.object + " => " + i + "\n" + + "nodes left: " + simulator.nodes.size()); + } assert(osd[i].load == OSD.Load.BUSY); osd[i].load = OSD.Load.IDLE; } diff --git a/sim/gobs/src/gobs/sim/PDSIFiles.java b/sim/gobs/src/gobs/sim/PDSIFiles.java index 4c9a065..1c8d987 100644 --- a/sim/gobs/src/gobs/sim/PDSIFiles.java +++ b/sim/gobs/src/gobs/sim/PDSIFiles.java @@ -70,8 +70,8 @@ public class PDSIFiles nodes = Tools.integer(properties, "node.count"); if (Tools.has(properties, "node.osds")) disksPerNode = Tools.integer(properties, "node.osds"); - if (Tools.has(properties, "replicas")) - presetReplicas = Tools.integer(properties, "replicas"); + if (Tools.has(properties, "file.replicas")) + presetReplicas = Tools.integer(properties, "file.replicas"); interpolate(); } diff --git a/sim/gobs/src/gobs/sim/Placement.java b/sim/gobs/src/gobs/sim/Placement.java index d47fea6..7aee9e3 100644 --- a/sim/gobs/src/gobs/sim/Placement.java +++ b/sim/gobs/src/gobs/sim/Placement.java @@ -93,6 +93,8 @@ public abstract class Placement simulator.printNodeObjects(simulator.nodes); simulator.print("rebuilds", "perform: " + copy); copy.destination.add(copy.object); + copy.source.pushes++; + copy.destination.pulls++; } /** diff --git a/sim/gobs/src/gobs/sim/RAID.java b/sim/gobs/src/gobs/sim/RAID.java index 6f38770..ee3ba24 100755 --- a/sim/gobs/src/gobs/sim/RAID.java +++ b/sim/gobs/src/gobs/sim/RAID.java @@ -130,4 +130,18 @@ public class RAID { return spec; } + + public static void main(String[] args) + { + double tb30 = 30.0 * Math.pow(10,12); + double mb400 = 400.0 * Math.pow(10, 6); + double day = 24*60*60; + double yr = 365*day; + RAID raid = new RAID(args[0], tb30, mb400); + double yrs = Double.parseDouble(args[1]); + double secs = yrs*yr; + double mttf = raid.computeMTTF(secs); + System.out.println("MTTF: " + mttf/yr); + System.out.println("Disk: " + tb30/mb400/day); + } } diff --git a/sim/gobs/src/gobs/sim/Rebuild.java b/sim/gobs/src/gobs/sim/Rebuild.java index b1518e1..fbda591 100644 --- a/sim/gobs/src/gobs/sim/Rebuild.java +++ b/sim/gobs/src/gobs/sim/Rebuild.java @@ -7,7 +7,7 @@ import gobs.util.*; /** Manages a rebuild after a fault. - Is only executed ({@link #rebuild}) once. Thus rebuild variables + Is only executed {@link rebuild} once. Thus rebuild variables can logically be declared at class level. */ public class Rebuild @@ -257,7 +257,10 @@ public class Rebuild { Copy copy = simulator.placer.regimen(object); if (copy != null) + { + copy.cause = Cause.VALIDATION; result.add(copy); + } } return result; } @@ -284,15 +287,53 @@ public class Rebuild break; if (! sites.contains(node)) { + simulator.print("rebuilds", "removing extra: " + object + - " on: " + node); + " on: " + node + "\n" + + "holders: " + holders + "\n" + + "sites: " + sites); + failCopiesExtra(object, node); node.delete(object.id); hcount--; } } } + void failCopiesExtra(Gob object, Node source) + { + for (Iterator<Copy> it = simulator.todo.iterator(); it.hasNext(); ) + { + Copy copy = it.next(); + if (copy.object.equals(object) && + (copy.source.equals(source) || + copy.destination.equals(source))) + { + simulator.print("rebuilds", "stop copy from extra (todo): " + + copy); + it.remove(); + } + } + + for (Iterator<Event> it = simulator.inflight.iterator(); it.hasNext(); ) + { + Event event = it.next(); + if (! (event instanceof Copy)) + continue; + Copy copy = (Copy) event; + if (copy.object.equals(object) && + (copy.source.equals(source) || + copy.destination.equals(source))) + { + simulator.print("rebuilds", + "stop copy from extra (inflight): " + copy); + copy.source.stopStreamFrom(copy); + copy.destination.stopStreamTo(copy); + it.remove(); + } + } + } + void completeRepair(Repair repair) { /* if (simulator.bool("print.rebuilds")) diff --git a/sim/gobs/src/gobs/sim/Simulator.java b/sim/gobs/src/gobs/sim/Simulator.java index ed6f2ac..6428c79 100644 --- a/sim/gobs/src/gobs/sim/Simulator.java +++ b/sim/gobs/src/gobs/sim/Simulator.java @@ -12,7 +12,7 @@ import gobs.util.*; import static gobs.util.Tools.bail; /** - * <h3> GOBS placement simulator. </h3> + * GOBS placement simulator. * * Command-line arguments: {@link #main} * */ @@ -124,7 +124,7 @@ public class Simulator /** Copies that the Simulator will perform in the future. */ - Set<Copy> todo = new LinkedHashSet<Copy>(); + NavigableSet<Copy> todo = new TreeSet<Copy>(); /** The Set of known Events that will finish in the future. @@ -138,6 +138,11 @@ public class Simulator Set<BigInteger> lost = new HashSet<BigInteger>(); /** + Points in simulated time in which an object was lost: + */ + Set<Double> lossEvents = new HashSet<Double>(); + + /** Nodes that are temporarily down. */ Set<Node> down = new LinkedHashSet<Node>(); @@ -865,24 +870,43 @@ public class Simulator */ void performCopies(Fault fault, Set<? extends Event> set) { + List<Event> done = new ArrayList<Event>(); + + while (true) + { + boolean found = false; + Copy copy = extractCopy(set); + if (copy == null) + break; + if (copy.destination.contains(copy.object.id)) + { + System.out.println("unnecessary: " + copy); + continue; + } + List<Copy> extras = fault.rebuild.performCopy(copy); + for (Copy extra : extras) + print("rebuilds", "extra: " + extra); + done.add(copy); + } + for (Event event : done) + set.remove(event); + } + + Copy extractCopy(Set<? extends Event> set) + { + Copy result = null; Iterator<? extends Event> it = set.iterator(); while (it.hasNext()) { Event event = it.next(); if (event instanceof Copy) { - Copy copy = (Copy) event; - if (copy.destination.contains(copy.object.id)) - { - System.out.println("unnecessary: " + copy); - continue; - } - List<Copy> extras = fault.rebuild.performCopy(copy); - for (Copy extra : extras) - print("rebuilds", "extra: " + extra); + result = (Copy) event; it.remove(); + break; } } + return result; } /** @@ -974,11 +998,11 @@ public class Simulator computeTraffic(); computeLosses(); + System.out.println("FAULTS: " + faultCount); System.out.println("FINAL_NODES: " + nodes.size()); System.out.println("FLEXIBILITIES: " + df.format(stats.avgFlexibility())); System.out.println("FINAL_OBJECTS: " + countObjects()); - System.out.println("SIMULATED_TIME: " + Time.toString(time)); stats.results.setProperty("time", df.format(time)); @@ -990,10 +1014,12 @@ public class Simulator for (Rebuild rebuild : rebuilds) if (rebuild.dataLost) count++; + int events = lossEvents.size(); System.out.println("LOST_OBJECTS: " + lost.size()); stats.results.setProperty("lost", ff.format(lost.size())); stats.results.setProperty("failures", ff.format(count)); + stats.results.setProperty("lossevents", ff.format(events)); } /** @@ -1248,6 +1274,7 @@ public class Simulator { System.out.println("LOST: " + object.toString(this)); lost.add(object.id); + lossEvents.add(new Double(time)); currentRebuild.dataLost = true; } } @@ -1273,16 +1300,28 @@ public class Simulator void fsck() { print("FSCK:"); + int missing = 0; for (File file : files) for (Gob object : file.objects) { if (lost.contains(object.id)) + { + missing++; continue; + } Sites sites = placer.sites(object); + boolean found = false; for (Node node : sites.get()) + { if (! node.contains(object.id)) print("OBJECT_MISSING: " + object); + else + found = true; + } + if (!found) + missing++; } + stats.results.setProperty("fsckMissing", ""+missing); } void strays() diff --git a/sim/gobs/src/gobs/sweep/AbstractSweep.java b/sim/gobs/src/gobs/sweep/AbstractSweep.java index 682b4e1..d7671c0 100644 --- a/sim/gobs/src/gobs/sweep/AbstractSweep.java +++ b/sim/gobs/src/gobs/sweep/AbstractSweep.java @@ -3,6 +3,8 @@ package gobs.sweep; import java.util.*; +import org.jfree.data.xy.XYSeries; + import gobs.sim.*; import gobs.util.*; @@ -267,12 +269,13 @@ public abstract class AbstractSweep return all(min,max); else if (step.equals("tens")) return tens(min,max); - else if (step.equals("exps")) + else + */ + if (step.equals("exps")) return exps(min,max); else if (step.equals("mids")) return mids(min,max); - else */ - if (step.startsWith("interval")) + else if (step.startsWith("interval")) return interval(step, min, max); else if (step.startsWith("literal")) return literal(step); @@ -282,6 +285,61 @@ public abstract class AbstractSweep } /** + Sweep over all integer values between min and max that + are the product of min and 2^i for some i. + */ + public static List<String> exps(double min, double max) + { + List<String> result = new ArrayList<String>(); + + double p = 1.0; + double v = min; + + while (v*p < max) + { + result.add(new Double(v*p).toString()); + p *= 2; + } + result.add(new Double(max).toString()); + + return result; + } + + /** + Sweep over all integer values between min and max that + are the product of min and 2^i for some i, and the values + between those. + */ + public static List<String> mids(double min, double max) + { + List<String> result = new ArrayList<String>(); + + double v = min; + double p = 1; + double previous = -1; + + double vp = 0; + while ((vp = v*p) < max) + { + if (previous > 0) + { + double m = (vp+previous) / 2; + if (m != vp) + result.add(new Double(m).toString()); + } + result.add(new Double(vp).toString()); + previous = vp; + p *= 2; + } + double m = (max+previous) / 2; + if (m != previous) + result.add(new Double(m).toString()); + result.add(new Double(max).toString()); + + return result; + } + + /** Sweep over all values between min and max stepping over the interval argument. @param arg The argument, e.g., "interval:4". @@ -333,6 +391,19 @@ public abstract class AbstractSweep this.properties = getopts(args); } + public static XYSeries scaleX(XYSeries series, double scale) + { + XYSeries result = new XYSeries(series.getKey()); + for (int i = 0; i < series.getItemCount(); i++) + { + Number x = series.getX(i); + Number y = series.getY(i); + double dx = x.doubleValue()/scale; + result.add(dx,y); + } + return result; + } + static Set<String> values(Collection<Simulator> list, String name) { Set<String> result = new HashSet<String>(); diff --git a/sim/gobs/src/gobs/sweep/Helpers.java b/sim/gobs/src/gobs/sweep/Helpers.java index 418d975..bb3972a 100755 --- a/sim/gobs/src/gobs/sweep/Helpers.java +++ b/sim/gobs/src/gobs/sweep/Helpers.java @@ -35,7 +35,7 @@ public class Helpers properties.setProperty("node.count", ""+nodes); properties.setProperty("node.osds", ""+nodeDevices); int replicas = Tools.integer(properties, system+".replicas"); - properties.setProperty("replicas", ""+replicas); + properties.setProperty("file.replicas", ""+replicas); return raid; } } diff --git a/sim/gobs/src/gobs/sweep/x/DataLoss01.java b/sim/gobs/src/gobs/sweep/x/DataLoss01.java index 043061b..fd37159 100644 --- a/sim/gobs/src/gobs/sweep/x/DataLoss01.java +++ b/sim/gobs/src/gobs/sweep/x/DataLoss01.java @@ -15,7 +15,7 @@ import gobs.plot.*; * Modifies MTTF. * Reports number of objects lost for varying MTTF * - * Usage: DataLoss01 -n <repeats> -o <output.eps> *.cfg + * Usage: DataLoss01 -n repeats -o output.eps *.cfg * */ class DataLoss01 @@ -43,6 +43,8 @@ class DataLoss01 scale = 3600; else if (timeunit.equals("days")) scale = 86400; + else if (timeunit.equals("years")) + scale = 365*86400; } List<String> valuesMTTF = Helpers.valuesMTTF(properties); @@ -70,7 +72,10 @@ class DataLoss01 } for (XYSeries series : allSeries) - collection.addSeries(series); + { + XYSeries scaled = AbstractSweep.scaleX(series, scale); + collection.addSeries(scaled); + } MultiplePlot.configure(properties); MultiplePlot.plot(collection, "", @@ -101,7 +106,7 @@ class DataLoss01 lost += Tools.integer(simulator.stats.results, "lost"); total++; } - series.add(Long.parseLong(mttf), lost); + series.add(Double.parseDouble(mttf), lost); } results.add(series); diff --git a/sim/gobs/src/gobs/sweep/x/DataLoss03.java b/sim/gobs/src/gobs/sweep/x/DataLoss03.java new file mode 100644 index 0000000..15040a0 --- /dev/null +++ b/sim/gobs/src/gobs/sweep/x/DataLoss03.java @@ -0,0 +1,131 @@ + +package gobs.sweep.x; + +import java.math.BigInteger; +import java.text.DecimalFormat; +import java.util.*; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.sim.*; +import gobs.sweep.*; +import gobs.util.*; +import gobs.plot.*; + +/** + Loss event rates. +*/ + +public class DataLoss03 +{ + static Properties properties; + + static DecimalFormat df = new DecimalFormat("0.0"); + + static double duration; + static double scale; + + public static void main(String[] args) + { + properties = Sweep.getopts(args); + + AbstractSweep.seed(properties); + + String output = Tools.string(properties, "output"); + + List<String> systems = Tools.list(properties, "systems"); + + String timeunit = "seconds"; + scale = 1; + if (Tools.has(properties, "plot.timeunit")) + { + timeunit = Tools.string(properties, "plot.timeunit"); + if (timeunit.equals("minutes")) + scale = 60; + else if (timeunit.equals("hours")) + scale = 3600; + else if (timeunit.equals("days")) + scale = 86400; + else if (timeunit.equals("years")) + scale = 365*86400; + } + + duration = Tools.decimal(properties, "simulator.duration"); + + List<String> valuesMTTF = Helpers.valuesMTTF(properties); + + System.out.println("DataLoss01: systems: " + systems + "\n" + + " mttf: " + valuesMTTF); + + final XYSeriesCollection collection = new XYSeriesCollection(); + List<XYSeries> allSeries = new ArrayList<XYSeries>(); + + properties.setProperty("impl.placement", "Nearest"); + for (String skip : new String[] { "false", "true" }) + { + for (String system : systems) + { + properties.setProperty("system", system); + Helpers.setupRAID(properties, system); + String mttc = Tools.string(properties, system+".mttc"); + properties.setProperty("mttc", mttc); + properties.setProperty("regimen.skip", skip); + Sweep pSweep = new Sweep(properties); + pSweep.repeats = Tools.integer(properties, "repeats"); + pSweep.execute("disk.mttf", valuesMTTF); + + String t = skip.equals("true") ? "latent" : "active"; + List<XYSeries> pSeries = + plots(pSweep, "NEAREST-"+system+"-"+t, valuesMTTF); + allSeries.addAll(pSeries); + } + } + + for (XYSeries series : allSeries) + { + XYSeries scaled = AbstractSweep.scaleX(series, scale); + collection.addSeries(scaled); + } + + MultiplePlot.configure(properties); + MultiplePlot.plot(collection, "", + "Mean Time to Disk Failure (years)", + "Loss Events per Year", output); + } + + /** + Plot average number of lost objects for each MTTF. + @param label E.g., "PAST" + */ + static List<XYSeries> plots(Sweep sweep, String label, + List<String> valuesMTTF) + { + List<XYSeries> results = new ArrayList<XYSeries>(); + + XYSeries series = new XYSeries(label); + for (String mttf : valuesMTTF) + { + double time = 0.0; + double lost = 0; + double total = 0.0; + for (Simulator simulator : sweep.sims.values()) + if (simulator.string("disk.mttf").equals(mttf)) + { + System.out.println(simulator.string("tag")); + time += Tools.decimal(simulator.stats.results, "time"); + lost += + Tools.integer(simulator.stats.results, "lossevents"); + total++; + } + double lostRate = (lost/total)*(scale/duration); + System.out.println("lost: " + lost + " total: " + total + + " scale: " + scale + " duration: " + duration); + + series.add(Double.parseDouble(mttf), lostRate); + } + results.add(series); + + return results; + } +} diff --git a/sim/gobs/src/gobs/sweep/x/DataLoss04.java b/sim/gobs/src/gobs/sweep/x/DataLoss04.java new file mode 100644 index 0000000..6851a90 --- /dev/null +++ b/sim/gobs/src/gobs/sweep/x/DataLoss04.java @@ -0,0 +1,131 @@ + +package gobs.sweep.x; + +import java.math.BigInteger; +import java.text.DecimalFormat; +import java.util.*; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.sim.*; +import gobs.sweep.*; +import gobs.util.*; +import gobs.plot.*; + +/** + Lost object count based on fsck(). +*/ + +public class DataLoss04 +{ + static Properties properties; + + static DecimalFormat df = new DecimalFormat("0.0"); + + static double duration; + static double scale; + + public static void main(String[] args) + { + properties = Sweep.getopts(args); + + AbstractSweep.seed(properties); + + String output = Tools.string(properties, "output"); + + List<String> systems = Tools.list(properties, "systems"); + + String timeunit = "seconds"; + scale = 1; + if (Tools.has(properties, "plot.timeunit")) + { + timeunit = Tools.string(properties, "plot.timeunit"); + if (timeunit.equals("minutes")) + scale = 60; + else if (timeunit.equals("hours")) + scale = 3600; + else if (timeunit.equals("days")) + scale = 86400; + else if (timeunit.equals("years")) + scale = 365*86400; + } + + duration = Tools.decimal(properties, "simulator.duration"); + + List<String> valuesMTTF = Helpers.valuesMTTF(properties); + + System.out.println("DataLoss01: systems: " + systems + "\n" + + " mttf: " + valuesMTTF); + + final XYSeriesCollection collection = new XYSeriesCollection(); + List<XYSeries> allSeries = new ArrayList<XYSeries>(); + + properties.setProperty("impl.placement", "Nearest"); + for (String skip : new String[] { "false", "true" }) + { + for (String system : systems) + { + properties.setProperty("system", system); + Helpers.setupRAID(properties, system); + String mttc = Tools.string(properties, system+".mttc"); + properties.setProperty("mttc", mttc); + properties.setProperty("regimen.skip", skip); + Sweep pSweep = new Sweep(properties); + pSweep.repeats = Tools.integer(properties, "repeats"); + pSweep.execute("disk.mttf", valuesMTTF); + + String t = skip.equals("true") ? "latent" : "active"; + List<XYSeries> pSeries = + plots(pSweep, "NEAREST-"+system+"-"+t, valuesMTTF); + allSeries.addAll(pSeries); + } + } + + for (XYSeries series : allSeries) + { + XYSeries scaled = AbstractSweep.scaleX(series, scale); + collection.addSeries(scaled); + } + + MultiplePlot.configure(properties); + MultiplePlot.plot(collection, "", + "Mean Time to Disk Failure (years)", + "Objects Lost per Year", output); + } + + /** + Plot average number of lost objects for each MTTF. + @param label E.g., "PAST" + */ + static List<XYSeries> plots(Sweep sweep, String label, + List<String> valuesMTTF) + { + List<XYSeries> results = new ArrayList<XYSeries>(); + + XYSeries series = new XYSeries(label); + for (String mttf : valuesMTTF) + { + double time = 0.0; + double lost = 0; + double total = 0.0; + for (Simulator simulator : sweep.sims.values()) + if (simulator.string("disk.mttf").equals(mttf)) + { + System.out.println(simulator.string("tag")); + time += Tools.decimal(simulator.stats.results, "time"); + lost += + Tools.integer(simulator.stats.results, "fsckMissing"); + total++; + } + double lostRate = (lost/total)*(scale/duration); + System.out.println("lost: " + lost + " total: " + total + + " scale: " + scale + " duration: " + duration); + + series.add(Double.parseDouble(mttf), lostRate); + } + results.add(series); + + return results; + } +} diff --git a/sim/gobs/src/gobs/sweep/x/Decluster04.java b/sim/gobs/src/gobs/sweep/x/Decluster04.java index fe6be91..074fcd3 100644 --- a/sim/gobs/src/gobs/sweep/x/Decluster04.java +++ b/sim/gobs/src/gobs/sweep/x/Decluster04.java @@ -73,8 +73,7 @@ class Decluster04 collection.addSeries(s); } - if (Tools.enabled(properties, "plot.blackandwhite")) - MultiplePlot.bw = true; + MultiplePlot.configure(properties); MultiplePlot.plot(collection, "", "nodes involved", "load fraction(%)", output); } diff --git a/sim/gobs/src/gobs/sweep/x/Rebuild01.java b/sim/gobs/src/gobs/sweep/x/Rebuild01.java index dbc4e3b..4dfb355 100644 --- a/sim/gobs/src/gobs/sweep/x/Rebuild01.java +++ b/sim/gobs/src/gobs/sweep/x/Rebuild01.java @@ -70,7 +70,7 @@ class Rebuild01 for (XYSeries series : allSeries) { - XYSeries scaled = scaleX(series, scale); + XYSeries scaled = AbstractSweep.scaleX(series, scale); collection.addSeries(scaled); } @@ -79,17 +79,4 @@ class Rebuild01 "time ("+timeunit+")", "copies in flight", output); } - - static XYSeries scaleX(XYSeries series, double scale) - { - XYSeries result = new XYSeries(series.getKey()); - for (int i = 0; i < series.getItemCount(); i++) - { - Number x = series.getX(i); - Number y = series.getY(i); - double dx = x.doubleValue()/scale; - result.add(dx,y); - } - return result; - } } diff --git a/sim/gobs/src/gobs/sweep/x/Rebuild02.java b/sim/gobs/src/gobs/sweep/x/Rebuild02.java index a4fd2c7..bf3ac91 100644 --- a/sim/gobs/src/gobs/sweep/x/Rebuild02.java +++ b/sim/gobs/src/gobs/sweep/x/Rebuild02.java @@ -75,7 +75,7 @@ class Rebuild02 String mttc = Tools.string(properties, system+".mttc"); properties.setProperty("mttc", mttc); properties.setProperty("raid", spec); - properties.setProperty("node.count", ""+nodes); + properties.setProperty("node.count", ""+nodes); properties.setProperty("node.osds", ""+nodeDevices); properties.setProperty("regimen.skip", skip); properties.setProperty("system", system); @@ -98,7 +98,6 @@ class Rebuild02 for (XYSeries series : allSeries) collection.addSeries(series); - MultiplePlot.configure(properties); MultiplePlot.plot(collection, "", "time ("+timeunit+")", "copies in flight", @@ -162,7 +161,7 @@ class Rebuild02 series.add(d/scale, sample); d += interval; } - series.add(d, 0.0); + series.add(d/scale, 0.0); } static double longestRebuild(List<Rebuild> rebuilds) diff --git a/sim/gobs/sweeps/dc03kpv01.cfg b/sim/gobs/sweeps/dc03kpv01.cfg index 10c086a..d6248fb 100644 --- a/sim/gobs/sweeps/dc03kpv01.cfg +++ b/sim/gobs/sweeps/dc03kpv01.cfg @@ -2,6 +2,7 @@ include = kpv01.cfg system_100-700.cfg fs_pdsi_1EB.cfg \ dc_output.cfg user00.cfg +plot.blackandwhite = true plot.force.domain.upper = 1100 fault.count = 1 diff --git a/sim/gobs/sweeps/dc03kpv03.cfg b/sim/gobs/sweeps/dc03kpv03.cfg index c5d72e3..1793894 100644 --- a/sim/gobs/sweeps/dc03kpv03.cfg +++ b/sim/gobs/sweeps/dc03kpv03.cfg @@ -2,4 +2,6 @@ include = kpv03.cfg fs_pdsi_1EB.cfg system_100-700.cfg \ dc_output.cfg user00.cfg +plot.blackandwhite = true plot.force.domain.upper = 1100 +fault.count = 1 diff --git a/sim/gobs/sweeps/dc03kpv05.cfg b/sim/gobs/sweeps/dc03kpv05.cfg index abdfb49..bec2338 100644 --- a/sim/gobs/sweeps/dc03kpv05.cfg +++ b/sim/gobs/sweeps/dc03kpv05.cfg @@ -2,4 +2,6 @@ include = kpv05.cfg system_100-700.cfg fs_pdsi_1EB.cfg \ dc_output.cfg user00.cfg +plot.blackandwhite = true plot.force.domain.upper = 1100 +fault.count = 1 diff --git a/sim/gobs/sweeps/dc04kpv01.cfg b/sim/gobs/sweeps/dc04kpv01.cfg index 1b6f9bd..42ec269 100644 --- a/sim/gobs/sweeps/dc04kpv01.cfg +++ b/sim/gobs/sweeps/dc04kpv01.cfg @@ -2,4 +2,5 @@ include = kpv01.cfg system_600.cfg fs_pdsi_1EB.cfg \ dc_output.cfg user00.cfg +plot.blackandwhite = true fault.count = 1 diff --git a/sim/gobs/sweeps/dc_output.cfg b/sim/gobs/sweeps/dc_output.cfg index f26b687..b134390 100644 --- a/sim/gobs/sweeps/dc_output.cfg +++ b/sim/gobs/sweeps/dc_output.cfg @@ -4,7 +4,7 @@ # Print all properties at simulation start print.properties = true # Print nodes and objects at startup (true/false) -print.initialnodes = false +print.initialnodes = true # Print nodes and objects after each rebuild (true/false) print.nodes = false # Print object tranfers during rebuilds (true/false) diff --git a/sim/gobs/sweeps/disks-full.cfg b/sim/gobs/sweeps/disks-full.cfg index fe2f5ce..84741a6 100644 --- a/sim/gobs/sweeps/disks-full.cfg +++ b/sim/gobs/sweeps/disks-full.cfg @@ -19,4 +19,3 @@ san.replicas = 2 target.node.osds = 4 target.raid = 4+1 target.replicas = 3 - diff --git a/sim/gobs/sweeps/disks-small.cfg b/sim/gobs/sweeps/disks-small.cfg index 56b17c0..4ba91e4 100644 --- a/sim/gobs/sweeps/disks-small.cfg +++ b/sim/gobs/sweeps/disks-small.cfg @@ -6,16 +6,13 @@ disk.speed = 3000 cluster.node.osds = 1 cluster.raid = 1+0 cluster.replicas = 4 -cluster.mttc = 20000 # RAID: 4 @ 4+2 san.node.osds = 4 san.raid = 4+2 san.replicas = 2 -san.mttc = 20000 # RAID: 2 @ 3+1 target.node.osds = 2 target.raid = 3+1 target.replicas = 3 -target.mttc = 40000 diff --git a/sim/gobs/sweeps/dl01-full.cfg b/sim/gobs/sweeps/dl01-full.cfg index 5e3fad1..e9229b6 100644 --- a/sim/gobs/sweeps/dl01-full.cfg +++ b/sim/gobs/sweeps/dl01-full.cfg @@ -9,12 +9,13 @@ include = disks-full.cfg fs_pdsi_1EB_chunky.cfg faults_real_range_dl01.cfg \ # Bit-length of addresses: B = 32 -systems = san;target +systems = target -# 1 year: 31536000 -simulator.duration = 3153600 +# 1 year: 31536000 +simulator.duration = 31536000 plot.blackandwhite = true +plot.timeunit = years # Clear some memory after each simulator run sweep.free = true diff --git a/sim/gobs/sweeps/dl03-full.cfg b/sim/gobs/sweeps/dl03-full.cfg new file mode 100644 index 0000000..a08e56a --- /dev/null +++ b/sim/gobs/sweeps/dl03-full.cfg @@ -0,0 +1,39 @@ + +# Simulator configuration + +# Basics: + +include = disks-full.cfg fs_pdsi_1EB_vchunky.cfg faults_real_range_dl03.cfg \ + user00.cfg dl_output.cfg + +# Bit-length of addresses: +B = 32 + +systems = san +# target + +# 1 year: 31536000 +# 2 years: 63072000 +# 5 years: 157680000 +simulator.duration = 31536000 +#simulator.duration = 63072000 +simulator.duration = 157680000 + +plot.blackandwhite = true +plot.timeunit = years + +# Clear some memory after each simulator run +sweep.free = true + +# Do a fsck +fsck = true +# Check for stray objects +strays = true + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/dl03-full.zsh b/sim/gobs/sweeps/dl03-full.zsh new file mode 100755 index 0000000..af478af --- /dev/null +++ b/sim/gobs/sweeps/dl03-full.zsh @@ -0,0 +1,9 @@ +#!/bin/zsh + +NAME="dl03-full" +SEED="seed=821" +CLASS=gobs.sweep.x.DataLoss03 + +SWEEPDIR=$( dirname $0 ) + +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/dl04-full.cfg b/sim/gobs/sweeps/dl04-full.cfg new file mode 100644 index 0000000..8f3c1b9 --- /dev/null +++ b/sim/gobs/sweeps/dl04-full.cfg @@ -0,0 +1,36 @@ + +# Simulator configuration + +# Basics: + +include = disks-full.cfg fs_pdsi_1EB_vchunky.cfg faults_real_range_dl03.cfg \ + user00.cfg dl_output.cfg + +# Bit-length of addresses: +B = 32 + +systems = san;target + +# 1 year: 31536000 +# 2 years: 63072000 +# 5 years: 157680000 +simulator.duration = 31536000 +#simulator.duration = 63072000 +#simulator.duration = 157680000 + +plot.blackandwhite = true +plot.timeunit = years + +# Clear some memory after each simulator run +sweep.free = true + +# Do a fsck +fsck = true +# Check for stray objects +strays = true + +replica.source = primary + +### Choose a NodeScheme implementation: + +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/dl04-full.zsh b/sim/gobs/sweeps/dl04-full.zsh new file mode 100755 index 0000000..01bc12f --- /dev/null +++ b/sim/gobs/sweeps/dl04-full.zsh @@ -0,0 +1,11 @@ +#!/bin/zsh + +NAME="dl04-full" +# SEED="seed=489" +CLASS=gobs.sweep.x.DataLoss04 + +SWEEPDIR=$( dirname $0 ) + +source ${SWEEPDIR}/sweep.zsh + +# Bad seeds: 173 diff --git a/sim/gobs/sweeps/faults_10.cfg b/sim/gobs/sweeps/faults_10.cfg new file mode 100644 index 0000000..6d30ba7 --- /dev/null +++ b/sim/gobs/sweeps/faults_10.cfg @@ -0,0 +1,16 @@ + +mttr.reboot = 3600 +mttr.disk = 86400 + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 315360000 +disk.mttf.max = 315360000 +disk.mttf.step = interval:10000000 + +# 3 months: 7776000 +#cluster.mttc = 7776000000 + +san.mttc = 31536000 +target.mttc = 31536000 diff --git a/sim/gobs/sweeps/faults_real_range_dl01.cfg b/sim/gobs/sweeps/faults_real_range_dl01.cfg index f0e0620..02c2d30 100644 --- a/sim/gobs/sweeps/faults_real_range_dl01.cfg +++ b/sim/gobs/sweeps/faults_real_range_dl01.cfg @@ -1,20 +1,23 @@ mttr.reboot = 3600 -mttr.disk = 3600 +mttr.disk = 36000 # Disk MTTF: 1-100 years. -# 1 month: 25920000 -# 10 years: 315360000 # M K -disk.mttf.min = 31536000 -disk.mttf.max = 3153600000 -disk.mttf.step = literal:31536000,31536000000 +# 1 month: 2592000 +# 1 years: 31536000 +disk.mttf.min = 12536000 +disk.mttf.max = 35360000 +#disk.mttf.max = 3153600000 +disk.mttf.step = mids +# literal:31536000,31536000000 # Artificially short -disk.mttf.step = literal:3153600,3153600000 +#disk.mttf.step = literal:3153600,3153600000 -# 3 months: 7776000 -cluster.mttc = 7776000 -san.mttc = 7776000 -target.mttc = 7776000 +# 3 months: 7776000 +# 1 year: 31536000 +cluster.mttc = 315360000 +san.mttc = 315360000 +target.mttc = 315360000 diff --git a/sim/gobs/sweeps/faults_real_range_dl03.cfg b/sim/gobs/sweeps/faults_real_range_dl03.cfg new file mode 100644 index 0000000..891370c --- /dev/null +++ b/sim/gobs/sweeps/faults_real_range_dl03.cfg @@ -0,0 +1,24 @@ + +mttr.reboot = 3600 +mttr.disk = 36000 + +# Disk MTTF: 1-100 years. + +# M K +# 1 month: 2592000 +# 1 years: 31536000 +disk.mttf.min = 9536000 +disk.mttf.max = 3153600000 +#disk.mttf.max = 31536000 +#disk.mttf.max = 3153600000 +disk.mttf.step = literal:9536000,15536000,20536000,40536000,60536000,157680000 +# literal:31536000,31536000000 + +# Artificially short +#disk.mttf.step = literal:3153600,3153600000 + +# 3 months: 7776000 +# 1 year: 31536000 +cluster.mttc = 315360000 +san.mttc = 315360000 +target.mttc = 315360000 diff --git a/sim/gobs/sweeps/faults_real_zaps_only.cfg b/sim/gobs/sweeps/faults_real_zaps_only.cfg index 74eab4a..aa4a2f9 100644 --- a/sim/gobs/sweeps/faults_real_zaps_only.cfg +++ b/sim/gobs/sweeps/faults_real_zaps_only.cfg @@ -5,8 +5,8 @@ mttr.disk = 86400 # 1 month: 25920000 # 10 years: 315360000 # M K -disk.mttf.min = 315360000 -disk.mttf.max = 315360000 +disk.mttf.min = 31536000 +disk.mttf.max = 31536000 disk.mttf.step = interval:10000000 # 3 months: 7776000 diff --git a/sim/gobs/sweeps/faults_small.cfg b/sim/gobs/sweeps/faults_small.cfg index 6451ca3..285e670 100644 --- a/sim/gobs/sweeps/faults_small.cfg +++ b/sim/gobs/sweeps/faults_small.cfg @@ -2,6 +2,6 @@ mttr.reboot = 10 mttr.disk = 50 -disk.mttf.min = 10000 +disk.mttf.min = 20000 disk.mttf.max = 20000 disk.mttf.step = interval:4000 diff --git a/sim/gobs/sweeps/faults_small_zaps_only.cfg b/sim/gobs/sweeps/faults_small_zaps_only.cfg new file mode 100644 index 0000000..ff0af50 --- /dev/null +++ b/sim/gobs/sweeps/faults_small_zaps_only.cfg @@ -0,0 +1,11 @@ + +mttr.reboot = 10 +mttr.disk = 500 + +disk.mttf.min = 20000 +disk.mttf.max = 20000 +disk.mttf.step = interval:4000 + +cluster.mttc = 20000000 +san.mttc = 20000000 +target.mttc = 40000000 diff --git a/sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg b/sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg index 50ced8f..5f3d006 100644 --- a/sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg +++ b/sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg @@ -4,8 +4,8 @@ B = 32 # Number of files: -files = 20 +files = 40 # FileFactory implementation PlainFiles impl.filefactory = PDSIFiles # 20% size 0, 60% size 1K-100K, 20% size 100K-10M -file.pdsi.points = 0.1,1K;0.1,4T;0.2,10P;1.0,30P +file.pdsi.points = 0.1,1K;0.1,4T;0.2,20P;1.0,30P diff --git a/sim/gobs/sweeps/fs_pdsi_1EB_vchunky.cfg b/sim/gobs/sweeps/fs_pdsi_1EB_vchunky.cfg new file mode 100644 index 0000000..28b90b5 --- /dev/null +++ b/sim/gobs/sweeps/fs_pdsi_1EB_vchunky.cfg @@ -0,0 +1,11 @@ + +# FS PDSI 1EB - CHUNKY + +B = 32 + +# Number of files: +files = 8 +# FileFactory implementation PlainFiles +impl.filefactory = PDSIFiles +# 20% size 0, 60% size 1K-100K, 20% size 100K-10M +file.pdsi.points = 0.1,1K;0.1,4T;0.2,50P;1.0,300P diff --git a/sim/gobs/sweeps/fs_pdsi_small.cfg b/sim/gobs/sweeps/fs_pdsi_small.cfg index 8314908..6f81ea5 100644 --- a/sim/gobs/sweeps/fs_pdsi_small.cfg +++ b/sim/gobs/sweeps/fs_pdsi_small.cfg @@ -2,7 +2,7 @@ # FS PDSI SMALL # Number of files: -files = 30 +files = 20 # FileFactory implementation PlainFiles impl.filefactory = PDSIFiles # 20% size 0, 40% size 1K-1M, 40% size 1M-20M diff --git a/sim/gobs/sweeps/rb01-long.cfg b/sim/gobs/sweeps/rb01-long.cfg index b3ab945..473582b 100644 --- a/sim/gobs/sweeps/rb01-long.cfg +++ b/sim/gobs/sweeps/rb01-long.cfg @@ -3,8 +3,8 @@ # Basics: -include = disks-full-zaps-only.cfg user00.cfg \ - fs_pdsi_1EB_chunky.cfg rb_output.cfg +include = disks-full-zaps-only.cfg faults_10.cfg \ + fs_pdsi_1EB_chunky.cfg user00.cfg rb_output.cfg systems = target @@ -25,7 +25,7 @@ B = 20 # Number of faults: # 1 month: 2592000 -# 1 month: 31536000 +# 1 year: 31536000 # 10 years: 315360000 # M K simulator.duration = 31536000 diff --git a/sim/gobs/sweeps/rb02-full.cfg b/sim/gobs/sweeps/rb02-full.cfg index 872fc11..bc44bc0 100644 --- a/sim/gobs/sweeps/rb02-full.cfg +++ b/sim/gobs/sweeps/rb02-full.cfg @@ -9,9 +9,10 @@ include = disks-full.cfg fs_pdsi_1EB_chunky.cfg faults_real_zaps_only.cfg \ # Statistics: # Rebuild traffic over time analysis: -plot.traffic.interval = 3600 +plot.traffic.interval = 10800 plot.blackandwhite = true plot.timeunit = hours +# plot.force.domain.upper = 60 regimen.skip = false @@ -20,7 +21,8 @@ systems = san;target # Bit-length of addresses: B = 32 -fault.count = 10 +# Last time was 1 +fault.count = 20 vertexgroups.prefix = 6 # Replica selection: See Replicated.locateSource() diff --git a/sim/gobs/sweeps/rb02-full.zsh b/sim/gobs/sweeps/rb02-full.zsh index a4d6e5d..49dd9ff 100755 --- a/sim/gobs/sweeps/rb02-full.zsh +++ b/sim/gobs/sweeps/rb02-full.zsh @@ -1,7 +1,7 @@ #!/bin/zsh NAME="rb02-full" -SEED="seed=821" +#SEED="seed=821" CLASS=gobs.sweep.x.Rebuild02 SWEEPDIR=$( dirname $0 ) diff --git a/sim/gobs/sweeps/rb02-small.cfg b/sim/gobs/sweeps/rb02-small.cfg index fe850d2..b851c90 100644 --- a/sim/gobs/sweeps/rb02-small.cfg +++ b/sim/gobs/sweeps/rb02-small.cfg @@ -3,13 +3,17 @@ # Basics: -include = disks-small.cfg user00.cfg fs_pdsi_small.cfg rb_output.cfg +include = disks-small.cfg fs_pdsi_small.cfg faults_small_zaps_only.cfg \ + user00.cfg rb_output.cfg # Statistics: +systems = target + # Rebuild traffic over time analysis: -plot.traffic.interval = 50 +plot.traffic.interval = 100 plot.blackandwhite = true +plot.force.domain.upper = 3000 regimen.skip = false diff --git a/sim/gobs/sweeps/rb02-small.zsh b/sim/gobs/sweeps/rb02-small.zsh index 8ca4b01..5b343a1 100755 --- a/sim/gobs/sweeps/rb02-small.zsh +++ b/sim/gobs/sweeps/rb02-small.zsh @@ -1,7 +1,7 @@ #!/bin/zsh NAME="rb02-small" -SEED="seed=821" +# SEED="seed=821" CLASS=gobs.sweep.x.Rebuild02 SWEEPDIR=$( dirname $0 ) diff --git a/sim/gobs/sweeps/rb_output.cfg b/sim/gobs/sweeps/rb_output.cfg index 26a200a..11d4f2c 100644 --- a/sim/gobs/sweeps/rb_output.cfg +++ b/sim/gobs/sweeps/rb_output.cfg @@ -4,14 +4,14 @@ # Print all properties at simulation start print.properties = true # Print nodes and objects at startup (true/false) -print.initialnodes = false +print.initialnodes = true # Print nodes and objects after each rebuild (true/false) print.nodes = false # Print object tranfers during rebuilds (true/false) print.rebuilds = false # Print file information (true/false) -print.files = false -print.filesobjects = false +print.files = true +print.filesobjects = true # Print load change statistics (true/false) print.loaddiffs = true # Print user accesses diff --git a/sim/gobs/sweeps/sweep.zsh b/sim/gobs/sweeps/sweep.zsh index d4edc90..71d38b7 100644 --- a/sim/gobs/sweeps/sweep.zsh +++ b/sim/gobs/sweeps/sweep.zsh @@ -17,6 +17,8 @@ start=$( date +%s ) } >& ${NAME}.out stop=$( date +%s ) +[[ ${BEEP} == "1" ]] && beeps & + if (( CODE )) then less ${NAME}.out hooks/post-receive -- Triton Repository

1 0

Triton Repository branch, master, updated. 2f58f0721fd0eb023ac697593338f8a82e0b76e3
by noreply＠mcs.anl.gov 10 Mar '10

10 Mar '10

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via 2f58f0721fd0eb023ac697593338f8a82e0b76e3 (commit) via d7f7e2e56b4f6ba2367e717923a28892b67f3fc4 (commit) via 0e7377233578416aa5a9db670cc25cf9ccc35468 (commit) via 57e98b8a976275f8379810bdfcec4fa0615f3211 (commit) via 1da14f4624c76aa64b0eb4eae886c0328d36e5b8 (commit) via 16e546a721ab457b75e8ab75148fda28a22da3a7 (commit) from 7c8131a56d34d2e3924aa15f83493f25afc9f022 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 2f58f0721fd0eb023ac697593338f8a82e0b76e3 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Wed Mar 10 17:38:37 2010 -0600 EOL fixes. commit d7f7e2e56b4f6ba2367e717923a28892b67f3fc4 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Wed Mar 10 17:37:20 2010 -0600 New and modified input configurations for DSN draft. commit 0e7377233578416aa5a9db670cc25cf9ccc35468 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Wed Mar 10 17:36:34 2010 -0600 Moved time arithmetic function. commit 57e98b8a976275f8379810bdfcec4fa0615f3211 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Wed Mar 10 17:35:38 2010 -0600 Minor modifications to plots and sweeps for DSN draft. commit 1da14f4624c76aa64b0eb4eae886c0328d36e5b8 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Wed Mar 10 17:33:57 2010 -0600 Add some comments and profiling output. commit 16e546a721ab457b75e8ab75148fda28a22da3a7 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Wed Mar 10 17:32:48 2010 -0600 Fix erroneous rebuild time calculation. ----------------------------------------------------------------------- Summary of changes: sim/gobs/src/gobs/plot/AbstractPlot.java | 30 ++++++- sim/gobs/src/gobs/plot/MultiLogXPlot.java | 12 ++- sim/gobs/src/gobs/plot/MultiplePlot.java | 10 +-- sim/gobs/src/gobs/plot/Plot.java | 2 +- sim/gobs/src/gobs/sim/Rebuild.java | 14 ++- sim/gobs/src/gobs/sim/Simulator.java | 23 +++++- sim/gobs/src/gobs/sweep/AbstractSweep.java | 60 ++++++++++++- sim/gobs/src/gobs/sweep/Helpers.java | 5 +- sim/gobs/src/gobs/sweep/Sweep.java | 11 ++- sim/gobs/src/gobs/sweep/x/DataLoss01.java | 112 +++++++++++------------- sim/gobs/src/gobs/sweep/x/DataLoss02.java | 4 +- sim/gobs/src/gobs/sweep/x/Decluster03.java | 13 +--- sim/gobs/src/gobs/sweep/x/Rebuild01.java | 52 +++++++---- sim/gobs/src/gobs/sweep/x/Rebuild02.java | 126 ++++++++++++-------------- sim/gobs/src/gobs/util/Time.java | 28 ++++++- sim/gobs/src/gobs/util/Tools.java | 14 +--- sim/gobs/sweeps/dc03kpv01.cfg | 6 +- sim/gobs/sweeps/dc03kpv03.cfg | 4 +- sim/gobs/sweeps/dc03kpv05.cfg | 5 +- sim/gobs/sweeps/dc04kpv01.cfg | 5 +- sim/gobs/sweeps/dc04kpv03.cfg | 6 +- sim/gobs/sweeps/dc04kpv05.cfg | 6 +- sim/gobs/sweeps/dc_output.cfg | 11 +-- sim/gobs/sweeps/decluster03.zsh | 2 +- sim/gobs/sweeps/disks-full-zaps-only.cfg | 19 ++++ sim/gobs/sweeps/disks-full.cfg | 19 +--- sim/gobs/sweeps/disks-long.cfg | 18 ++++ sim/gobs/sweeps/disks-medium-zaps-only.cfg | 38 ++++++++ sim/gobs/sweeps/disks-medium.cfg | 15 +--- sim/gobs/sweeps/disks-small.cfg | 9 -- sim/gobs/sweeps/dl01-full.cfg | 33 +++++++ sim/gobs/sweeps/dl01-full.zsh | 9 ++ sim/gobs/sweeps/dl01-medium.cfg | 20 ++--- sim/gobs/sweeps/dl01-small.cfg | 18 ++--- sim/gobs/sweeps/dl01.cfg | 2 +- sim/gobs/sweeps/dl_output.cfg | 8 +- sim/gobs/sweeps/faults_medium.cfg | 10 ++ sim/gobs/sweeps/faults_medium_range_dl01.cfg | 18 ++++ sim/gobs/sweeps/faults_real.cfg | 10 ++ sim/gobs/sweeps/faults_real_range_dl01.cfg | 20 ++++ sim/gobs/sweeps/faults_real_zaps_only.cfg | 16 ++++ sim/gobs/sweeps/faults_small.cfg | 7 ++ sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg | 11 +++ sim/gobs/sweeps/fs_pdsi_medium.cfg | 2 +- sim/gobs/sweeps/helpers.zsh | 18 ++++ sim/gobs/sweeps/kpv05.cfg | 2 - sim/gobs/sweeps/rb01-long.cfg | 41 +++++++++ sim/gobs/sweeps/rb01-long.zsh | 11 +++ sim/gobs/sweeps/rb01-medium.cfg | 9 ++- sim/gobs/sweeps/rb02-full.cfg | 32 +++++++ sim/gobs/sweeps/rb02-full.zsh | 9 ++ sim/gobs/sweeps/rb02-medium.cfg | 31 +++++++ sim/gobs/sweeps/rb02-medium.zsh | 9 ++ sim/gobs/sweeps/rb02-small.cfg | 2 +- sim/gobs/sweeps/rb_output.cfg | 6 +- sim/gobs/sweeps/sweep.zsh | 5 +- sim/gobs/sweeps/system_100,200.cfg | 24 +++++ sim/gobs/sweeps/system_100,600.cfg | 16 ---- sim/gobs/sweeps/system_100-600.cfg | 16 ---- sim/gobs/sweeps/system_100-700.cfg | 23 +++++ sim/gobs/sweeps/system_600.cfg | 6 +- 61 files changed, 801 insertions(+), 322 deletions(-) create mode 100644 sim/gobs/sweeps/disks-full-zaps-only.cfg create mode 100644 sim/gobs/sweeps/disks-long.cfg create mode 100644 sim/gobs/sweeps/disks-medium-zaps-only.cfg create mode 100644 sim/gobs/sweeps/dl01-full.cfg create mode 100755 sim/gobs/sweeps/dl01-full.zsh create mode 100644 sim/gobs/sweeps/faults_medium.cfg create mode 100644 sim/gobs/sweeps/faults_medium_range_dl01.cfg create mode 100644 sim/gobs/sweeps/faults_real.cfg create mode 100644 sim/gobs/sweeps/faults_real_range_dl01.cfg create mode 100644 sim/gobs/sweeps/faults_real_zaps_only.cfg create mode 100644 sim/gobs/sweeps/faults_small.cfg create mode 100644 sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg create mode 100644 sim/gobs/sweeps/helpers.zsh create mode 100644 sim/gobs/sweeps/rb01-long.cfg create mode 100755 sim/gobs/sweeps/rb01-long.zsh create mode 100644 sim/gobs/sweeps/rb02-full.cfg create mode 100755 sim/gobs/sweeps/rb02-full.zsh create mode 100644 sim/gobs/sweeps/rb02-medium.cfg create mode 100755 sim/gobs/sweeps/rb02-medium.zsh create mode 100644 sim/gobs/sweeps/system_100,200.cfg create mode 100644 sim/gobs/sweeps/system_100-700.cfg Diff of changes: diff --git a/sim/gobs/src/gobs/plot/AbstractPlot.java b/sim/gobs/src/gobs/plot/AbstractPlot.java index be62a5f..4906fba 100755 --- a/sim/gobs/src/gobs/plot/AbstractPlot.java +++ b/sim/gobs/src/gobs/plot/AbstractPlot.java @@ -2,13 +2,37 @@ package gobs.plot; import java.io.IOException; import java.io.OutputStream; +import java.util.Properties; import org.apache.xmlgraphics.java2d.ps.EPSDocumentGraphics2D; +import gobs.util.Tools; + public class AbstractPlot { - static int w = 400; - static int h = 400; + static int width = 400; + static int height = 400; + + public static boolean bw = false; + public static boolean forceDomainUpper = false; + public static double forceDomainUpperValue = 0; + + public static void configure(Properties properties) + { + if (Tools.has(properties, "plot.force.domain.upper")) + forceDomainUpper + (Tools.decimal(properties, "plot.force.domain.upper")); + if (Tools.enabled(properties, "plot.blackandwhite")) + bw = true; + if (Tools.has(properties, "plot.width")) + width = Tools.integer(properties, "plot.width"); + } + + static void forceDomainUpper(double v) + { + forceDomainUpper = true; + forceDomainUpperValue = v; + } /** Get a writable EPS file at output. @@ -22,7 +46,7 @@ public class AbstractPlot g2d.setGraphicContext (new org.apache.xmlgraphics.java2d.GraphicContext()); - g2d.setupDocument(output, w, h); + g2d.setupDocument(output, width, height); return g2d; } diff --git a/sim/gobs/src/gobs/plot/MultiLogXPlot.java b/sim/gobs/src/gobs/plot/MultiLogXPlot.java index d275a0a..68dcf4a 100644 --- a/sim/gobs/src/gobs/plot/MultiLogXPlot.java +++ b/sim/gobs/src/gobs/plot/MultiLogXPlot.java @@ -19,6 +19,7 @@ import org.jfree.chart.axis.*; import org.jfree.chart.plot.PlotOrientation; import org.jfree.chart.plot.XYPlot; import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer; +import org.jfree.data.Range; import org.jfree.data.xy.XYSeries; import org.jfree.data.xy.XYSeriesCollection; @@ -29,9 +30,8 @@ import gobs.util.*; * */ public class MultiLogXPlot + extends AbstractPlot { - public static boolean bw = false; - /** Generate simple plot. @param collection The x,y data @@ -100,6 +100,14 @@ public class MultiLogXPlot LogarithmicAxis domainAxis = new LogarithmicAxis(xlabel); domainAxis.setLabelFont(labelFont); domainAxis.setTickLabelFont(tickFont); + if (forceDomainUpper) + { + Range range = plot.getDomainAxis().getRange(); + double lower = range.getLowerBound(); + Range adjusted = new Range(lower, forceDomainUpperValue); + domainAxis.setRange(adjusted); + System.out.println("range: " + domainAxis.getRange()); + } plot.setDomainAxis(domainAxis); chart.draw(g2d, rectangle); diff --git a/sim/gobs/src/gobs/plot/MultiplePlot.java b/sim/gobs/src/gobs/plot/MultiplePlot.java index 1c7521d..084b50c 100644 --- a/sim/gobs/src/gobs/plot/MultiplePlot.java +++ b/sim/gobs/src/gobs/plot/MultiplePlot.java @@ -27,9 +27,8 @@ import gobs.util.Bits; * */ public class MultiplePlot + extends AbstractPlot { - public static boolean bw = false; - /** Generate simple plot. @param collection The x,y data. @@ -46,9 +45,6 @@ public class MultiplePlot Rectangle2D.Double rectangle = null; OutputStream out = null; - int w = 400; - int h = 400; - try { out = new FileOutputStream(output); @@ -58,11 +54,11 @@ public class MultiplePlot g2d.setGraphicContext (new org.apache.xmlgraphics.java2d.GraphicContext()); - rectangle = new Rectangle2D.Double(0, 0, w, h); + rectangle = new Rectangle2D.Double(0, 0, width, height); g2d.setGraphicContext (new org.apache.xmlgraphics.java2d.GraphicContext()); - g2d.setupDocument(out, w, h); + g2d.setupDocument(out, width, height); } catch (IOException e) { diff --git a/sim/gobs/src/gobs/plot/Plot.java b/sim/gobs/src/gobs/plot/Plot.java index 685db0f..695c7f6 100644 --- a/sim/gobs/src/gobs/plot/Plot.java +++ b/sim/gobs/src/gobs/plot/Plot.java @@ -30,7 +30,7 @@ public class Plot EPSDocumentGraphics2D g2d = null; OutputStream out = null; - Rectangle2D.Double rectangle = new Rectangle2D.Double(0, 0, w, h); + Rectangle2D.Double rectangle = new Rectangle2D.Double(0, 0, width, height); try { diff --git a/sim/gobs/src/gobs/sim/Rebuild.java b/sim/gobs/src/gobs/sim/Rebuild.java index 0cda51c..b1518e1 100644 --- a/sim/gobs/src/gobs/sim/Rebuild.java +++ b/sim/gobs/src/gobs/sim/Rebuild.java @@ -446,11 +446,17 @@ public class Rebuild { double result = 0.0; for (Copy copy : entries) - if (copy.finish > result) - result = copy.finish; + { + double t = copy.finish-start; + if (t > result) + result = t; + } for (Copy copy : repair.entries) - if (copy.finish > result) - result = copy.finish; + { + double t = copy.finish-start; + if (t > result) + result = t; + } return result; } diff --git a/sim/gobs/src/gobs/sim/Simulator.java b/sim/gobs/src/gobs/sim/Simulator.java index 703d1e9..ed6f2ac 100644 --- a/sim/gobs/src/gobs/sim/Simulator.java +++ b/sim/gobs/src/gobs/sim/Simulator.java @@ -19,9 +19,19 @@ import static gobs.util.Tools.bail; public class Simulator { + /** + Bit-length of addresses. + */ int B; + + /** + Number of server nodes. + */ int N; + /** + 2^B + */ public BigInteger M; int fileCount; @@ -374,7 +384,7 @@ public class Simulator // MTTC = span/mttfLoss = # loss events, so: dataLossFraction = mttc/events; - print("mttf: " + df.format(mttf)); + print("mttf: " + Time.toString(mttf)); print("dataLossFraction: " + df.format(dataLossFraction)); DRand engine = new DRand(Bits.nextInt()); @@ -529,9 +539,11 @@ public class Simulator void place() { + Date start = new Date(); print("allocating: " + objects.size()); placer.place(objects, nodes); - + Date stop = new Date(); + print("took: " + Time.diff(start, stop)); origCount = countObjects(nodes); // print("OBJECT_COUNT: " + objects.size()); @@ -717,6 +729,7 @@ public class Simulator while (! complete()) { fault = next; + Date start = new Date(); System.out.println("fault: " + faultCount); Node failed = induceFault(fault); Collection<Gob> damaged = failed.collection(); @@ -726,6 +739,8 @@ public class Simulator Collection<Copy> fixes = failCopies(failed); todo.addAll(fixes); next = rebuild(fault, failed, damaged); + Date stop = new Date(); + print("took: " + Time.diff(start, stop)); } finish(fault); } @@ -1163,10 +1178,12 @@ public class Simulator */ void computeTraffic() { + Date start = new Date(); if (! has("plot.traffic.interval")) return; System.out.println("TRAFFIC: "); + System.out.println("REBUILDS:" + rebuilds.size()); double interval = decimal("plot.traffic.interval"); List<Double> intervals = new ArrayList<Double>(); @@ -1186,6 +1203,8 @@ public class Simulator stats.results.setProperty("traffic-x", intervals.toString()); stats.results.setProperty("traffic-y", sample.toString()); + Date finish = new Date(); + System.out.println("took: " + Time.diff(start, finish)); } /** diff --git a/sim/gobs/src/gobs/sweep/AbstractSweep.java b/sim/gobs/src/gobs/sweep/AbstractSweep.java index bb4c1c5..682b4e1 100644 --- a/sim/gobs/src/gobs/sweep/AbstractSweep.java +++ b/sim/gobs/src/gobs/sweep/AbstractSweep.java @@ -205,7 +205,7 @@ public abstract class AbstractSweep } /** - Sweep over all integer values between min and max + Sweep over all values between min and max stepping over the interval argument. @param arg The argument, e.g., "interval:4". @return The list, e.g., min, min+4, min+8, max. @@ -259,6 +259,64 @@ public abstract class AbstractSweep return result; } + public static List<String> values(String step, + double min, double max) + { + /* + if (step.equals("all")) + return all(min,max); + else if (step.equals("tens")) + return tens(min,max); + else if (step.equals("exps")) + return exps(min,max); + else if (step.equals("mids")) + return mids(min,max); + else */ + if (step.startsWith("interval")) + return interval(step, min, max); + else if (step.startsWith("literal")) + return literal(step); + else + Tools.bail("Sweep.values(): Bad step: " + step); + return null; + } + + /** + Sweep over all values between min and max + stepping over the interval argument. + @param arg The argument, e.g., "interval:4". + @return The list, e.g., min, min+4, min+8, max. + */ + static List<String> interval(String arg, double min, double max) + { + String[] tokens = arg.split(":"); + + double interval = 0; + try + { + interval = Double.parseDouble(tokens[1]); + } + catch (NumberFormatException e) + { + Tools.bail("AbstractSweep: interval must be an double!"); + } + + Tools.check(interval > 0, + "AbstractSweep: interval must be a positive double!"); + + List<String> result = new ArrayList<String>(); + result.add(""+min); + int i = 1; + while (min + interval*i < max) + { + result.add("" + (min+interval*i)); + i++; + } + if (!result.contains(""+max)) + result.add(""+max); + return result; + } + /** Set properties from memory. */ diff --git a/sim/gobs/src/gobs/sweep/Helpers.java b/sim/gobs/src/gobs/sweep/Helpers.java index 4ec29e2..418d975 100755 --- a/sim/gobs/src/gobs/sweep/Helpers.java +++ b/sim/gobs/src/gobs/sweep/Helpers.java @@ -14,8 +14,8 @@ public class Helpers { public static List<String> valuesMTTF(Properties properties) { - int minMTTF = Tools.integer(properties, "disk.mttf.min"); - int maxMTTF = Tools.integer(properties, "disk.mttf.max"); + double minMTTF = Tools.decimal(properties, "disk.mttf.min"); + double maxMTTF = Tools.decimal(properties, "disk.mttf.max"); String stepMTTF = Tools.string(properties, "disk.mttf.step"); List<String> valuesMTTF = Sweep.values(stepMTTF, minMTTF, maxMTTF); @@ -38,5 +38,4 @@ public class Helpers properties.setProperty("replicas", ""+replicas); return raid; } - } diff --git a/sim/gobs/src/gobs/sweep/Sweep.java b/sim/gobs/src/gobs/sweep/Sweep.java index ce2263e..a3eb10b 100644 --- a/sim/gobs/src/gobs/sweep/Sweep.java +++ b/sim/gobs/src/gobs/sweep/Sweep.java @@ -17,6 +17,11 @@ import gobs.util.*; public class Sweep extends AbstractSweep { + static + { + System.out.println("SWEEP_START: " + Time.sdf(new Date())); + } + public Map<String,Simulator> sims = new Hashtable<String,Simulator>(); @@ -56,12 +61,14 @@ public class Sweep { for (int i = 0; i < repeats; i++) { - System.out.println("\nEXECUTE:"); + String tag = xname+"="+v+";unique="+i; + System.out.println("\nEXECUTE: " + tag); properties.setProperty("unique", new Integer(i).toString()); properties.setProperty(xname, v); + properties.setProperty("tag", tag); Simulator simulator = new Simulator(properties); simulator.execute(); - sims.put(xname+"="+v+";unique="+i, simulator); + sims.put(tag, simulator); } } } diff --git a/sim/gobs/src/gobs/sweep/x/DataLoss01.java b/sim/gobs/src/gobs/sweep/x/DataLoss01.java index 58b3f88..043061b 100644 --- a/sim/gobs/src/gobs/sweep/x/DataLoss01.java +++ b/sim/gobs/src/gobs/sweep/x/DataLoss01.java @@ -30,57 +30,52 @@ class DataLoss01 String output = Tools.string(properties, "output"); - int minNodes = Tools.integer(properties, "nodes.min"); - int maxNodes = Tools.integer(properties, "nodes.max"); - String stepNodes = Tools.string(properties, "nodes.step"); - List<String> valuesNodes = - Sweep.values(stepNodes, minNodes, maxNodes); - - int minMTTF = Tools.integer(properties, "mttf.min"); - int maxMTTF = Tools.integer(properties, "mttf.max"); - String stepMTTF = Tools.string(properties, "mttf.step"); - List<String> valuesMTTF = - Sweep.values(stepMTTF, minMTTF, maxMTTF); - - System.out.println("DataLoss01: nodes: " + valuesNodes + "\n" + - " mttf: " + valuesMTTF); - - final XYSeriesCollection collection = new XYSeriesCollection(); + List<String> systems = Tools.list(properties, "systems"); - properties.setProperty("impl.placement", "Nearest"); - Sweep pSweep = new Sweep(properties); - pSweep.repeats = Tools.integer(properties, "repeats"); - pSweep.execute("nodes", valuesNodes, "mttf", valuesMTTF); + String timeunit = "seconds"; + double scale = 1; + if (Tools.has(properties, "plot.timeunit")) + { + timeunit = Tools.string(properties, "plot.timeunit"); + if (timeunit.equals("minutes")) + scale = 60; + else if (timeunit.equals("hours")) + scale = 3600; + else if (timeunit.equals("days")) + scale = 86400; + } - List<XYSeries> allSeries = new ArrayList<XYSeries>(); + List<String> valuesMTTF = Helpers.valuesMTTF(properties); - List<XYSeries> pSeries = plots(pSweep, "PAST", - valuesNodes, valuesMTTF); - allSeries.addAll(pSeries); + System.out.println("DataLoss01: systems: " + systems + "\n" + + " mttf: " + valuesMTTF); - /* - properties.setProperty("impl.placement", "Kademlia"); - ArraySweep kSweep = new ArraySweep(properties); - kSweep.repeats = Tools.integer(properties, "repeats"); - kSweep.execute("Kademlia", "nodes", values, "distribution"); - allSeries.addAll(kSweep.collection); + final XYSeriesCollection collection = new XYSeriesCollection(); + List<XYSeries> allSeries = new ArrayList<XYSeries>(); - properties.setProperty("impl.placement", "VertexGroups"); - ArraySweep vgSweep = new ArraySweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); - allSeries.addAll(vgSweep.collection); - */ + properties.setProperty("impl.placement", "Nearest"); + for (String system : systems) + { + properties.setProperty("system", system); + Helpers.setupRAID(properties, system); + String mttc = Tools.string(properties, system+".mttc"); + properties.setProperty("mttc", mttc); + Sweep pSweep = new Sweep(properties); + pSweep.repeats = Tools.integer(properties, "repeats"); + pSweep.execute("disk.mttf", valuesMTTF); + + List<XYSeries> pSeries = plots(pSweep, "NEAREST-"+system, + valuesMTTF); + allSeries.addAll(pSeries); + } for (XYSeries series : allSeries) collection.addSeries(series); + MultiplePlot.configure(properties); MultiplePlot.plot(collection, "", "Mean Time to Disk Failure", - "Mean Time to Object Loss", output); - - // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); + "Objects Lost per Year", output); } /** @@ -88,33 +83,28 @@ class DataLoss01 @param label E.g., "PAST" */ static List<XYSeries> plots(Sweep sweep, String label, - List<String> valuesNodes, List<String> valuesMTTF) { List<XYSeries> results = new ArrayList<XYSeries>(); - for (String nodes : valuesNodes) + XYSeries series = new XYSeries(label); + for (String mttf : valuesMTTF) { - XYSeries series = new XYSeries(label + " N=" + nodes); - for (String mttf : valuesMTTF) - { - double time = 0.0; - int lost = 0; - double total = 0.0; - for (Simulator simulator : sweep.sims.values()) - if (simulator.string("nodes").equals(nodes) && - simulator.string("mttf").equals(mttf)) - { - System.out.println(simulator.string("tag")); - time += Tools.decimal(simulator.stats.results, "time"); - lost += Tools.integer(simulator.stats.results, "lost"); - total++; - } - if (lost > 0) - series.add(Integer.parseInt(mttf), time/lost); - } - results.add(series); + double time = 0.0; + int lost = 0; + double total = 0.0; + for (Simulator simulator : sweep.sims.values()) + if (simulator.string("disk.mttf").equals(mttf)) + { + System.out.println(simulator.string("tag")); + time += Tools.decimal(simulator.stats.results, "time"); + lost += Tools.integer(simulator.stats.results, "lost"); + total++; + } + series.add(Long.parseLong(mttf), lost); } + results.add(series); + return results; } diff --git a/sim/gobs/src/gobs/sweep/x/DataLoss02.java b/sim/gobs/src/gobs/sweep/x/DataLoss02.java index 61973e0..3faf206 100755 --- a/sim/gobs/src/gobs/sweep/x/DataLoss02.java +++ b/sim/gobs/src/gobs/sweep/x/DataLoss02.java @@ -31,8 +31,8 @@ public class DataLoss02 String output = Tools.string(properties, "output"); - int minMTTF = Tools.integer(properties, "disk.mttf.min"); - int maxMTTF = Tools.integer(properties, "disk.mttf.max"); + double minMTTF = Tools.decimal(properties, "disk.mttf.min"); + double maxMTTF = Tools.decimal(properties, "disk.mttf.max"); String stepMTTF = Tools.string(properties, "disk.mttf.step"); List<String> valuesMTTF = Sweep.values(stepMTTF, minMTTF, maxMTTF); diff --git a/sim/gobs/src/gobs/sweep/x/Decluster03.java b/sim/gobs/src/gobs/sweep/x/Decluster03.java index ace67ab..2977916 100644 --- a/sim/gobs/src/gobs/sweep/x/Decluster03.java +++ b/sim/gobs/src/gobs/sweep/x/Decluster03.java @@ -58,17 +58,8 @@ class Decluster03 values, "maxTotal"); collection.addSeries(series); } - /* - properties.setProperty("impl.placement", "VertexGroups"); - Sweep vgSweep = new Sweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - series = vgSweep.perform("Vertex Groups", "node.count", - values, "maxTotal"); - collection.addSeries(series); - */ - - if (Tools.enabled(properties, "plot.blackandwhite")) - MultiLogXPlot.bw = true; + + MultiLogXPlot.configure(properties); MultiLogXPlot.plot(collection, "", "node.count", "load fraction(%)", output); } diff --git a/sim/gobs/src/gobs/sweep/x/Rebuild01.java b/sim/gobs/src/gobs/sweep/x/Rebuild01.java index 7152eb6..dbc4e3b 100644 --- a/sim/gobs/src/gobs/sweep/x/Rebuild01.java +++ b/sim/gobs/src/gobs/sweep/x/Rebuild01.java @@ -30,6 +30,19 @@ class Rebuild01 String output = Tools.string(properties, "output"); + String timeunit = "seconds"; + double scale = 1; + if (Tools.has(properties, "plot.timeunit")) + { + timeunit = Tools.string(properties, "plot.timeunit"); + if (timeunit.equals("minutes")) + scale = 60; + else if (timeunit.equals("hours")) + scale = 3600; + else if (timeunit.equals("days")) + scale = 86400; + } + List<String> valuesMTTF = Helpers.valuesMTTF(properties); System.out.println("Rebuild01: " + valuesMTTF); @@ -50,30 +63,33 @@ class Rebuild01 List<XYSeries> series = pSweep.execute("NEAREST-"+system+" MTTF=", "disk.mttf", valuesMTTF, "traffic-x", "traffic-y"); + for (XYSeries s : series) + s.setKey("NEAREST-target"); allSeries.addAll(series); } - /* - properties.setProperty("impl.placement", "Kademlia"); - ArraySweep kSweep = new ArraySweep(properties); - kSweep.repeats = Tools.integer(properties, "repeats"); - kSweep.execute("Kademlia", "nodes", values, "distribution"); - allSeries.addAll(kSweep.collection); - - properties.setProperty("impl.placement", "VertexGroups"); - ArraySweep vgSweep = new ArraySweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); - allSeries.addAll(vgSweep.collection); - */ - for (XYSeries series : allSeries) - collection.addSeries(series); + { + XYSeries scaled = scaleX(series, scale); + collection.addSeries(scaled); + } - if (Tools.enabled(properties, "plot.blackandwhite")) - MultiplePlot.bw = true; + MultiplePlot.configure(properties); MultiplePlot.plot(collection, "", - "time", "copies in flight", + "time ("+timeunit+")", "copies in flight", output); } + + static XYSeries scaleX(XYSeries series, double scale) + { + XYSeries result = new XYSeries(series.getKey()); + for (int i = 0; i < series.getItemCount(); i++) + { + Number x = series.getX(i); + Number y = series.getY(i); + double dx = x.doubleValue()/scale; + result.add(dx,y); + } + return result; + } } diff --git a/sim/gobs/src/gobs/sweep/x/Rebuild02.java b/sim/gobs/src/gobs/sweep/x/Rebuild02.java index fc4b7fd..a4fd2c7 100644 --- a/sim/gobs/src/gobs/sweep/x/Rebuild02.java +++ b/sim/gobs/src/gobs/sweep/x/Rebuild02.java @@ -34,12 +34,22 @@ class Rebuild02 String output = Tools.string(properties, "output"); double interval = Tools.decimal(properties, "plot.traffic.interval"); + String timeunit = "seconds"; + double scale = 1; + if (Tools.has(properties, "plot.timeunit")) + { + timeunit = Tools.string(properties, "plot.timeunit"); + if (timeunit.equals("minutes")) + scale = 60; + else if (timeunit.equals("hours")) + scale = 3600; + } double diskSize = Tools.decimal(properties, "disk.size"); double diskSpeed = Tools.decimal(properties, "disk.speed"); - int minMTTF = Tools.integer(properties, "disk.mttf.min"); - int maxMTTF = Tools.integer(properties, "disk.mttf.max"); + double minMTTF = Tools.decimal(properties, "disk.mttf.min"); + double maxMTTF = Tools.decimal(properties, "disk.mttf.max"); String stepMTTF = Tools.string(properties, "disk.mttf.step"); List<String> valuesMTTF = Sweep.values(stepMTTF, minMTTF, maxMTTF); @@ -50,72 +60,49 @@ class Rebuild02 List<XYSeries> allSeries = new ArrayList<XYSeries>(); - properties.setProperty("impl.placement", "Nearest"); - List<String> systems = Tools.list(properties, "systems"); - List<Simulator> sims = new ArrayList<Simulator>(); - for (String system : systems) + for (String skip : new String[] { "false", "true" }) { - int nodeDevices = Tools.integer(properties, - system+".node.osds"); - String spec = Tools.string(properties, system+".raid"); - RAID raid = new RAID(spec, diskSize, diskSpeed); - int nodes = raid.estimateNodes(properties, nodeDevices); - properties.setProperty("raid", spec); - properties.setProperty("node.count", ""+nodes); - properties.setProperty("node.osds", ""+nodeDevices); - properties.setProperty("regimen.skip", "false"); - properties.setProperty("system", system); - - Sweep pySweep = new Sweep(properties); - pySweep.repeats = Tools.integer(properties, "repeats"); - - pySweep.execute("disk.mttf", valuesMTTF); - sims.addAll(pySweep.sims.values()); - } + properties.setProperty("impl.placement", "Nearest"); + List<String> systems = Tools.list(properties, "systems"); + List<Simulator> sims = new ArrayList<Simulator>(); + for (String system : systems) + { + int nodeDevices = Tools.integer(properties, + system+".node.osds"); + String spec = Tools.string(properties, system+".raid"); + RAID raid = new RAID(spec, diskSize, diskSpeed); + int nodes = raid.estimateNodes(properties, nodeDevices); + String mttc = Tools.string(properties, system+".mttc"); + properties.setProperty("mttc", mttc); + properties.setProperty("raid", spec); + properties.setProperty("node.count", ""+nodes); + properties.setProperty("node.osds", ""+nodeDevices); + properties.setProperty("regimen.skip", skip); + properties.setProperty("system", system); + + Sweep pySweep = new Sweep(properties); + pySweep.repeats = Tools.integer(properties, "repeats"); + + pySweep.execute("disk.mttf", valuesMTTF); + sims.addAll(pySweep.sims.values()); + } - List<XYSeries> pySeries = plots(sims, "PAST-Y", - "system", systems, - "disk.mttf", valuesMTTF, - interval); - allSeries.addAll(pySeries); - - /* - properties.setProperty("impl.placement", "Nearest"); - Sweep pnSweep = new Sweep(properties); - pnSweep.repeats = Tools.integer(properties, "repeats"); - pnSweep.execute("nodes", values); - List<XYSeries> pnSeries = plots(pnSweep, "PAST-N", "nodes", - values, interval); - allSeries.addAll(pnSeries); - */ - /* - properties.setProperty("impl.placement", "Kademlia"); - Sweep kSweep = new Sweep(properties); - kSweep.repeats = Tools.integer(properties, "repeats"); - kSweep.execute("nodes", values); - - List<XYSeries> kSeries = plots(kSweep, "Kademlia", "nodes", - values, interval); - allSeries.addAll(kSeries); - */ - - /* - properties.setProperty("impl.placement", "VertexGroups"); - ArraySweep vgSweep = new ArraySweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); - allSeries.addAll(vgSweep.collection); - */ + String t = skip.equals("true") ? "latent" : "active"; + List<XYSeries> pySeries = plots(sims, "NEAREST-"+t, + "system", systems, + "disk.mttf", valuesMTTF, + interval, scale); + allSeries.addAll(pySeries); + } for (XYSeries series : allSeries) collection.addSeries(series); - MultiplePlot.plot(collection, "Rebuild Traffic", - "time", "copies in flight", - output); - // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); + MultiplePlot.configure(properties); + MultiplePlot.plot(collection, "", + "time ("+timeunit+")", "copies in flight", + output); Sweep.reportRuntime(); } @@ -123,7 +110,7 @@ class Rebuild02 static List<XYSeries> plots(List<Simulator> sims, String label, String x1, List<String> values1, String x2, List<String> values2, - double interval) + double interval, double scale) { List<XYSeries> results = new ArrayList<XYSeries>(); @@ -132,13 +119,14 @@ class Rebuild02 for (String v2 : values2) { XYSeries series = - new XYSeries(label + " SYSTEM=" + v1 + ":" + v2); + // new XYSeries(label + " " + v1 + ":" + v2); + new XYSeries(label + " " + v1 + " "); List<Rebuild> rebuilds = new ArrayList<Rebuild>(); for (Simulator simulator : sims) if (Tools.string(simulator.properties, x1).equals(v1) && Tools.string(simulator.properties, x2).equals(v2)) rebuilds.addAll(simulator.rebuilds); - loadsToSeries(rebuilds, interval, series); + loadsToSeries(rebuilds, interval, scale, series); results.add(series); } } @@ -150,24 +138,28 @@ class Rebuild02 Store in series. */ static void loadsToSeries(List<Rebuild> rebuilds, double interval, - XYSeries series) + double scale, XYSeries series) { + System.out.println("loadsToSeries():"); int count; double longest = longestRebuild(rebuilds); + System.out.println("longest: " + longest); double d = 0.0; while (d <= longest) { + System.out.println("sample: " + d); + System.out.println("rebuilds: " + rebuilds.size()); double total = 0.0; for (Rebuild rebuild : rebuilds) { // rebuild.printOffset(d, Cause.ANY); if (Tools.bool(properties, "regimen.skip")) - total += rebuild.loadOffset(d, Cause.ANY); + total += rebuild.loadOffset(d, Cause.REPAIR); else total += rebuild.loadOffset(d, Cause.FAULT); } double sample = total/rebuilds.size(); - series.add(d, sample); + series.add(d/scale, sample); d += interval; } series.add(d, 0.0); diff --git a/sim/gobs/src/gobs/util/Time.java b/sim/gobs/src/gobs/util/Time.java index 260ee02..8b5a00b 100755 --- a/sim/gobs/src/gobs/util/Time.java +++ b/sim/gobs/src/gobs/util/Time.java @@ -1,6 +1,7 @@ package gobs.util; -import java.text.DecimalFormat; +import java.text.*; +import java.util.Date; import static java.lang.Math.floor; @@ -12,19 +13,33 @@ public class Time public static final double hour = 60*minute; public static final double day = 24*hour; + static DecimalFormat zf = new DecimalFormat("0"); static DecimalFormat zz = new DecimalFormat("00"); + static DateFormat sdf = new SimpleDateFormat("MM/dd/yyyy hh:mm a"); public String toString() { return toString(value); } + /** + Get d2-d1 as mm:ss + */ + public static String diff(Date d1, Date d2) + { + long t1 = d1.getTime()/1000; + long t2 = d2.getTime()/1000; + double t = t2 - t1; + + return toString(t); + } + public static String toString(double time) { StringBuilder sb = new StringBuilder(32); if (time > day) { - sb.append(floor(time/day)).append(" "); + sb.append(zf.format(floor(time/day))).append(" "); time %= day; } if (time > hour || sb.length() > 0) @@ -42,6 +57,15 @@ public class Time } /** + Primarily for real times. + sdf = SimpleDateFormat + */ + public static String sdf(Date date) + { + return sdf.format(date); + } + + /** Debugging only. */ public static void main(String args[]) diff --git a/sim/gobs/src/gobs/util/Tools.java b/sim/gobs/src/gobs/util/Tools.java index fa0221f..acacab8 100644 --- a/sim/gobs/src/gobs/util/Tools.java +++ b/sim/gobs/src/gobs/util/Tools.java @@ -3,8 +3,8 @@ package gobs.util; import java.io.IOException; import java.math.BigInteger; +import java.text.DecimalFormat; import java.util.Arrays; -import java.util.Date; import java.util.List; import java.util.Properties; @@ -14,6 +14,8 @@ import java.util.Properties; public class Tools { + static DecimalFormat df = new DecimalFormat("00"); + public static int integer(Properties properties, String variable) { int result = 0; @@ -284,14 +286,4 @@ public class Tools return result; } - - public static String diff(Date d1, Date d2) - { - long t1 = d1.getTime(); - long t2 = d2.getTime(); - long t = t2-t1; - long s = t%60; - long m = t/60; - return "" + m + ":" + s; - } } diff --git a/sim/gobs/sweeps/dc03kpv01.cfg b/sim/gobs/sweeps/dc03kpv01.cfg index a51bc26..10c086a 100644 --- a/sim/gobs/sweeps/dc03kpv01.cfg +++ b/sim/gobs/sweeps/dc03kpv01.cfg @@ -1,3 +1,7 @@ -include = kpv01.cfg system_20-1000.cfg dc_output.cfg user00.cfg +include = kpv01.cfg system_100-700.cfg fs_pdsi_1EB.cfg \ + dc_output.cfg user00.cfg +plot.force.domain.upper = 1100 + +fault.count = 1 diff --git a/sim/gobs/sweeps/dc03kpv03.cfg b/sim/gobs/sweeps/dc03kpv03.cfg index 24e03f5..c5d72e3 100644 --- a/sim/gobs/sweeps/dc03kpv03.cfg +++ b/sim/gobs/sweeps/dc03kpv03.cfg @@ -1,3 +1,5 @@ -include = kpv03.cfg system_20-1000.cfg dc_output.cfg user00.cfg +include = kpv03.cfg fs_pdsi_1EB.cfg system_100-700.cfg \ + dc_output.cfg user00.cfg +plot.force.domain.upper = 1100 diff --git a/sim/gobs/sweeps/dc03kpv05.cfg b/sim/gobs/sweeps/dc03kpv05.cfg index aab6a3f..abdfb49 100644 --- a/sim/gobs/sweeps/dc03kpv05.cfg +++ b/sim/gobs/sweeps/dc03kpv05.cfg @@ -1,2 +1,5 @@ -include = kpv05.cfg system_20-1000.cfg dc_output.cfg user00.cfg +include = kpv05.cfg system_100-700.cfg fs_pdsi_1EB.cfg \ + dc_output.cfg user00.cfg + +plot.force.domain.upper = 1100 diff --git a/sim/gobs/sweeps/dc04kpv01.cfg b/sim/gobs/sweeps/dc04kpv01.cfg index 9db537c..1b6f9bd 100644 --- a/sim/gobs/sweeps/dc04kpv01.cfg +++ b/sim/gobs/sweeps/dc04kpv01.cfg @@ -1,2 +1,5 @@ -include = kpv01.cfg system_600.cfg dc_output.cfg user00.cfg +include = kpv01.cfg system_600.cfg fs_pdsi_1EB.cfg \ + dc_output.cfg user00.cfg + +fault.count = 1 diff --git a/sim/gobs/sweeps/dc04kpv03.cfg b/sim/gobs/sweeps/dc04kpv03.cfg index 9db537c..c19a005 100644 --- a/sim/gobs/sweeps/dc04kpv03.cfg +++ b/sim/gobs/sweeps/dc04kpv03.cfg @@ -1,2 +1,6 @@ -include = kpv01.cfg system_600.cfg dc_output.cfg user00.cfg +include = kpv03.cfg system_600.cfg fs_pdsi_1EB.cfg \ + dc_output.cfg user00.cfg + +fault.count = 1 +plot.blackandwhite = true diff --git a/sim/gobs/sweeps/dc04kpv05.cfg b/sim/gobs/sweeps/dc04kpv05.cfg index 9db537c..4962c40 100644 --- a/sim/gobs/sweeps/dc04kpv05.cfg +++ b/sim/gobs/sweeps/dc04kpv05.cfg @@ -1,2 +1,6 @@ -include = kpv01.cfg system_600.cfg dc_output.cfg user00.cfg +include = kpv05.cfg system_600.cfg fs_pdsi_1EB.cfg \ + dc_output.cfg user00.cfg + +fault.count = 1 +plot.blackandwhite = true diff --git a/sim/gobs/sweeps/dc_output.cfg b/sim/gobs/sweeps/dc_output.cfg index b3b7d43..f26b687 100644 --- a/sim/gobs/sweeps/dc_output.cfg +++ b/sim/gobs/sweeps/dc_output.cfg @@ -4,17 +4,16 @@ # Print all properties at simulation start print.properties = true # Print nodes and objects at startup (true/false) -print.initialnodes = true +print.initialnodes = false # Print nodes and objects after each rebuild (true/false) -print.nodes = true +print.nodes = false # Print object tranfers during rebuilds (true/false) -print.rebuilds = true +print.rebuilds = false # Print file information (true/false) -print.files = true -print.filesobjects = true +print.files = false +print.filesobjects = false # Print load change statistics (true/false) print.loaddiffs = true # Print user accesses print.accesses = false -plot.blackandwhite = true diff --git a/sim/gobs/sweeps/decluster03.zsh b/sim/gobs/sweeps/decluster03.zsh index 4b9cc9a..d3a5ff1 100755 --- a/sim/gobs/sweeps/decluster03.zsh +++ b/sim/gobs/sweeps/decluster03.zsh @@ -6,7 +6,7 @@ NAME=$1 NAME="dc03${NAME}" -# SEED="seed=821" +SEED="seed=822" CLASS=gobs.sweep.x.Decluster03 SWEEPDIR=$( dirname $0 ) diff --git a/sim/gobs/sweeps/disks-full-zaps-only.cfg b/sim/gobs/sweeps/disks-full-zaps-only.cfg new file mode 100644 index 0000000..87e9e0c --- /dev/null +++ b/sim/gobs/sweeps/disks-full-zaps-only.cfg @@ -0,0 +1,19 @@ + +# 30 TB T G M K +disk.size = 30000000000000 +disk.speed = 400000000 + +# RAID: None +cluster.node.osds = 1 +cluster.raid = 1+0 +cluster.replicas = 4 + +# RAID: 4 @ 4+2 +san.node.osds = 8 +san.raid = 8+2 +san.replicas = 2 + +# RAID: 2 @ 3+1 +target.node.osds = 4 +target.raid = 4+1 +target.replicas = 3 diff --git a/sim/gobs/sweeps/disks-full.cfg b/sim/gobs/sweeps/disks-full.cfg index 93f56f4..fe2f5ce 100644 --- a/sim/gobs/sweeps/disks-full.cfg +++ b/sim/gobs/sweeps/disks-full.cfg @@ -1,31 +1,22 @@ -mttr.reboot = 100 -mttr.disk = 500 - # 30 TB T G M K disk.size = 30000000000000 disk.speed = 400000000 -disk.mttf.min = 20000000 -disk.mttf.max = 30000000 -disk.mttf.step = interval:10000000 - -systems = san;target +# MTTC: 3 months # RAID: None cluster.node.osds = 1 cluster.raid = 1+0 cluster.replicas = 4 -cluster.mttc = 20000 -# RAID: 4 @ 4+2 +# RAID: 8 @ 8+2 san.node.osds = 8 san.raid = 8+2 san.replicas = 2 -san.mttc = 20000 -# RAID: 2 @ 3+1 +# RAID: 4 @ 4+1 target.node.osds = 4 -target.raid = 3+1 +target.raid = 4+1 target.replicas = 3 -target.mttc = 60000 + diff --git a/sim/gobs/sweeps/disks-long.cfg b/sim/gobs/sweeps/disks-long.cfg new file mode 100644 index 0000000..132a5dc --- /dev/null +++ b/sim/gobs/sweeps/disks-long.cfg @@ -0,0 +1,18 @@ + +mttr.reboot = 10 +mttr.disk = 50 + +disk.size = 500000 +disk.speed = 3000 + +disk.mttf.min = 10000 +disk.mttf.max = 10000 +disk.mttf.step = literal:10000 + +systems = target + +# RAID: 2 @ 3+1 +target.node.osds = 2 +target.raid = 3+1 +target.replicas = 3 +target.mttc = 40000 diff --git a/sim/gobs/sweeps/disks-medium-zaps-only.cfg b/sim/gobs/sweeps/disks-medium-zaps-only.cfg new file mode 100644 index 0000000..6922b8a --- /dev/null +++ b/sim/gobs/sweeps/disks-medium-zaps-only.cfg @@ -0,0 +1,38 @@ + +# 1 day +mttr.reboot = 3600 +mttr.disk = 86400 + +# 5GB +disk.size = 5000000000 +# 100MB/s +disk.speed = 100000000 + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 315360000 +disk.mttf.max = 315360000 +disk.mttf.step = interval:10000000 + +systems = san;target + +# 3 months: 7776000 + +# RAID: None +cluster.node.osds = 1 +cluster.raid = 1+0 +cluster.replicas = 4 +cluster.mttc = 30000000000 + +# RAID: 4 @ 4+2 +san.node.osds = 8 +san.raid = 4+2 +san.replicas = 2 +san.mttc = 30000000000 + +# RAID: 2 @ 3+1 +target.node.osds = 2 +target.raid = 3+1 +target.replicas = 3 +target.mttc = 30000000000 diff --git a/sim/gobs/sweeps/disks-medium.cfg b/sim/gobs/sweeps/disks-medium.cfg index 0be9e02..e794faf 100644 --- a/sim/gobs/sweeps/disks-medium.cfg +++ b/sim/gobs/sweeps/disks-medium.cfg @@ -1,32 +1,25 @@ -mttr.reboot = 100 -mttr.disk = 500 - # 5GB disk.size = 5000000000 # 100MB/s disk.speed = 100000000 -disk.mttf.min = 20000 -disk.mttf.max = 30000 -disk.mttf.step = interval:10000 - -systems = san;target +# 3 months: 7776000 # RAID: None cluster.node.osds = 1 cluster.raid = 1+0 cluster.replicas = 4 -cluster.mttc = 20000 +cluster.mttc = 7776000 # RAID: 4 @ 4+2 san.node.osds = 8 san.raid = 4+2 san.replicas = 2 -san.mttc = 20000 +san.mttc = 7776000 # RAID: 2 @ 3+1 target.node.osds = 2 target.raid = 3+1 target.replicas = 3 -target.mttc = 60000 +target.mttc = 7776000 diff --git a/sim/gobs/sweeps/disks-small.cfg b/sim/gobs/sweeps/disks-small.cfg index 5d47470..56b17c0 100644 --- a/sim/gobs/sweeps/disks-small.cfg +++ b/sim/gobs/sweeps/disks-small.cfg @@ -1,16 +1,7 @@ -mttr.reboot = 10 -mttr.disk = 50 - disk.size = 500000 disk.speed = 3000 -disk.mttf.min = 6000 -disk.mttf.max = 10000 -disk.mttf.step = interval:4000 - -systems = san;target - # RAID: None cluster.node.osds = 1 cluster.raid = 1+0 diff --git a/sim/gobs/sweeps/dl01-full.cfg b/sim/gobs/sweeps/dl01-full.cfg new file mode 100644 index 0000000..5e3fad1 --- /dev/null +++ b/sim/gobs/sweeps/dl01-full.cfg @@ -0,0 +1,33 @@ + +# Simulator configuration + +# Basics: + +include = disks-full.cfg fs_pdsi_1EB_chunky.cfg faults_real_range_dl01.cfg \ + user00.cfg dl_output.cfg + +# Bit-length of addresses: +B = 32 + +systems = san;target + +# 1 year: 31536000 +simulator.duration = 3153600 + +plot.blackandwhite = true + +# Clear some memory after each simulator run +sweep.free = true + +# Do a fsck +fsck = true +# Check for stray objects +strays = true + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/dl01-full.zsh b/sim/gobs/sweeps/dl01-full.zsh new file mode 100755 index 0000000..ffbae45 --- /dev/null +++ b/sim/gobs/sweeps/dl01-full.zsh @@ -0,0 +1,9 @@ +#!/bin/zsh + +NAME="dl01-full" +SEED="seed=821" +CLASS=gobs.sweep.x.DataLoss01 + +SWEEPDIR=$( dirname $0 ) + +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/dl01-medium.cfg b/sim/gobs/sweeps/dl01-medium.cfg index 820d382..9980cdb 100644 --- a/sim/gobs/sweeps/dl01-medium.cfg +++ b/sim/gobs/sweeps/dl01-medium.cfg @@ -3,16 +3,18 @@ # Basics: -include = system_30-100.cfg faults-range02.cfg user00.cfg \ - fs_pdsi_medium.cfg dl_output.cfg +include = disks-medium.cfg fs_pdsi_medium.cfg faults_medium_range_dl01.cfg \ + user00.cfg dl_output.cfg # Bit-length of addresses: B = 32 -fault.count = 30 -fault.dataloss = 0.5 +systems = san;target -node.diskspernode = 1 +# fault.count = 100 +simulator.duration = 31536000 + +plot.blackandwhite = true # Clear some memory after each simulator run sweep.free = true @@ -22,18 +24,10 @@ fsck = true # Check for stray objects strays = true -### Choose a PlacementScheme implementation: - -#impl.placement = Kademlia -impl.placement = Nearest -#impl.placement = VertexGroups - vertexgroups.prefix = 6 # Replica selection: See Replicated.locateSource() replica.source = primary ### Choose a NodeScheme implementation: -# impl.nodefactory = RandomNodes impl.nodefactory = SimpleNodes - diff --git a/sim/gobs/sweeps/dl01-small.cfg b/sim/gobs/sweeps/dl01-small.cfg index 0afc246..c40d2a5 100644 --- a/sim/gobs/sweeps/dl01-small.cfg +++ b/sim/gobs/sweeps/dl01-small.cfg @@ -3,16 +3,17 @@ # Basics: -include = system_10-20.cfg faults-range01.cfg user00.cfg \ - fs_pdsi_mini.cfg dl_output.cfg +include = disks-small.cfg fs_pdsi_mini.cfg faults_small.cfg \ + user00.cfg dl_output.cfg # Bit-length of addresses: B = 16 -fault.count = 10 -fault.dataloss = 0.5 +systems = san;target -node.diskspernode = 1 +fault.count = 100 + +plot.blackandwhite = true # Clear some memory after each simulator run sweep.free = true @@ -22,18 +23,11 @@ fsck = true # Check for stray objects strays = true -### Choose a PlacementScheme implementation: - -#impl.placement = Kademlia -impl.placement = Nearest -#impl.placement = VertexGroups - vertexgroups.prefix = 6 # Replica selection: See Replicated.locateSource() replica.source = primary ### Choose a NodeScheme implementation: -# impl.nodefactory = RandomNodes impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/dl01.cfg b/sim/gobs/sweeps/dl01.cfg index 5ab20d2..6691f99 100644 --- a/sim/gobs/sweeps/dl01.cfg +++ b/sim/gobs/sweeps/dl01.cfg @@ -11,7 +11,7 @@ include = dl_output.cfg sweep.free = true # Rebuild traffic over time analysis: -plot.traffic.interval = 10 +# plot.traffic.interval = 10 ### Parameters for GOBS Simulator run diff --git a/sim/gobs/sweeps/dl_output.cfg b/sim/gobs/sweeps/dl_output.cfg index c9372d3..f26b687 100644 --- a/sim/gobs/sweeps/dl_output.cfg +++ b/sim/gobs/sweeps/dl_output.cfg @@ -4,13 +4,13 @@ # Print all properties at simulation start print.properties = true # Print nodes and objects at startup (true/false) -print.initialnodes = true +print.initialnodes = false # Print nodes and objects after each rebuild (true/false) -print.nodes = true +print.nodes = false # Print object tranfers during rebuilds (true/false) -print.rebuilds = true +print.rebuilds = false # Print file information (true/false) -print.files = true +print.files = false print.filesobjects = false # Print load change statistics (true/false) print.loaddiffs = true diff --git a/sim/gobs/sweeps/faults_medium.cfg b/sim/gobs/sweeps/faults_medium.cfg new file mode 100644 index 0000000..7c01ad7 --- /dev/null +++ b/sim/gobs/sweeps/faults_medium.cfg @@ -0,0 +1,10 @@ + +mttr.reboot = 3600 +mttr.disk = 86400 + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 315360000 +disk.mttf.max = 315360000 +disk.mttf.step = interval:10000 diff --git a/sim/gobs/sweeps/faults_medium_range_dl01.cfg b/sim/gobs/sweeps/faults_medium_range_dl01.cfg new file mode 100644 index 0000000..9fbae40 --- /dev/null +++ b/sim/gobs/sweeps/faults_medium_range_dl01.cfg @@ -0,0 +1,18 @@ + +mttr.reboot = 3600 +mttr.disk = 86400 + +# Disk MTTF: 1-100 years. + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +#disk.mttf.min = 31536000 +#disk.mttf.max = 31536000000 +#disk.mttf.step = literal:31536000,31536000000 +#exps + +# Experimental short values +disk.mttf.min = 3153 +disk.mttf.max = 3153600 +disk.mttf.step = literal:31536000,31536000000 diff --git a/sim/gobs/sweeps/faults_real.cfg b/sim/gobs/sweeps/faults_real.cfg new file mode 100644 index 0000000..d2892bb --- /dev/null +++ b/sim/gobs/sweeps/faults_real.cfg @@ -0,0 +1,10 @@ + +mttr.reboot = 3600 +mttr.disk = 86400 + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 315360000 +disk.mttf.max = 315360000 +disk.mttf.step = interval:10000000 diff --git a/sim/gobs/sweeps/faults_real_range_dl01.cfg b/sim/gobs/sweeps/faults_real_range_dl01.cfg new file mode 100644 index 0000000..f0e0620 --- /dev/null +++ b/sim/gobs/sweeps/faults_real_range_dl01.cfg @@ -0,0 +1,20 @@ + +mttr.reboot = 3600 +mttr.disk = 3600 + +# Disk MTTF: 1-100 years. + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 31536000 +disk.mttf.max = 3153600000 +disk.mttf.step = literal:31536000,31536000000 + +# Artificially short +disk.mttf.step = literal:3153600,3153600000 + +# 3 months: 7776000 +cluster.mttc = 7776000 +san.mttc = 7776000 +target.mttc = 7776000 diff --git a/sim/gobs/sweeps/faults_real_zaps_only.cfg b/sim/gobs/sweeps/faults_real_zaps_only.cfg new file mode 100644 index 0000000..74eab4a --- /dev/null +++ b/sim/gobs/sweeps/faults_real_zaps_only.cfg @@ -0,0 +1,16 @@ + +mttr.reboot = 3600 +mttr.disk = 86400 + +# 1 month: 25920000 +# 10 years: 315360000 +# M K +disk.mttf.min = 315360000 +disk.mttf.max = 315360000 +disk.mttf.step = interval:10000000 + +# 3 months: 7776000 +cluster.mttc = 7776000000 +san.mttc = 7776000000 +target.mttc = 7776000000 + diff --git a/sim/gobs/sweeps/faults_small.cfg b/sim/gobs/sweeps/faults_small.cfg new file mode 100644 index 0000000..6451ca3 --- /dev/null +++ b/sim/gobs/sweeps/faults_small.cfg @@ -0,0 +1,7 @@ + +mttr.reboot = 10 +mttr.disk = 50 + +disk.mttf.min = 10000 +disk.mttf.max = 20000 +disk.mttf.step = interval:4000 diff --git a/sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg b/sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg new file mode 100644 index 0000000..50ced8f --- /dev/null +++ b/sim/gobs/sweeps/fs_pdsi_1EB_chunky.cfg @@ -0,0 +1,11 @@ + +# FS PDSI 1EB - CHUNKY + +B = 32 + +# Number of files: +files = 20 +# FileFactory implementation PlainFiles +impl.filefactory = PDSIFiles +# 20% size 0, 60% size 1K-100K, 20% size 100K-10M +file.pdsi.points = 0.1,1K;0.1,4T;0.2,10P;1.0,30P diff --git a/sim/gobs/sweeps/fs_pdsi_medium.cfg b/sim/gobs/sweeps/fs_pdsi_medium.cfg index dadaef7..7cc56e9 100644 --- a/sim/gobs/sweeps/fs_pdsi_medium.cfg +++ b/sim/gobs/sweeps/fs_pdsi_medium.cfg @@ -5,4 +5,4 @@ impl.filefactory = PDSIFiles # Number of files: files = 50 # 10% size 0, 90% size 4TB-1PB -file.pdsi.points = 0.1,10G;0.6,100G;1.0,1T +file.pdsi.points = 0.1,10G;0.6,100G;1.0,2T diff --git a/sim/gobs/sweeps/helpers.zsh b/sim/gobs/sweeps/helpers.zsh new file mode 100644 index 0000000..940504d --- /dev/null +++ b/sim/gobs/sweeps/helpers.zsh @@ -0,0 +1,18 @@ + +tformat() +# Convert seconds to hh:mm:ss +{ + local -Z 2 T=$1 + local -Z 2 M + + if (( T <= 60 )) + then + print "${T}" + elif (( T <= 3600 )) + then + M=$(( T/60 )) + print "${M}:$( tformat $(( T%60 )) )" + else + print "$(( T/3600 )):$( tformat $(( T%3600 )) )" + fi +} diff --git a/sim/gobs/sweeps/kpv05.cfg b/sim/gobs/sweeps/kpv05.cfg index 9c0337c..0df9c8b 100644 --- a/sim/gobs/sweeps/kpv05.cfg +++ b/sim/gobs/sweeps/kpv05.cfg @@ -1,8 +1,6 @@ ### Kademlia vs. PAST vs. Vertex Groups 05 -include = kpv_output.cfg - ### Choose a PlacementScheme implementation: # Chosen by sweep driver. diff --git a/sim/gobs/sweeps/rb01-long.cfg b/sim/gobs/sweeps/rb01-long.cfg new file mode 100644 index 0000000..b3ab945 --- /dev/null +++ b/sim/gobs/sweeps/rb01-long.cfg @@ -0,0 +1,41 @@ + +# Simulator configuration + +# Basics: + +include = disks-full-zaps-only.cfg user00.cfg \ + fs_pdsi_1EB_chunky.cfg rb_output.cfg + +systems = target + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 86400 +plot.blackandwhite = true +plot.width = 900 +plot.timeunit = days + +### Parameters for GOBS Simulator run + +### General parameters: + +# Bit-length of addresses: +B = 20 + +# Number of faults: +# 1 month: 2592000 +# 1 month: 31536000 +# 10 years: 315360000 +# M K +simulator.duration = 31536000 +#simulator.duration = 86400 + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/rb01-long.zsh b/sim/gobs/sweeps/rb01-long.zsh new file mode 100755 index 0000000..1cf2593 --- /dev/null +++ b/sim/gobs/sweeps/rb01-long.zsh @@ -0,0 +1,11 @@ +#!/bin/zsh + +# Long run for plot along top + +NAME="rb01-long" +SEED="seed=822" +CLASS=gobs.sweep.x.Rebuild01 + +SWEEPDIR=$( dirname $0 ) + +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/rb01-medium.cfg b/sim/gobs/sweeps/rb01-medium.cfg index 1f4c671..c4e7fd8 100644 --- a/sim/gobs/sweeps/rb01-medium.cfg +++ b/sim/gobs/sweeps/rb01-medium.cfg @@ -8,9 +8,13 @@ include = disks-medium.cfg user00.cfg \ # Statistics: +systems = target + # Rebuild traffic over time analysis: -plot.traffic.interval = 10 +plot.traffic.interval = 3600 plot.blackandwhite = true +plot.width = 900 +plot.timeunit = hours ### Parameters for GOBS Simulator run @@ -20,7 +24,8 @@ plot.blackandwhite = true B = 20 # Number of faults: -simulator.duration = 1000 +# 1 day +simulator.duration = 86400 vertexgroups.prefix = 6 # Replica selection: See Replicated.locateSource() diff --git a/sim/gobs/sweeps/rb02-full.cfg b/sim/gobs/sweeps/rb02-full.cfg new file mode 100644 index 0000000..872fc11 --- /dev/null +++ b/sim/gobs/sweeps/rb02-full.cfg @@ -0,0 +1,32 @@ + +# Simulator configuration + +# Basics: + +include = disks-full.cfg fs_pdsi_1EB_chunky.cfg faults_real_zaps_only.cfg \ + user00.cfg rb_output.cfg + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 3600 +plot.blackandwhite = true +plot.timeunit = hours + +regimen.skip = false + +systems = san;target + +# Bit-length of addresses: +B = 32 + +fault.count = 10 + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/rb02-full.zsh b/sim/gobs/sweeps/rb02-full.zsh new file mode 100755 index 0000000..a4d6e5d --- /dev/null +++ b/sim/gobs/sweeps/rb02-full.zsh @@ -0,0 +1,9 @@ +#!/bin/zsh + +NAME="rb02-full" +SEED="seed=821" +CLASS=gobs.sweep.x.Rebuild02 + +SWEEPDIR=$( dirname $0 ) + +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/rb02-medium.cfg b/sim/gobs/sweeps/rb02-medium.cfg new file mode 100644 index 0000000..4182b82 --- /dev/null +++ b/sim/gobs/sweeps/rb02-medium.cfg @@ -0,0 +1,31 @@ + +# Simulator configuration + +# Basics: + +include = disks-medium-zaps-only.cfg fs_pdsi_medium.cfg \ + user00.cfg rb_output.cfg + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 3456000 +plot.blackandwhite = true +plot.timeunit = hours + +regimen.skip = false + +# Bit-length of addresses: +B = 20 + +fault.count = 10 +# simulator.duration = 86400 + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/rb02-medium.zsh b/sim/gobs/sweeps/rb02-medium.zsh new file mode 100755 index 0000000..1276c7b --- /dev/null +++ b/sim/gobs/sweeps/rb02-medium.zsh @@ -0,0 +1,9 @@ +#!/bin/zsh + +NAME="rb02-medium" +SEED="seed=821" +CLASS=gobs.sweep.x.Rebuild02 + +SWEEPDIR=$( dirname $0 ) + +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/rb02-small.cfg b/sim/gobs/sweeps/rb02-small.cfg index f9159ff..fe850d2 100644 --- a/sim/gobs/sweeps/rb02-small.cfg +++ b/sim/gobs/sweeps/rb02-small.cfg @@ -9,6 +9,7 @@ include = disks-small.cfg user00.cfg fs_pdsi_small.cfg rb_output.cfg # Rebuild traffic over time analysis: plot.traffic.interval = 50 +plot.blackandwhite = true regimen.skip = false @@ -26,4 +27,3 @@ replica.source = primary # impl.nodefactory = RandomNodes impl.nodefactory = SimpleNodes - diff --git a/sim/gobs/sweeps/rb_output.cfg b/sim/gobs/sweeps/rb_output.cfg index a997cd0..26a200a 100644 --- a/sim/gobs/sweeps/rb_output.cfg +++ b/sim/gobs/sweeps/rb_output.cfg @@ -4,11 +4,11 @@ # Print all properties at simulation start print.properties = true # Print nodes and objects at startup (true/false) -print.initialnodes = true +print.initialnodes = false # Print nodes and objects after each rebuild (true/false) -print.nodes = true +print.nodes = false # Print object tranfers during rebuilds (true/false) -print.rebuilds = true +print.rebuilds = false # Print file information (true/false) print.files = false print.filesobjects = false diff --git a/sim/gobs/sweeps/sweep.zsh b/sim/gobs/sweeps/sweep.zsh index 3629f70..d4edc90 100644 --- a/sim/gobs/sweeps/sweep.zsh +++ b/sim/gobs/sweeps/sweep.zsh @@ -5,12 +5,13 @@ GOBS=${SWEEPDIR}/.. source ${SWEEPDIR}/classpath.zsh source ${SWEEPDIR}/build.zsh +source ${SWEEPDIR}/helpers.zsh [[ ${N} == "" ]] && N=1 start=$( date +%s ) { - java -ea -Xmx1000M -cp ${CP} ${CLASS} \ + java -ea -Xmx4000M -cp ${CP} ${CLASS} \ -n ${N} -p sweeps ${NAME}.cfg ${SEED} -o ${NAME}.eps CODE=$? } >& ${NAME}.out @@ -23,4 +24,4 @@ else print "wrote: ${NAME}.out" fi -print "took: $(( stop - start ))" +print "took: $( tformat $(( stop - start )) )" diff --git a/sim/gobs/sweeps/system_100,200.cfg b/sim/gobs/sweeps/system_100,200.cfg new file mode 100644 index 0000000..02bd8cb --- /dev/null +++ b/sim/gobs/sweeps/system_100,200.cfg @@ -0,0 +1,24 @@ + +### System size: servers and files/objects +### 20-1000 servers, selected exponentially +### 3000 files * 10 objects * 3 replicas = 90,000 objects + +# Number of servers: +nodes.min = 100 +nodes.max = 200 +nodes.step = literal:100,200 + +### General parameters: + +# Bit-length of addresses: +B = 32 + +node.osds = 1 +disk.size = 500000000 +disk.speed = 1000000 +disk.mttf = 1 +mttc = 1000 +mttr.reboot = 1 +mttr.disk = 1 +raid = 1+0 + diff --git a/sim/gobs/sweeps/system_100,600.cfg b/sim/gobs/sweeps/system_100,600.cfg index e3af48b..ec6422e 100644 --- a/sim/gobs/sweeps/system_100,600.cfg +++ b/sim/gobs/sweeps/system_100,600.cfg @@ -12,22 +12,6 @@ nodes.step = literal:100,600 # Bit-length of addresses: B = 32 -# Number of files: -files = 100 - -# FileFactory implementation PlainFiles -impl.filefactory = PlainFiles -# File size: -file.size = 14 -# Strip size: -file.strip = 4 -# File/object replicas: -# file.replicas = 3 -# Number of objects per file: -file.width = 10 - -mttf = 1 -mttr = 1 node.osds = 1 disk.size = 500000000 diff --git a/sim/gobs/sweeps/system_100-600.cfg b/sim/gobs/sweeps/system_100-600.cfg index e6c2192..3d33389 100644 --- a/sim/gobs/sweeps/system_100-600.cfg +++ b/sim/gobs/sweeps/system_100-600.cfg @@ -12,22 +12,6 @@ nodes.step = exps # Bit-length of addresses: B = 32 -# Number of files: -files = 100 - -# FileFactory implementation PlainFiles -impl.filefactory = PlainFiles -# File size: -file.size = 14 -# Strip size: -file.strip = 4 -# File/object replicas: -# file.replicas = 3 -# Number of objects per file: -file.width = 10 - -mttf = 1 -mttr = 1 node.osds = 1 disk.size = 500000000 diff --git a/sim/gobs/sweeps/system_100-700.cfg b/sim/gobs/sweeps/system_100-700.cfg new file mode 100644 index 0000000..97ab861 --- /dev/null +++ b/sim/gobs/sweeps/system_100-700.cfg @@ -0,0 +1,23 @@ + +### System size: servers and files/objects +### 20-1000 servers, selected exponentially +### 3000 files * 10 objects * 3 replicas = 90,000 objects + +# Number of servers: +nodes.min = 100 +nodes.max = 700 +nodes.step = exps + +### General parameters: + +# Bit-length of addresses: +B = 32 + +node.osds = 1 +disk.size = 500000000 +disk.speed = 1000000 +disk.mttf = 1 +mttc = 1000 +mttr.reboot = 1 +mttr.disk = 1 +raid = 1+0 diff --git a/sim/gobs/sweeps/system_600.cfg b/sim/gobs/sweeps/system_600.cfg index c476879..0217462 100644 --- a/sim/gobs/sweeps/system_600.cfg +++ b/sim/gobs/sweeps/system_600.cfg @@ -4,7 +4,7 @@ ### 3000 files * 10 objects * 3 replicas = 90,000 objects # Number of servers: -nodes.min = 100 +nodes.min = 600 nodes.max = 600 nodes.step = literal:600 @@ -26,9 +26,6 @@ file.strip = 4 # Number of objects per file: file.width = 10 -mttf = 1 -mttr = 1 - node.osds = 1 disk.size = 500000000 disk.speed = 1000000 @@ -37,4 +34,3 @@ mttc = 1000 mttr.reboot = 1 mttr.disk = 1 raid = 1+0 -fault.count = 1 hooks/post-receive -- Triton Repository

1 0

Triton Repository branch, master, updated. 7c8131a56d34d2e3924aa15f83493f25afc9f022
by noreply＠mcs.anl.gov 09 Mar '10

09 Mar '10

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via 7c8131a56d34d2e3924aa15f83493f25afc9f022 (commit) via 5e4b5a847362c886b3595cc3ffe838fa7bf59566 (commit) from 201961c65c83dbe6444f7403e40e1221c5220189 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 7c8131a56d34d2e3924aa15f83493f25afc9f022 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 17:16:22 2010 -0500 fix suspected typo in triton_error_to_strings() commit 5e4b5a847362c886b3595cc3ffe838fa7bf59566 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 17:16:06 2010 -0500 experimenting with error codes ----------------------------------------------------------------------- Summary of changes: code/src/common/triton-error.c | 2 +- code/src/versioned-osd/prototype/tests/vosd1.ae | 22 +++++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) Diff of changes: diff --git a/code/src/common/triton-error.c b/code/src/common/triton-error.c index d2b97a9..8127087 100644 --- a/code/src/common/triton-error.c +++ b/code/src/common/triton-error.c @@ -145,7 +145,7 @@ triton_ret_t triton_error_to_strings(triton_ret_t error, triton_string_t **strin assert(count && strings); *count = error->count + 1; strs = malloc(sizeof(triton_string_t) * (*count)); - if(!*strings) + if(!strs) { return TRITON_ERR_NOMEM; } diff --git a/code/src/versioned-osd/prototype/tests/vosd1.ae b/code/src/versioned-osd/prototype/tests/vosd1.ae index 4de1478..f6a26d6 100644 --- a/code/src/versioned-osd/prototype/tests/vosd1.ae +++ b/code/src/versioned-osd/prototype/tests/vosd1.ae @@ -6,6 +6,8 @@ #include "src/aesop/aesop.h" #include "src/versioned-osd/prototype/versioned-osd.hae" +static void triton_perror(const char *s, triton_ret_t ret); + int main(int argc, char *argv[]) { int iret; @@ -15,22 +17,22 @@ int main(int argc, char *argv[]) ret = triton_debug_init(); if(ret != TRITON_SUCCESS) { - /* TODO: handle this properly, print message, etc. */ - assert(0); + triton_perror("triton_debug_init", ret); + return(-1); } - ret = triton_debug_stderr_enable("all"); + ret = triton_debug_stdout_enable("all"); if(ret != TRITON_SUCCESS) { - /* TODO: handle this properly, print message, etc. */ - assert(0); + triton_perror("triton_stdout_enable", ret); + return(-1); } ret = vosd_init("/tmp/cosd"); if(ret != TRITON_SUCCESS) { - /* TODO: handle this properly, print message, etc. */ - assert(0); + triton_perror("vosd_init", ret); + return(-1); } iret = ae_context_create(&ctx, 1, "bdb"); @@ -55,6 +57,12 @@ int main(int argc, char *argv[]) return 0; } +static void triton_perror(const char *s, triton_ret_t ret) +{ + /* TODO: fix this */ + fprintf(stderr, "%s: I have no clue how to print this error.\n", s); +} + /* * Local variables: * c-indent-level: 4 hooks/post-receive -- Triton Repository

1 0

Triton Repository branch, master, updated. 201961c65c83dbe6444f7403e40e1221c5220189
by noreply＠mcs.anl.gov 09 Mar '10

09 Mar '10

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via 201961c65c83dbe6444f7403e40e1221c5220189 (commit) via 0b8b87ae6c8707a6691ee63139477ce23eff7d7c (commit) via 7cb6f958c59fbf71e50f66531ecc670afd9dd474 (commit) from 0a7f810a04631a2c81b0a29a110ec6b4597d8fcb (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 201961c65c83dbe6444f7403e40e1221c5220189 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 16:58:23 2010 -0500 experimenting with debugging api commit 0b8b87ae6c8707a6691ee63139477ce23eff7d7c Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 16:56:26 2010 -0500 successful return code for debug_init() commit 7cb6f958c59fbf71e50f66531ecc670afd9dd474 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 16:50:35 2010 -0500 error handling for debug register without init ----------------------------------------------------------------------- Summary of changes: code/src/common/triton-debug.c | 9 ++++++++- code/src/common/triton-error.c | 3 ++- code/src/common/triton-error.h | 1 + .../prototype/bdb-resource/bdb-resource.c | 12 ++++++++++-- code/src/versioned-osd/prototype/tests/vosd1.ae | 18 +++++++++++++++++- 5 files changed, 38 insertions(+), 5 deletions(-) Diff of changes: diff --git a/code/src/common/triton-debug.c b/code/src/common/triton-debug.c index 61c8d7b..7a201aa 100644 --- a/code/src/common/triton-debug.c +++ b/code/src/common/triton-debug.c @@ -3,7 +3,7 @@ #include "src/common/triton-hash.h" #include <stdio.h> -static struct triton_hash_table *triton_debug_mask_table; +static struct triton_hash_table *triton_debug_mask_table = NULL; uint64_t triton_debug_enabled_major = 0; uint32_t triton_debug_enabled_all = 0; @@ -36,6 +36,7 @@ triton_ret_t triton_debug_init(void) { return TRITON_ERR_NOMEM; } + return(TRITON_SUCCESS); } static triton_ret_t triton_idebug_add_mask(int major, char *str, uint32_t *value) @@ -43,7 +44,13 @@ static triton_ret_t triton_idebug_add_mask(int major, char *str, uint32_t *value uint32_t tmpval; struct triton_hash_link *llink; struct triton_debug_mask_entry *entry; + triton_mutex_lock(&triton_debug_mutex); + if(!triton_debug_mask_table) + { + triton_mutex_unlock(&triton_debug_mutex); + return(TRITON_ERR_NO_INIT); + } llink = triton_hash_search(triton_debug_mask_table, str); triton_mutex_unlock(&triton_debug_mutex); if(llink) diff --git a/code/src/common/triton-error.c b/code/src/common/triton-error.c index 4bd2103..d2b97a9 100644 --- a/code/src/common/triton-error.c +++ b/code/src/common/triton-error.c @@ -24,7 +24,8 @@ TRITON_ERROR_DEF(ERR_NOSYS, 5, "Not implemented"); TRITON_ERROR_DEF(ERR_OVERFLOW, 6, "Overflow"); TRITON_ERROR_DEF(ERR_SERVICE_NOTFOUND, 7, "Service not found"); TRITON_ERROR_DEF(ERR_MSG_METHOD_NOTFOUND, 8, "The message method was not found"); -TRITON_ERROR_DEF(ERR_AGAIN, 9, "Out of memory"); +TRITON_ERROR_DEF(ERR_AGAIN, 9, "Resource temporarily unavailable"); +TRITON_ERROR_DEF(ERR_NO_INIT, 10, "Component not initialized"); /* Used for errors where no mapping from an error number exists */ TRITON_ERROR_DEF(ERR_UNKNOWN, 4242, "Unknown"); diff --git a/code/src/common/triton-error.h b/code/src/common/triton-error.h index 6e61701..b4699da 100644 --- a/code/src/common/triton-error.h +++ b/code/src/common/triton-error.h @@ -59,6 +59,7 @@ TRITON_ERROR_DECL(ERR_OVERFLOW); TRITON_ERROR_DECL(ERR_SERVICE_NOTFOUND); TRITON_ERROR_DECL(ERR_MSG_METHOD_NOTFOUND); TRITON_ERROR_DECL(ERR_AGAIN); +TRITON_ERROR_DECL(ERR_NO_INIT); TRITON_ERROR_DECL(ERR_UNKNOWN); diff --git a/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c b/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c index 4e219bb..61c0c24 100644 --- a/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c +++ b/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c @@ -22,6 +22,7 @@ #include "src/common/triton-hash.h" #include "src/versioned-osd/prototype/bdb-resource/bdb-resource.h" #include "src/common/triton-error.h" +#include "src/common/triton-debug.h" #define BDB_DEFAULT_OPCACHE_SIZE 1024 @@ -31,6 +32,7 @@ static triton_mutex_t bdb_mutex = TRITON_MUTEX_INITIALIZER; static int triton_bdb_resource_id; static enum progress_mode triton_bdb_progress_mode = TRITON_PROG_NONE; +static triton_debug_mask_t bdb_r_mask; /* counter used by poll function in thread-per-op case to know if ops have * finished since last poll @@ -261,6 +263,13 @@ triton_ret_t bdb_init(enum progress_mode mode) { triton_ret_t ret; + /* register a debugging mask for use in this resource */ + ret = triton_debug_add_mask("bdb.resource", &bdb_r_mask); + if(ret != TRITON_SUCCESS) + { + return(ret); + } + ae_ops_init(&bdb_oplist); ret = TRITON_OPCACHE_INIT(struct bdb_op, op, BDB_DEFAULT_OPCACHE_SIZE, @@ -299,8 +308,7 @@ void bdb_finalize(void) ae_resource_unregister(triton_bdb_resource_id); ae_opcache_destroy(bdb_opcache); - /* TODO: make this a debugging message */ - printf("deadlock_counter: %lu\n", deadlock_counter); + triton_debug(bdb_r_mask, "bdb deadlock_counter: %lu\n", deadlock_counter); return; } diff --git a/code/src/versioned-osd/prototype/tests/vosd1.ae b/code/src/versioned-osd/prototype/tests/vosd1.ae index 8fd1123..4de1478 100644 --- a/code/src/versioned-osd/prototype/tests/vosd1.ae +++ b/code/src/versioned-osd/prototype/tests/vosd1.ae @@ -10,9 +10,22 @@ int main(int argc, char *argv[]) { int iret; triton_ret_t ret; - ae_context_t ctx; + ret = triton_debug_init(); + if(ret != TRITON_SUCCESS) + { + /* TODO: handle this properly, print message, etc. */ + assert(0); + } + + ret = triton_debug_stderr_enable("all"); + if(ret != TRITON_SUCCESS) + { + /* TODO: handle this properly, print message, etc. */ + assert(0); + } + ret = vosd_init("/tmp/cosd"); if(ret != TRITON_SUCCESS) { @@ -36,6 +49,9 @@ int main(int argc, char *argv[]) vosd_finalize(); + /* TODO: where is this function? */ + /* triton_debug_finalize(); */ + return 0; } hooks/post-receive -- Triton Repository

1 0

Triton Repository branch, master, updated. 0a7f810a04631a2c81b0a29a110ec6b4597d8fcb
by noreply＠mcs.anl.gov 09 Mar '10

09 Mar '10

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via 0a7f810a04631a2c81b0a29a110ec6b4597d8fcb (commit) from 5eebf775924b7106986643717194c10cbd6c08d7 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 0a7f810a04631a2c81b0a29a110ec6b4597d8fcb Author: Samuel Lang <slang(a)bblogin.mcs.anl.gov> Date: Tue Mar 9 15:44:39 2010 -0600 fix op callback stuff ----------------------------------------------------------------------- Summary of changes: code/src/aesop/op.h | 5 +++-- code/src/common/resources/scheduling/sched.c | 6 +++--- code/src/common/resources/timer/timer.c | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) Diff of changes: diff --git a/code/src/aesop/op.h b/code/src/aesop/op.h index b8d6493..bf1a626 100644 --- a/code/src/aesop/op.h +++ b/code/src/aesop/op.h @@ -6,7 +6,7 @@ typedef struct ae_op { - void (*callback)(void *ptr, int ret); + void *callback; int (*op_worker)(struct ae_op* op); void *user_ptr; ae_hints_t hints; @@ -21,7 +21,8 @@ typedef struct ae_op ((_op != NULL) ? \ ((_type *)((char *)(_op) - (unsigned long)((&((_type *)0)->_member)))) : NULL) -#define ae_invoke_callback(_op, _result) (_op)->callback((_op)->user_ptr, _result) +#define ae_invoke_callback(_op, __ret_type, _result) \ + ((void (*)(void *, __ret_type))(_op)->callback)((_op)->user_ptr, _result) #define ae_op_fill(_op, _cb, _up, _hints, _ctx) \ do { \ diff --git a/code/src/common/resources/scheduling/sched.c b/code/src/common/resources/scheduling/sched.c index 5a8349b..e45704f 100644 --- a/code/src/common/resources/scheduling/sched.c +++ b/code/src/common/resources/scheduling/sched.c @@ -109,7 +109,7 @@ int triton_sched_notify_all(triton_sched_t *sched) while(!ae_ops_empty(&sched->opqueue)) { op = ae_ops_dequeue(&sched->opqueue); - ae_invoke_callback(op, 0); + ae_invoke_callback(op, int, 0); result = ae_op_entry(op, triton_sched_id, op); if(result->internal) { @@ -135,7 +135,7 @@ int triton_sched_notify_next(triton_sched_t *sched) if(!ae_ops_empty(&sched->opqueue)) { op = ae_ops_dequeue(&sched->opqueue); - ae_invoke_callback(op, 0); + ae_invoke_callback(op, int, 0); result = ae_op_entry(op, triton_sched_id, op); if(result->internal) { @@ -158,7 +158,7 @@ int triton_sched_notify_specific(triton_sched_t *sched, triton_sched_id *id) sched->innotify = 1; ae_ops_del(&id->op); - ae_invoke_callback(&id->op, 0); + ae_invoke_callback(&id->op, int, 0); if(id->internal) { ae_opcache_put(sched_opcache, &id->op); diff --git a/code/src/common/resources/timer/timer.c b/code/src/common/resources/timer/timer.c index 1c3ddab..8b93a48 100644 --- a/code/src/common/resources/timer/timer.c +++ b/code/src/common/resources/timer/timer.c @@ -141,7 +141,7 @@ static int triton_timer_poll(ae_context_t context, int millisecs) ret = nanosleep(&ts_diff, &rem); if(ret != 0) { - ae_invoke_callback(gop, errno); + ae_invoke_callback(gop, int, errno); triton_mutex_unlock(&timer_mutex); return -EINTR; } hooks/post-receive -- Triton Repository

1 0

Triton Repository branch, master, updated. 5eebf775924b7106986643717194c10cbd6c08d7
by noreply＠mcs.anl.gov 09 Mar '10

09 Mar '10

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via 5eebf775924b7106986643717194c10cbd6c08d7 (commit) via 0a792f211f1648f6d5066f2a53d55e1547f1afd2 (commit) from 8f21c162e235f7e2c62e965eaa98ee03bd8beb98 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 5eebf775924b7106986643717194c10cbd6c08d7 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 16:34:21 2010 -0500 bug fix and error handling to vosd test stub commit 0a792f211f1648f6d5066f2a53d55e1547f1afd2 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 16:33:16 2010 -0500 fix memory bug in ae_opcache_destroy ----------------------------------------------------------------------- Summary of changes: code/src/aesop/opcache.c | 2 +- code/src/versioned-osd/prototype/tests/vosd1.ae | 30 +++++++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) Diff of changes: diff --git a/code/src/aesop/opcache.c b/code/src/aesop/opcache.c index 7193ed2..9bee3fd 100644 --- a/code/src/aesop/opcache.c +++ b/code/src/aesop/opcache.c @@ -73,8 +73,8 @@ void ae_opcache_destroy(ae_opcache_t cache) { free(cache->array[i]); } - free(cache); triton_mutex_unlock(&cache->mutex); + free(cache); return; } diff --git a/code/src/versioned-osd/prototype/tests/vosd1.ae b/code/src/versioned-osd/prototype/tests/vosd1.ae index 0687ef6..8fd1123 100644 --- a/code/src/versioned-osd/prototype/tests/vosd1.ae +++ b/code/src/versioned-osd/prototype/tests/vosd1.ae @@ -1,21 +1,41 @@ #include <stdio.h> #include <errno.h> +#include <assert.h> +#include <stdio.h> #include "src/aesop/aesop.h" #include "src/versioned-osd/prototype/versioned-osd.hae" int main(int argc, char *argv[]) { + int iret; + triton_ret_t ret; + ae_context_t ctx; - vosd_init("/tmp/cosd"); + ret = vosd_init("/tmp/cosd"); + if(ret != TRITON_SUCCESS) + { + /* TODO: handle this properly, print message, etc. */ + assert(0); + } - ae_context_create(&ctx, 1, "vosd1"); + iret = ae_context_create(&ctx, 1, "bdb"); + if(iret != 0) + { + fprintf(stderr, "Error: ae_context_create returned %d\n", iret); + return(-1); + } - ae_context_destroy(ctx); - + iret = ae_context_destroy(ctx); + if(iret != 0) + { + fprintf(stderr, "Error: ae_context_destroy returned %d\n", iret); + return(-1); + } + vosd_finalize(); - + return 0; } hooks/post-receive -- Triton Repository

1 0

Triton Repository branch, master, updated. 8f21c162e235f7e2c62e965eaa98ee03bd8beb98
by noreply＠mcs.anl.gov 09 Mar '10

09 Mar '10

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via 8f21c162e235f7e2c62e965eaa98ee03bd8beb98 (commit) via 884cfd6d73225457377cfe0d8fdcee4941a6c658 (commit) from 945f6bf4c3db9b8994c702128433c8c758560e0c (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 8f21c162e235f7e2c62e965eaa98ee03bd8beb98 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 16:21:37 2010 -0500 add explicit -lpthread for vosd tests commit 884cfd6d73225457377cfe0d8fdcee4941a6c658 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 16:21:10 2010 -0500 spaces->tabs in cscope rule ----------------------------------------------------------------------- Summary of changes: code/Makefile.in | 4 ++-- .../src/versioned-osd/prototype/tests/module.mk.in | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) Diff of changes: diff --git a/code/Makefile.in b/code/Makefile.in index a02bfc4..ace9bdb 100644 --- a/code/Makefile.in +++ b/code/Makefile.in @@ -604,8 +604,8 @@ endif # rule for generating cscope information cscope: - find @SRC_ABSOLUTE_TOP@ -iname "*.[ch]" -o -iname "*.sm" \ + find @SRC_ABSOLUTE_TOP@ -iname "*.[ch]" -o -iname "*.sm" \ > $(srcdir)/cscope.files - ( cd @SRC_ABSOLUTE_TOP@; cscope -be -i @SRC_ABSOLUTE_TOP@/cscope.files ) + ( cd @SRC_ABSOLUTE_TOP@; cscope -be -i @SRC_ABSOLUTE_TOP@/cscope.files ) diff --git a/code/src/versioned-osd/prototype/tests/module.mk.in b/code/src/versioned-osd/prototype/tests/module.mk.in index 8942340..18ed3e8 100644 --- a/code/src/versioned-osd/prototype/tests/module.mk.in +++ b/code/src/versioned-osd/prototype/tests/module.mk.in @@ -1,3 +1,5 @@ DIR := src/versioned-osd/prototype/tests AETESTSRC += $(DIR)/vosd1.ae + +MODLIBS_$(DIR) = -lpthread hooks/post-receive -- Triton Repository

1 0

Triton Repository branch, master, updated. 945f6bf4c3db9b8994c702128433c8c758560e0c
by noreply＠mcs.anl.gov 09 Mar '10

09 Mar '10

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via 945f6bf4c3db9b8994c702128433c8c758560e0c (commit) from ac4fb113bebc7d01f7b72a70b71baf9f46af12db (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 945f6bf4c3db9b8994c702128433c8c758560e0c Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 16:13:00 2010 -0500 fix #include in versioned-osd ----------------------------------------------------------------------- Summary of changes: code/src/versioned-osd/prototype/versioned-osd.ae | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) Diff of changes: diff --git a/code/src/versioned-osd/prototype/versioned-osd.ae b/code/src/versioned-osd/prototype/versioned-osd.ae index d4f61b9..9653d53 100644 --- a/code/src/versioned-osd/prototype/versioned-osd.ae +++ b/code/src/versioned-osd/prototype/versioned-osd.ae @@ -15,7 +15,7 @@ #include "src/common/triton-base.h" #include "src/aesop/aesop.h" #include "src/common/triton-hash.h" -#include "src/versioned-osd/prototype/bdb-resource/bdb-resource.h" +#include "src/versioned-osd/prototype/bdb-resource/bdb-resource.hae" #include "src/versioned-osd/prototype/versioned-osd.hae" hooks/post-receive -- Triton Repository

1 0

Triton Repository branch, master, updated. ac4fb113bebc7d01f7b72a70b71baf9f46af12db
by noreply＠mcs.anl.gov 09 Mar '10

09 Mar '10

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via ac4fb113bebc7d01f7b72a70b71baf9f46af12db (commit) from cf64e10dd90ca3a21550b9d3e10ae73926ed8983 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit ac4fb113bebc7d01f7b72a70b71baf9f46af12db Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 15:47:30 2010 -0500 updates to new ae_opcache_complete_op() arguments ----------------------------------------------------------------------- Summary of changes: code/src/aesop/parser/tests/blocking/btest-impl.c | 14 +++++++------- .../prototype/bdb-resource/bdb-resource.c | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) Diff of changes: diff --git a/code/src/aesop/parser/tests/blocking/btest-impl.c b/code/src/aesop/parser/tests/blocking/btest-impl.c index 50f1df8..707cc78 100644 --- a/code/src/aesop/parser/tests/blocking/btest-impl.c +++ b/code/src/aesop/parser/tests/blocking/btest-impl.c @@ -60,7 +60,7 @@ static void *tctest1_threadfun(void *ptr) op = (struct ae_op *)ptr; bop = ae_op_entry(op, struct btest_op, op); *(bop->value) += 1; - ae_opcache_complete_op(test_opcache, &bop->op, 0); + ae_opcache_complete_op(test_opcache, &bop->op, int, 0); return NULL; } @@ -282,40 +282,40 @@ static int btest_poll(ae_context_t context, int ms) if(b) { *(b->value) += 1; - ae_opcache_complete_op(test_opcache, &b->op, 0); + ae_opcache_complete_op(test_opcache, &b->op, int, 0); } b = poll_list(&list2); if(b) { *(b->value) += 2; - ae_opcache_complete_op(test_opcache, &b->op, 0); + ae_opcache_complete_op(test_opcache, &b->op, int, 0); } b = poll_list(&list3); if(b) { *(b->value) += 3; - ae_opcache_complete_op(test_opcache, &b->op, 0); + ae_opcache_complete_op(test_opcache, &b->op, int, 0); } b = poll_list(&clist); if(b) { - ae_opcache_complete_op(test_opcache, &b->op, -1); + ae_opcache_complete_op(test_opcache, &b->op, int, -1); } b = poll_list(&rlist); if(b) { - ae_opcache_complete_op(test_opcache, &b->op, random()); + ae_opcache_complete_op(test_opcache, &b->op, int, random()); } s = poll_sleep_list(); if(s) { sleep(s->sleep); - ae_opcache_complete_op(sleep_opcache, &s->op, 0); + ae_opcache_complete_op(sleep_opcache, &s->op, int, 0); } return 0; } diff --git a/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c b/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c index 4d153bc..4e219bb 100644 --- a/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c +++ b/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c @@ -72,7 +72,7 @@ static void* thread_fn(void* foo) /* TODO: how do we convert triton_ret_t to int here? */ /* last argument should be triton_ret_t? */ - ae_opcache_complete_op(bdb_opcache, op, 0); + ae_opcache_complete_op(bdb_opcache, op, int, 0); /* wake up any pollers */ triton_mutex_lock(&completed_counter_mutex); @@ -218,7 +218,7 @@ static int bdb_poll(ae_context_t context, int millisecs) /* TODO: how do we convert triton_ret_t to int here? */ /* last argument should be triton_ret_t? */ - ae_opcache_complete_op(bdb_opcache, a_op, 0); + ae_opcache_complete_op(bdb_opcache, a_op, int, 0); return 0; } hooks/post-receive -- Triton Repository

1 0

Triton Repository branch, master, updated. cf64e10dd90ca3a21550b9d3e10ae73926ed8983
by noreply＠mcs.anl.gov 09 Mar '10

09 Mar '10

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via cf64e10dd90ca3a21550b9d3e10ae73926ed8983 (commit) from 44e8b3b6a9e89a9f20f1914e6da59abb2268782a (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit cf64e10dd90ca3a21550b9d3e10ae73926ed8983 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Mar 9 14:59:28 2010 -0500 minor fix to complete_op() return type usage ----------------------------------------------------------------------- Summary of changes: code/src/aesop/opcache.h | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) Diff of changes: diff --git a/code/src/aesop/opcache.h b/code/src/aesop/opcache.h index 5e3aea8..1a2ef7e 100644 --- a/code/src/aesop/opcache.h +++ b/code/src/aesop/opcache.h @@ -3,6 +3,7 @@ #define __OPCACHE_H__ #include "src/aesop/op.h" +#include "src/common/triton-error.h" typedef struct ae_opcache *ae_opcache_t; @@ -16,7 +17,7 @@ typedef struct ae_opcache *ae_opcache_t; #define ae_opcache_complete_op(__opcache, __op, __ret_type, __error_code) do { \ void (*__callback)(void *, __ret_type) = (__op)->callback; \ void* __user_ptr = (__op)->user_ptr; \ - int __saved_error_code = __error_code; \ + __ret_type __saved_error_code = __error_code; \ ae_opcache_put(__opcache, __op); \ __callback(__user_ptr, __saved_error_code); \ } while(0) hooks/post-receive -- Triton Repository

1 0