This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "".
The branch, master has been updated
via 1d24a5e649db31750462a2e9503ce0bcc58d6dd7 (commit)
from d534b334aa3d20f39aeab4293bbf87d25ffe16ee (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 1d24a5e649db31750462a2e9503ce0bcc58d6dd7
Author: Shane Snyder <ssnyder(a)login1.mcs.anl.gov>
Date: Sun Jun 22 12:53:20 2014 -0500
All content for swim paper added
-----------------------------------------------------------------------
Summary of changes:
papers/swim/REFERENCES.bib | 43 +++++
papers/swim/figs/swim-results.pdf | Bin 0 -> 7310 bytes
papers/swim/sim_results/swim.plt | 19 ++
papers/swim/sim_results/swim_results | 7 +
papers/swim/swim-report.tex | 309 +++++++++++++++++++++-------------
5 files changed, 258 insertions(+), 120 deletions(-)
create mode 100644 papers/swim/figs/swim-results.pdf
create mode 100644 papers/swim/sim_results/swim.plt
create mode 100644 papers/swim/sim_results/swim_results
Diff of changes:
diff --git a/papers/swim/REFERENCES.bib b/papers/swim/REFERENCES.bib
index 5f60b19..bbbcd92 100644
--- a/papers/swim/REFERENCES.bib
+++ b/papers/swim/REFERENCES.bib
@@ -169,3 +169,46 @@ language={English}
author={{Argonne National Laboratory}},
year={2014},
}
+
+@inproceedings{cope2011codes,
+ title={Codes: Enabling co-design of multilayer Exascale storage architectures},
+ author={Cope, Jason and Liu, Ning and Lang, Sam and Carns, Phil and Carothers, Chris and Ross, Robert},
+ booktitle={Proceedings of the Workshop on Emerging Supercomputing Technologies},
+ year={2011}
+}
+
+@inproceedings{liu2012role,
+ title={On the role of burst buffers in leadership-class storage systems},
+ author={Liu, Ning and Cope, Jason and Carns, Philip and Carothers, Christopher and Ross, Robert and Grider, Gary and Crume, Adam and Maltzahn, Carlos},
+ booktitle={Mass Storage Systems and Technologies (MSST), 2012 IEEE 28th Symposium on},
+ pages={1--11},
+ year={2012},
+ organization={IEEE}
+}
+
+@inproceedings{ross:2013,
+ author = {Barnes,Jr., Peter D. and Carothers, Christopher D. and Jefferson, David R. and LaPre, Justin M.},
+ title = {Warp Speed: Executing Time Warp on 1,966,080 Cores},
+ booktitle = {Proceedings of the 2013 ACM SIGSIM Conference on Principles of Advanced Discrete Simulation},
+ series = {SIGSIM-PADS '13},
+ year = {2013},
+ isbn = {978-1-4503-1920-1},
+ location = {Montr\&\#169;al, Qu\&\#233;bec, Canada},
+ pages = {327--336},
+ numpages = {10},
+ url = {http://doi.acm.org/10.1145/2486092.2486134},
+ doi = {10.1145/2486092.2486134},
+ acmid = {2486134},
+ publisher = {ACM},
+ address = {New York, NY, USA},
+ keywords = {blue gene/q, parallel discrete-event simulation, time warp},
+}
+
+@inproceedings{loggp,
+ title={LogGP: incorporating long messages into the LogP model—one step closer towards a realistic model for parallel computation},
+ author={Alexandrov, Albert and Ionescu, Mihai F and Schauser, Klaus E and Scheiman, Chris},
+ booktitle={Proceedings of the seventh annual ACM symposium on Parallel algorithms and architectures},
+ pages={95--105},
+ year={1995},
+ organization={ACM}
+}
diff --git a/papers/swim/figs/swim-results.pdf b/papers/swim/figs/swim-results.pdf
new file mode 100644
index 0000000..053bc84
Binary files /dev/null and b/papers/swim/figs/swim-results.pdf differ
diff --git a/papers/swim/sim_results/swim.plt b/papers/swim/sim_results/swim.plt
new file mode 100644
index 0000000..5c55258
--- /dev/null
+++ b/papers/swim/sim_results/swim.plt
@@ -0,0 +1,19 @@
+set term po eps color dashed 16 size 4in,3in
+set output "swim-results.eps"
+
+set xlabel "group size"
+set ylabel "time (secs)"
+
+set log x; set xtics (128, 256, 512, 1024, 2048, 4096)
+
+set key left top
+
+set style line 1 lt 1 lw 4
+set style line 2 lt 2 lw 4
+set style line 3 lt 3 lw 4
+set style line 4 lt 4 lw 4
+
+plot "swim_results" using 1:2 title "expected detection time" w linespoints ls 1, \
+ "swim_results" using 1:3 title "expected dissemination time" w linespoints ls 2, \
+ "swim_results" using 1:4 title "measured detection time" w linespoints ls 3, \
+ "swim_results" using 1:5 title "measured dissemination time" w linespoints ls 4
diff --git a/papers/swim/sim_results/swim_results b/papers/swim/sim_results/swim_results
new file mode 100644
index 0000000..0f2b8a3
--- /dev/null
+++ b/papers/swim/sim_results/swim_results
@@ -0,0 +1,7 @@
+## members ## t_detect (s) ## t_dissem (s) ## sim_detect (s) ## sim_dissem (s) ##
+ 128 .3178 .9726 .5787 .7200
+ 256 .3171 1.1104 .5387 .8600
+ 512 .3168 1.2485 .6187 .9400
+ 1024 .3166 1.3868 .5787 1.1000
+ 2048 .3165 1.5252 .5987 1.2200
+ 4096 .3164 1.6637 .4987 1.3400
diff --git a/papers/swim/swim-report.tex b/papers/swim/swim-report.tex
index 659f457..b8e446c 100644
--- a/papers/swim/swim-report.tex
+++ b/papers/swim/swim-report.tex
@@ -50,10 +50,11 @@
\usepackage{url}
\usepackage{multirow}
\usepackage{amsmath}
+\usepackage{threeparttable}
\begin{document}
-\title{SWIM Group Membership Protocol as a Distributed Storage System Fault Detector}
+\title{Tuning Fault Detection in Massively Distributed Storage Systems}
\author{\IEEEauthorblockN{Shane Snyder\IEEEauthorrefmark{1}} \\
\IEEEauthorblockA{\IEEEauthorrefmark{1}Argonne National Laboratory}
@@ -65,8 +66,7 @@
%% Abstract %%
%%%%%%%%%%%%%%
\begin{abstract}
-This work analyzes the SWIM protocol, a scalable failure detection protocol
-used in numerous distributed and peer-to-peer applications.
+
\end{abstract}
%%%%%%%%%%%%%%%%%%
@@ -77,7 +77,7 @@ used in numerous distributed and peer-to-peer applications.
As the scale of modern distributed systems continues to grow, so too does
the frequency of system component failures. Ensuring efficient and correct
behavior in the presence of such failures requires both a reliable fault
-detection mechanism and a suitable stategy for fault recovery.
+detection mechanism and a suitable strategy for fault recovery.
Some example distributed services which rely on efficient
and accurate fault detection include distributed storage
systems~\cite{Weil:2006,Lakshman:2010} and reliable multicast
@@ -135,7 +135,7 @@ probabilistically.
In this work, we analyze the efficiency and scalability of
SWIM~\cite{Das:2002} (Scalable Weakly-consistent Infection-style Process
Group Membership Protocol), a group membership protocol designed to improve
-upon the of traditional group membership protocols. SWIM achieves
+upon the performance of traditional group membership protocols. SWIM achieves
this through the use of a randomized, probe-based
failure detection mechanism coupled with an efficient and robust
gossip-style\footnote{Also referred to as infection-style or epidemic
@@ -153,13 +153,18 @@ low-latency environment to ensure that its performance and semantics are
appropriate for use within the overall fault response strategy of a high
performance storage system.
-% TODO update when we have a better idea of the organization/content
The rest of this paper is organized as follows:
In Section \ref{sec:background}, we summarize the SWIM group membership
-protocol specification, as well as pertinent protocol parameters. In
-Section \ref{sec:swim-params} we analyze how specific parameters affect
-the performance of the SWIM protocol and reason about how to configure
-these parameters to effectively detect faults in Triton.
+protocol specification, as well as pertinent protocol parameters. Section
+\ref{sec:triton-implications} gives background on the semantics required
+of an effective fault detection mechanism in Triton. In Section
+\ref{sec:swim-params} we analyze how specific parameters affect the
+performance of the SWIM protocol and reason about how to configure these
+parameters to effectively detect faults in Triton. In Section
+\ref{sec:sim-results} we provide some initial protocol simulation results
+to validate our SWIM model and to analyze its performance across a range
+of large-scale process groups. Finally, in Section \ref{sec:conclusion}
+we conclude with our findings, as well as avenues for future research.
\section{Background: SWIM}
\label{sec:background}
@@ -226,6 +231,7 @@ group as members randomly ping (and ack) each other, ultimately resulting
in a complete dissemination of the update.
\section{Implications of using SWIM within Triton}
+\label{sec:triton-implications}
A number of issues must be considered in order to ensure that Triton
can accomodate unique properties of the SWIM protocol (and vice versa).
@@ -262,15 +268,56 @@ conservative.
\section{SWIM Parameter Selection}
\label{sec:swim-params}
+\begin{table*}
+\centering
+\begin{threeparttable}[t]
+\caption{Effect of SWIM period length and suspect timeout on protocol performance.\tnote{\textdagger}}
+\begin{tabular}{c c c | c c c c c}
+\hline
+$T'$ & members & $s$ & average network load per member (messages/sec) &
+t\_detect & t\_dissem & t\_suspect & t\_converge \\
+\hline
+\multirow{6}{*}{.1 ms} & \multirow{2}{*}{1024} & 10 & \multirow{6}{*}{20976.0} &
+\multirow{6}{*}{.16 ms} & \multirow{2}{*}{.69 ms} & 1 ms & 1.85 ms \\
+& & 100 & & & & 10 ms & 10.85 ms \\
+& \multirow{2}{*}{2048} & 10 & & & \multirow{2}{*}{.76 ms} & 1 ms & 1.92 ms \\
+& & 100 & & & & 10 ms & 10.92 ms \\
+& \multirow{2}{*}{4096} & 10 & & & \multirow{2}{*}{.83 ms} & 1 ms & 1.99 ms \\
+& & 100 & & & & 10 ms & 10.99 ms \\
+\hline
+\multirow{6}{*}{10 ms} & \multirow{2}{*}{1024} & 10 & \multirow{6}{*}{209.8} &
+\multirow{6}{*}{15.91 ms} & \multirow{2}{*}{69.34 ms} & 100 ms & 185.25 ms \\
+& & 100 & & & & 1 s & 1.09 s \\
+& \multirow{2}{*}{2048} & 10 & & & \multirow{2}{*}{76.26 ms} & 100 ms & 192.17 ms \\
+& & 100 & & & & 1 s & 1.09 s \\
+& \multirow{2}{*}{4096} & 10 & & & \multirow{2}{*}{83.19 ms} & 100 ms & 199.1 ms \\
+& & 100 & & & & 1 s & 1.1 s \\
+\hline
+\multirow{6}{*}{1 s} & \multirow{2}{*}{1024} & 10 & \multirow{6}{*}{2.1} &
+\multirow{6}{*}{1.59 s} & \multirow{2}{*}{6.93 s} & 10 s & 18.52 s \\
+& & 100 & & & & 100 s & 108.52 s \\
+& \multirow{2}{*}{2048} & 10 & & & \multirow{2}{*}{7.63 s} & 10 s & 19.22 s \\
+& & 100 & & & & 100 s & 109.22 s \\
+& \multirow{2}{*}{4096} & 10 & & & \multirow{2}{*}{8.32 s} & 10 s & 19.91 s \\
+& & 100 & & & & 100 s & 109.91 s \\
+\hline
+\end{tabular}
+\begin{tablenotes}
+\item[\textdagger] Note: The analytical results here are calculated assuming a 1 \% chance
+of node failure and packet loss, as well as a subgroup size ($k$) of 1.
+\end{tablenotes}
+\end{threeparttable}
+\label{table:converge}
+\end{table*}
+
Before configuring SWIM for use as the fault detection component of Triton,
it is important to consider how its parameters affect the efficiency and
accuracy of fault detection in a massively distributed storage system. We
specifically focus on the protocol period length ($T'$) and the suspicion
-timeout ($s$, measured in protocol period time units) parameters, as they
-have the greatest affect on the performance of the protocol. Below, we
-outline some fundamental bounds on these parameters required either
-inherently by the protocol specification or externally by the Triton
-storage system:
+timeout ($s$) parameters, as they have the greatest affect on the
+performance of the protocol. Below, we outline some fundamental bounds on
+these parameters required either inherently by the protocol specification
+or externally by the Triton storage system:
\begin{itemize}
\item \textbf{Network RTT}: According to the overview of the SWIM protocol
@@ -298,7 +345,9 @@ responsiveness to storage system clients and to minimize the potential for
data loss, it is important to detect faults and respond to them in a timely
fashion. Thus, we can upper bound the fault detection time of the protocol
such that its duration is on the same order as the duration of the triage
-phase.
+phase. According to the simulation results in \cite{triton-rebuild-deliverable},
+triaging a 768 server storage system containing a billion raw data objects
+can take up to 30 seconds, depending on the object placement strategy.
\item \textbf{Transient Failure Sensitivity}: While it is generally
desirable to detect and respond to failures in a distributed system as
quickly as possible, this approach is likely too aggressive for the fault
@@ -312,26 +361,40 @@ bound on the total fault detection time to account for the maximum
transient failure duration we are willing to ignore.
\end{itemize}
+\begin{table*}[t]
+\caption{Default SWIM parameters chosen for Triton.}
+\centering
+\begin{tabular}{c c c | c c c c c}
+\hline
+$T'$ & members & $s$ & average network load per member (messages/sec) &
+t\_detect & t\_dissem & t\_suspect & t\_converge \\
+\hline
+200 ms & 2048 & 75 & 10.5 & 318 ms & 1.53 s & 15 s & 16.85 s \\
+\hline
+\end{tabular}
+\label{table:swim-params}
+\end{table*}
+
The first two bounds are clearly directly related to the protocol period
length, $T'$, but the latter two are related to the time required to
confirm failures and disseminate failure notifications throughout the
system. The time taken to confirm a failure (which we refer to as
-$t_{confirm}$) dictates the protocol's sensitivity to transient failures,
+$t_{\mathit{confirm}}$) dictates the protocol's sensitivity to transient failures,
since the failure will propagate the group, ultimately removing the server
-from the system and triggering a storage rebuild. $t_{confirm}$ is defined
-below, where $t_{detect}$ is the time required to first detect an
-unresponsive node\footnote{As defined in \cite{Das:2002}, $t_{detect}$
+from the system and triggering a storage rebuild. $t_{\mathit{confirm}}$ is defined
+below, where $t_{\mathit{detect}}$ is the time required to first detect an
+unresponsive node\footnote{As defined in \cite{Das:2002}, $t_{\mathit{detect}}$
depends solely on the probability of a node failure in the system, as well
-as the selected protocol period length $T'$.} and $t_{suspect}$ is the time
-spent in the suspiciion subprotocol (so, clearly $t_{suspect} = s \times
+as the selected protocol period length $T'$.} and $t_{\mathit{suspect}}$ is the time
+spent in the suspiciion subprotocol (so, clearly $t_{\mathit{suspect}} = s \times
T'$):
\begin{equation*}
-t_{confirm} = t_{detect} + t_{suspect}
+t_{\mathit{confirm}} = t_{\mathit{detect}} + t_{\mathit{suspect}}
\end{equation*}
The time required to disseminate membership updates throughout the system
-(which we refer to as $t_{dissem}$)\footnote{Defined in \cite{Das:2002}
+(which we refer to as $t_{\mathit{dissem}}$)\footnote{Defined in \cite{Das:2002}
using epidemic (gossip) theory, depending only on the size of the process
group and the selected protocol period length $T'$.} is another crucial
factor in the overall performance of the protocol. A slower rate of
@@ -340,11 +403,11 @@ system potentially unresponsive to clients and increasing the risk of
permanent data loss. We can then define the total fault detection time as
the duration between a node failure and this failure notification being
disseminated to \emph{every} other member in the process group. We denote
-this convergence time of the protocol as $t_{converge}$ and define it
+this convergence time of the protocol as $t_{\mathit{converge}}$ and define it
below:
\begin{equation*}
-t_{converge} = t_{confirm} + t_{dissem}
+t_{\mathit{converge}} = t_{\mathit{confirm}} + t_{\mathit{dissem}}
\end{equation*}
In Table \ref{table:converge}, we analyze the performance of SWIM with
@@ -363,104 +426,110 @@ sensitivity to transient failures. This configuration strategy will allow for
the efficient detection and dissemination of faults with a decreased probability
of false-positive detections.
-In Table \ref{table:swim-params}.
+We choose to configure our SWIM simulation model using the parameters shown in
+Table \ref{table:swim-params}. Setting the protocol period length to 200 ms
+allows us to potentially detect faults and disseminate notifications in 318 ms
+and 1.53 s, respectively. Further, the average network load imposed by this
+configuration (about 2 KiB/s, assuming a constant 200 byte message size for
+pings and acks) is negligible compared to the bandwith of typical network
+interconnects in high-performance data centers. A suspicion timeout of 75
+protocol period lengths means the protocol should remain resilient to transient
+failures of up to 15 seconds, greatly reducing the probability of an unnecessary
+storage system rebuild. Also, the total time taken for the protocol to reach
+global consensus on a failed member theoretically should take around 17 seconds,
+the same order of magnitude as the average time taken to complete the triage
+phase of fault response.
-\begin{table*}[t]
-\caption{Effect of SWIM period length and suspect timeout on protocol performance.}
-\centering
-\begin{tabular}{c c c | c c c c c}
-\hline
-period length ($T'$) & members & suspect timeout ($s$) & average network load per member
-(messages/sec) & t\_detect & t\_dissem & t\_suspect & t\_converge \\
-\hline
-\multirow{6}{*}{.12 ms} & \multirow{2}{*}{1024} & 10 & \multirow{6}{*}{17480.1} &
-\multirow{6}{*}{.19 ms} & \multirow{2}{*}{.83 ms} & 1.2 ms & 2.22 ms \\
-& & 100 & & & & 12 ms & 13.02 ms \\
-& \multirow{2}{*}{2048} & 10 & & & \multirow{2}{*}{.92 ms} & 1.2 ms & 2.31 ms \\
-& & 100 & & & & 12 ms & 13.11 ms \\
-& \multirow{2}{*}{4096} & 10 & & & \multirow{2}{*}{1 ms} & 1.2 ms & 2.39 ms \\
-& & 100 & & & & 12 ms & 13.19 ms \\
-\hline
-\multirow{6}{*}{10 ms} & \multirow{2}{*}{1024} & 10 & \multirow{6}{*}{209.8} &
-\multirow{6}{*}{15.91 ms} & \multirow{2}{*}{69.34 ms} & 100 ms & 185.25 ms \\
-& & 100 & & & & 1 s & 1.09 s \\
-& \multirow{2}{*}{2048} & 10 & & & \multirow{2}{*}{76.26 ms} & 100 ms & 192.17 ms \\
-& & 100 & & & & 1 s & 1.09 s \\
-& \multirow{2}{*}{4096} & 10 & & & \multirow{2}{*}{83.19 ms} & 100 ms & 199.1 ms \\
-& & 100 & & & & 1 s & 1.1 s \\
-\hline
-\multirow{6}{*}{1 s} & \multirow{2}{*}{1024} & 10 & \multirow{6}{*}{2.1} &
-\multirow{6}{*}{1.59 s} & \multirow{2}{*}{6.93 s} & 10 s & 18.52 s \\
-& & 100 & & & & 100 s & 108.52 s \\
-& \multirow{2}{*}{2048} & 10 & & & \multirow{2}{*}{7.63 s} & 10 s & 19.22 s \\
-& & 100 & & & & 100 s & 109.22 s \\
-& \multirow{2}{*}{4096} & 10 & & & \multirow{2}{*}{8.32 s} & 10 s & 19.91 s \\
-& & 100 & & & & 100 s & 109.91 s \\
-\hline
-\end{tabular}
-\label{table:converge}
-\end{table*}
-
-\begin{table*}[t]
-\caption{Default SWIM parameters chosen for Triton.}
-\centering
-\begin{tabular}{c c c | c c c c c}
-\hline
-period length ($T'$) & members & suspect timeout ($s$) & average network load per member
-(messages/sec per member) & t\_detect & t\_dissem & t\_suspect & t\_converge \\
-\hline
-\hline
-\end{tabular}
-\label{table:swim-params}
-\end{table*}
-
-\section{Methodology}
-\label{sec:method}
-
-To determine the feasability of SWIM as the failure detection component
-of Triton, we have developed a complete model of the protocol using the
-CODES~\cite{TODO} storage simulation framework. CODES is built on top of
-ROSS~\cite{TODO}, a high-performance parallel discrete event simulator.
-We analyze the performance of the protocol when subjected to a number of
-different failure scenarios, including: network packet loss, hard node
-failures, transient node failures, and mixed failures.
-
-%% Simulation Results %%
\section{Simulation Results}
-\label{sec:results}
-
-\subsection{Network Packet Loss}
-\label{sec:results-packetloss}
-
-How does the protocol respond to varying levels of packet loss? The SWIM
-papers seem to indicate the protocol is resilient to packet loss rates of
-up to 15\%, although this is probably dependent on other protocol
-parameters.
-
-\subsection{Hard Node Failures}
-\label{sec:results-hardfailure}
-
-How quickly can the protocol detect hard node failures under different
-protocol parameters?
-
-\subsection{Transient Node Failures}
-\label{sec:results-transfailure}
-
-How does the protocol respond when subjected to transient failures of
-varying durations? Obviously, there is a necessary tradeoff here between
-false-positive rate and efficiency of detecting actually failed members.
-
-\subsection{Mixed Failures}
-\label{sec:results-mixedfailure}
-
-Here we would like to subject the protocol to a mixture of different
-failure scenarios (which may or may not overlap chronologically). The
-idea is to run a very long simulation (on the order of years) to analyze
-how the protocol responds.
+\label{sec:sim-results}
+
+To analyze the feasability of SWIM as the fault detection component of
+Triton, we have developed a complete model of the protocol using the
+CODES~\cite{cope2011codes,liu2012role} storage simulation framework. CODES
+is built on top of ROSS~\cite{ross:2013}, a high-performance parallel
+discrete event simulator capable of processing billions of events per
+second. Using this model of the SWIM group membership protocol, we hope to
+accomplish two things: validation of the model using the expected results
+discussed in Section \ref{sec:swim-params} and analysis of the protocol's
+performance when exposed to different failure scenarios. These failure
+scenarios include varying levels of network packet loss, varying durations
+of intermittent transient failures, and more complex scenarios which
+combine network packet loss, hard node failures, and transient failures.
+This analysis should provide sufficient data to determine the efficiency
+and accuracy of fault detection at the scale typical of large data centers.
+
+A critical component of the SWIM simulation model is the network
+interconnect abstraction used for delivering ping and ack messages to
+remote storage servers. We decide to use a LogGP network model~\cite{loggp}
+based on the Tukey analysis cluster at the Argonne Leadership Computing
+Facility. Tukey is composed of 96 compute nodes, each containing two 2 GHz
+AMD Opteron 6128 processors, 64 GiB of main memory, and a single-port
+Mellanox ConnectX 2 QDR Infiniband NIC. These nodes are interconnected by
+four Mellanox IS-5600 switches. Although this system is smaller scale than
+the systems we intend to simulate in this work, we believe its characteristics
+to be representative of a typical high-performance interconnect used in
+large-scale data centers. This network model is described in further detail
+and validated in \cite{triton-rebuild-deliverable}.
+
+\subsection{Validation}
+\label{sec:results-validate}
+
+Figure \ref{fig:sim-results} shows how the simulated performance of the
+SWIM protocol compares to the expected values given for our selected
+configuration. To be concise, we only consider the time taken to detect
+a fault and the time to disseminate this fault notification to all other
+non-faulty members -- the expected and measured time spent suspecting a
+faulty member is identical in all cases and dominates the total convergence
+time of the protocol. The measured detection time remains nearly constant
+at all scales, but does deviate from the expected results by nearly two
+protocol period lengths. However, this deviation is likely due to
+simplified assumptions made in calculating the expected
+value\footnote{For instance, the analytical SWIM model assumes immediate
+delivery of all pings and acks.}. Also, the time taken to first detect a
+failure is partially dependent on when the failure occurs relative to the
+start of the next protocol period. Similarly, the expected and measured
+dissemination times steadily increase with the size of the process group,
+but are offset by almost two protocol period lengths. Regardless, the
+time spent detecting and disseminating failures throughout the process
+group is minor compared to the time spent suspecting a member as failed.
+These results indicate that our SWIM configuration should reliably
+propagate fault notifications throughout the system in about 17 seconds
+(even at larger scales), while still remaining resilient to transient
+faults of up to 15 second durations, which still satisfies our design
+requirements given in Section \ref{sec:swim-params}.
+
+\begin{figure}[b]
+\centering
+\includegraphics[width=0.48\textwidth]{figs/swim-results.pdf}
+\caption{Expected and measured results for SWIM configured using parameters
+in Table \ref{table:swim-params}.}
+\label{fig:sim-results}
+\end{figure}
-%% Conclusion %%
\section{Conclusion}
-\label{sec:conclusions}
+\label{sec:conclusion}
+
+In this paper we presented findings on the performance of fault detection
+in the context of a large-scale high-performance storage system using the
+SWIM group membership protocol. This performance analysis was carried out
+using a simulated model of the SWIM protocol, where a single group member
+was subjected to a hard failure and the resulting fault detection time
+measured. Our results show that our model provides stable results that
+track well with the hypothetical results across a range of large-scale
+process groups. The results also suggest that the SWIM protocol can be
+configured to completely disseminate failure notification in around 17
+seconds at all scales, while remaining resilient to transient failures of
+up to 15 seconds and imposing a negligible network load.
+
+As part of continued research in this space, we hope to further analyze
+the use of SWIM as Triton's fault detection mechanism under more complex
+failure scenarios and across time spans of months or years. These failure
+scenarios could include, but are not limited to, intermittent transient
+failures of varying durations, multiple hard node failures, and varying
+levels of network packet loss. This analysis would provide more insight
+into the performance of Triton fault detection in the presence of a range
+of failure scenarios typical in large-scale data centers, and would assist
+in further tuning of fault detection parameters for this environment.
%%%%%%%%%%%%%%%%%%%%%
%% Acknowledgements %%
hooks/post-receive
--
