paper.tex

%% For double-blind review submission, w/o CCS and ACM Reference (max submission space)
\documentclass[acmsmall,10pt,review,screen,anonymous]{acmart}
\settopmatter{printfolios=true,printccs=false,printacmref=false}


%\setcitestyle{super,sort&compress}
\bibliographystyle{ACM-Reference-Format}
\citestyle{acmauthoryear}

\usepackage{booktabs} % For formal tables
\usepackage{pgf-umlsd}

\usepackage[ruled]{algorithm2e} % For algorithms

\usepackage{listings}

\lstset{
  language=Scala,
  basicstyle=\ttfamily,
  breaklines=true,
  columns=fixed,
  basewidth=0.5em,
  captionpos=b,
  tabsize=2,
  frame=top,frame=bottom,
}

% \usepackage{markdown}

\usepackage{color}
\usepackage{dot2texi}
\usepackage{tikz}
\usetikzlibrary{shapes,arrows}

\usepackage[inline]{enumitem}

\usepackage{glossaries}

\makeglossaries

\newglossaryentry{plug-in}{
  name={plug-in},
  description={
    is a Scala \lstinline{trait} designed to be instantiated by \href{https://javadoc.io/page/com.thoughtworks.feature/factory_2.11/latest/com/thoughtworks/feature/Factory.html}{\lstinline{Factory}}, along with other plug-ins
  }
}

\newglossaryentry{differentiable function}{
  name={differentiable function},
  description={
    is a Scala function that returns a \gls{differentiable expression}. It may represent a loss functions, a neural network or a subset of a neural network (e.g. a dense block in DenseNet\cite{iandola2014densenet})
  }
}

\newglossaryentry{differentiable expression}{
  name={differentiable expression},
  description={
    is a composable expression that supports operator overloading, whose type is \lstinline{DoubleLayer}, \lstinline{FloatLayer}, \lstinline{INDArrayLayer}, or other subtypes of \href{https://javadoc.io/page/com.thoughtworks.deeplearning/plugins-layers_2.11/latest/com/thoughtworks/deeplearning/plugins/Layers$Layer.html}{\lstinline{Layer}}. After a differentiable expression is built, it can perform \lstinline{forward} pass to create a differentiable \glspl{computational graph}.
  }
}

\newglossaryentry{trainable variable}{
  name={trainable variable},
  description={
    is a scalar or vector weight in a model, whose type is \lstinline{DoubleWeight}, \lstinline{FloatWeight}, \lstinline{INDArrayWeight}, or other subtypes of \href{https://javadoc.io/page/com.thoughtworks.deeplearning/plugins-weights_2.11/latest/com/thoughtworks/deeplearning/plugins/Weights$Weight.html}{\lstinline{Weight}}
  }  
}

\newglossaryentry{computational graph}{
  name={computational graph},
  description={
    is an asynchronous monadic data type that manages the life cycle of tapes, whose type is \href{https://javadoc.io/page/com.thoughtworks.raii/asynchronous_2.11/latest/com/thoughtworks/raii/asynchronous$$Do.html}{\lstinline{Do}}\lstinline{[}\href{https://javadoc.io/page/com.thoughtworks.deeplearning/deeplearning_2.11/latest/com/thoughtworks/deeplearning/DeepLearning$$Tape.html}{\lstinline{Tape[Data, Delta]}}\lstinline{]}
  }
}

% Metadata Information
% \acmJournal{PACMPL}
% \acmVolume{9}
% \acmNumber{4}
% \acmArticle{39}
% \acmYear{2010}
% \acmMonth{3}

% Copyright
% \setcopyright{acmcopyright}
% \setcopyright{acmlicensed}
% \setcopyright{rightsretained}
\setcopyright{none}

% DOI
% \acmDOI{0000001.0000001}

\newsavebox{\IdrisDeepLearning}

% Document starts
\begin{document}


% Title portion
\title{Monadic Deep Learning}
%  \titlenote{This is a titlenote}
\subtitle{Performing monadic automatic differentiation in parallel}
%  \subtitlenote{Subtitle note} 

\author{Yang, Bo}
\orcid{0000-0003-2757-9115}
\affiliation{
  \institution{ThoughtWorks, Inc}
%   \streetaddress{104 Jamestown Rd}
%   \city{Williamsburg}
%   \state{VA}
%   \postcode{23185}
%   \country{USA}
}
\email{atryyang@thoughtworks.com}
\author{Zhang, Zhihao}
\affiliation{%
  \institution{ThoughtWorks, Inc}
%   \streetaddress{104 Jamestown Rd}
%   \city{Williamsburg}
%   \state{VA}
%   \postcode{23185}
%   \country{USA}
}
\email{zhazhang@thoughtworks.com}
\author{Marisa Kirisame} 
\affiliation{
 \institution{University of Washington}
%  \streetaddress{Rono-Hills}
 \city{Seattle} 
 \state{Washington}
 \country{USA}}
\email{lolisa@marisa.moe}
\author{Shi, Kai} 
\authornote{The corresponding author}
\affiliation{
  \institution{ThoughtWorks, Inc}
%  \streetaddress{Rono-Hills}
%  \city{Seattle} 
%  \state{Washington}
%  \country{USA}
}
\email{kshi@thoughtworks.com}

\begin{abstract}
The Java and Scala community has built a very successful big data ecosystem.
However, most of neural networks running on it are modeled in dynamically typed programming languages.
These dynamically typed deep learning frameworks treat neural networks as differentiable expressions that contain many \glspl{trainable variable},
and perform automatic differentiation on those expressions when training them.

Until 2019, none of the learning frameworks in statically typed languages provided the expressive power of traditional frameworks.
Their users are not able to use custom algorithms unless creating plenty of boilerplate code for hard-coded back-propagation.

We solved this problem in DeepLearning.scala 2. Our contributions are:

\begin{itemize}
  \item We discovered a novel approach to perform automatic differentiation in reverse mode for statically typed functions that contain multiple \glspl{trainable variable}, and can interoperate freely with the metalanguage.
  \item We designed a set of monads and monad transformers, which allow users to create monadic expressions that represent dynamic neural networks.
  \item Along with these monads, we provide some applicative functors, to perform multiple calculations in parallel.
\end{itemize}

With these features, users of DeepLearning.scala were able to create complex neural networks in an intuitive and concise way, and still maintain type safety.
\end{abstract}


%
% The code below should be generated by the tool at
% http://dl.acm.org/ccs.cfm
% Please copy and paste the code instead of the example below. 
%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10002950.10003714.10003715.10003748</concept_id>
<concept_desc>Mathematics of computing~Automatic differentiation</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10010520.10010521.10010542.10010294</concept_id>
<concept_desc>Computer systems organization~Neural networks</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011006.10011008.10011009.10011021</concept_id>
<concept_desc>Software and its engineering~Multiparadigm languages</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011006.10011008.10011009.10011012</concept_id>
<concept_desc>Software and its engineering~Functional languages</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011006.10011008.10011009.10011011</concept_id>
<concept_desc>Software and its engineering~Object oriented languages</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011006.10011008.10011009.10011019</concept_id>
<concept_desc>Software and its engineering~Extensible languages</concept_desc>
<concept_significance>100</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011006.10011050.10011017</concept_id>
<concept_desc>Software and its engineering~Domain specific languages</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011006.10011066.10011067</concept_id>
<concept_desc>Software and its engineering~Object oriented frameworks</concept_desc>
<concept_significance>100</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011074.10011092.10011093</concept_id>
<concept_desc>Software and its engineering~Object oriented development</concept_desc>
<concept_significance>100</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011006.10011008.10011009.10010175</concept_id>
<concept_desc>Software and its engineering~Parallel programming languages</concept_desc>
<concept_significance>100</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011006.10011008.10011009.10011016</concept_id>
<concept_desc>Software and its engineering~Data flow languages</concept_desc>
<concept_significance>100</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}

\ccsdesc[500]{Computer systems organization~Neural networks}
\ccsdesc[500]{Software and its engineering~Multiparadigm languages}
\ccsdesc[300]{Mathematics of computing~Automatic differentiation}
\ccsdesc[300]{Software and its engineering~Functional languages}
\ccsdesc[300]{Software and its engineering~Object oriented languages}
\ccsdesc[300]{Software and its engineering~Domain specific languages}
\ccsdesc[100]{Software and its engineering~Extensible languages}
\ccsdesc[100]{Software and its engineering~Object oriented frameworks}
\ccsdesc[100]{Software and its engineering~Object oriented development}
\ccsdesc[100]{Software and its engineering~Parallel programming languages}
\ccsdesc[100]{Software and its engineering~Data flow languages}

%
% End generated code
%


\keywords{type class, path-dependent type, monad, scala}




\maketitle

% % The default list of authors is too long for headers.
\renewcommand{\shortauthors}{B. Yang et al.}




\begin{lrbox}{\IdrisDeepLearning}
\begin{lstlisting}[basicstyle=\footnotesize,language=Haskell]
interface DeepLearning Differentiable where
  Data : Type
  Delta : Type
  forward : Differentiable -> Do (Tape Data Delta)
\end{lstlisting}
\end{lrbox}

\input{body}


\end{document}