doc/.gitignore

+tech_report.aux
+tech_report.bbl
+tech_report.blg
+tech_report.fdb_latexmk
+tech_report.fls
+tech_report.log
+tech_report.pdf

doc/refs.bib

+@TechReport{SunRPC,
+  author =    {},
+  title =     {RFC 1057: Remote Procdeure Call Protocol Specification, Version 2},
+  institution =  {Sun Microsystems},
+  year =         {1988},
+  OPTkey =       {},
+  OPTtype =      {},
+  OPTnumber =    {},
+  OPTaddress =   {},
+  OPTmonth =     {},
+  OPTnote =      {},
+  OPTannote =    {}
+}
+@Misc{ASN1,
+  author =       {ITU-T},
+  title =        {ASN.1 project web page},
+  howpublished  ={\verb+http://www.itu.int/en/ITU-T/asn1/+},
+  OPTnote =    {}
+}
+ 
+@article{jastadd,
+ author = {Ekman, Torbj\"{o}rn and Hedin, G\"{o}rel},
+ title = {The Jastadd Extensible Java Compiler},
+ journal = {SIGPLAN Not.},
+ issue_date = {October 2007},
+ volume = {42},
+ number = {10},
+ month = oct,
+ year = {2007},
+ issn = {0362-1340},
+ pages = {1--18},
+ numpages = {18},
+ url = {http://doi.acm.org/10.1145/1297105.1297029},
+ doi = {10.1145/1297105.1297029},
+ acmid = {1297029},
+ publisher = {ACM},
+ address = {New York, NY, USA},
+ keywords = {OOP, compilers, declarative frameworks, extensibility, java, modularity},
+}

doc/tech_report.tex

 % *** en embryo of a technical report describing the labcomm design rationale and implementation ***
 
 \documentclass[a4paper]{article}
+\usepackage{listings}
 %\usepackage{verbatims}
 %\usepackage{todo}
 
 \begin{document}
 \title{Labcomm tech report}
 \author{Anders Blomdell and Sven Gesteg\aa{}rd Robertz }
-\date{embryo of draft, \today}
+\date{draft, \today}
 
-\maketitle 
+\maketitle
 
 \begin{abstract}
 
@@ -19,7 +20,7 @@ processor, and network used (e.g., byte order, etc).  It is primarily intended
 for situations where the overhead of communication has to be kept at a minimum,
 hence LabComm only requires one-way communication to operate. The one-way
 operation also has the added benefit of making LabComm suitable as a storage
-format. 
+format.
 
 LabComm provides self-describing channels, as communication starts with the
 transmission of an encoded description of all possible sample types that can
@@ -28,10 +29,13 @@ application sees fit.
 
 The LabComm system is based on a binary protocol and
 and a compiler that generates encoder/decoder routines for popular languages
-including C, Java, and Python.
+including C, Java, and Python. There is also a standard library for the
+languages supported by the compiler, containing generic routines for
+encoding and decoding types and samples, and interaction with
+application code.
 
 The LabComm compiler accepts type and sample declarations in a small language
-that is similar to C or Java type-declarations. 
+that is similar to C or Java type-declarations.
 \end{abstract}
 \section{Introduction}
 
@@ -39,15 +43,21 @@ that is similar to C or Java type-declarations.
 %[[http://asn1.org|ASN1]].
 
 LabComm has got it's inspiration from Sun RPC~\cite{SunRPC}
-and ASN1~\cite{ANS1}. LabComm is primarily intended for situations
+and ASN1~\cite{ASN1}. LabComm is primarily intended for situations
 where the overhead of communication has to be kept at a minimum, hence LabComm
 only requires one-way communication to operate. The one-way operation also has
-the added benefit of making LabComm suitable as a storage format. 
+the added benefit of making LabComm suitable as a storage format.
 
+Two-way comminication adds complexity, in particular for hand-shaking
+during establishment of connections, and the LabComm library provides
+support for (for instance) avoiding deadlocks during such hand-shaking.
+
+\pagebreak
 \section{Communication model}
 
 LabComm provides self-describing communication channels, by always transmitting
-a machine readable description of the data before actual data is sent.
+a machine readable description of the data before actual data is
+sent\footnote{Sometimes referred to as \emph{in-band} self-describing}.
 Therefore, communication on a LabComm channel has two phases
 
 \begin{enumerate}
@@ -61,9 +71,9 @@ During operation, LabComm will ensure (i.e., monitor) that a communication
 channel is fully configured, meaning that both ends agree on what messages that
 may be passed over that channel.  If an unregistered sample type is sent or
 received, the LabComm encoder or decoder will detect it and take action.
-
 In more dynamic applications, it is possible to reconfigure a channel in order to add,
-remove, or change the set of registerd sample types.
+remove, or change the set of registered sample types. This is discussed
+in Section~\ref{sec:reconfig}.
 
 The roles in setting up, and maintaining, the configuration of a channel are as follows:
 
@@ -77,7 +87,7 @@ The roles in setting up, and maintaining, the configuration of a channel are as
 \paragraph{The transmitter (encoder)}
 
 \begin{itemize}
- \item ensures that the signature of a sample is transmitted on the channel before samples are 
+ \item ensures that the signature of a sample is transmitted on the channel before samples are
        written to that channel
 \end{itemize}
 
@@ -88,11 +98,63 @@ The roles in setting up, and maintaining, the configuration of a channel are as
  \item if an unhandled signature is received, pauses the channel and informs the application
 \end{itemize}
 
+The fundamental communication model applies to all LabComm channels and
+deals with the individual unidirectional channels. In addition to that,
+the labcomm libraries support the implementation of higher-level
+handshaking and establishment of bidirectional channels both through
+means of interacting with the underlying transport layer (e.g., for
+marking packets containing signatures as \emph{important}, for
+transports that handle resending of dropped packets selectively), or
+requesting retransmission of signatures.
+
+In order to be both lean and generic, LabComm does not provide a
+complete protocol for establishing and negotiating bidirectional
+channels, but does provide support for building such protocols on top
+of LabComm.
+\subsection{Reconfiguration}
+\label{sec:reconfig}
+
+The fundamental communication model can be generalized to the life-cycle
+of a concrete communication channel, including the transport layer,
+between two end-points. Then, the communication phases are
+\begin{enumerate}
+  \item \emph{Establishment} of communication channel at the transport layer
+  \item \emph{Configuration} of the LabComm channel (registration of sample
+    types)
+  \item \emph{Operation} (transmission of samples)
+\end{enumerate}
+where it is possible to \emph{reconfigure} a channel by transitioning
+back from phase 3 to phase 2. That allows dynamic behaviour, as a sample
+type can be added or redefined at run-time. It also facilitates error
+handling in two-way channels.
+
+One example of this, more dynamic, view of a labcomm channel is that the
+action taken when an unregistered sample is sent or received is to
+revert back to the configuration phase and redo the handshaking to
+ensure that both sides agree on the set of sample types (i.e.,
+signatures) that are currently configured for the channel.
+
+From the system perspective, the LabComm protocol is involved in
+phases 2 and 3. The establishement of the \emph{transport-layer}
+channels is left to external application code. In the Labcomm library,
+that application code is connected to the LabComm routines through
+the \emph{reader} and \emph{writer} interfaces,
+with default implementations for sockets or file descriptors (i.e.,
+files and streams).
+
+
 \section{The Labcomm language}
 
-The following examples do not cover the entire language
-specification (see appendix~\ref{language_grammar}), but might serve as a
-gentle introduction to the LabComm language.
+The LabComm language is used to describe data types, and from such data
+descriptions, the compiler generates code for encoding and decoding
+samples. The language is quite similar to C struct declarations, with
+some exceptions. We will now introduce the language through a set of
+examples.
+
+These examples do not cover the entire language
+specification (see appendix~\ref{sec:LanguageGrammar} for the complete
+grammar), but serve as a gentle introduction to the LabComm
+language covering most common use-cases.
 
 \subsection{Primitive types}
 
@@ -107,6 +169,22 @@ gentle introduction to the LabComm language.
   sample string a_string;
 \end{verbatim}
 
+\subsection{The void type}
+
+There is a type, \verb+void+, which can be used to send
+a sample that contains no data. 
+
+\begin{verbatim}
+typedef void an_empty_type;
+
+sample an_empty_type no_data1;
+sample void no_data2;
+\end{verbatim}
+
+\verb+void+ type can may not be used as a field in a struct or
+the element type of an array.
+
+
 \subsection{Arrays}
 
 \begin{verbatim}
@@ -120,7 +198,7 @@ gentle introduction to the LabComm language.
 
 \begin{enumerate}
 \item In contrast to C, LabComm supports both fixed and variable (denoted
-by \verb+_+) sized arrays.
+by~\verb+_+) sized arrays.
 
 \item In contrast to Java, LabComm supports multidimensional arrays and not
 only arrays of arrays.
@@ -136,7 +214,46 @@ only arrays of arrays.
   } a_struct;
 \end{verbatim}
 
-\section{User defined types}
+\subsection{Sample type refereces}
+
+In addition to the primitive types, a sample may contain
+a reference to a sample type. References are declared using
+the \verb+sample+ keyword.
+
+Examples:
+
+\begin{verbatim}
+sample sample a_ref;
+
+sample sample ref_list[4];
+
+sample struct { 
+    sample ref1;
+    sample ref2;
+    int x;
+    int y;
+} refs_and_ints;
+\end{verbatim}
+
+Sample references are need to be registered on both encoder and decoder
+side, using the functions
+
+\begin{verbatim}
+int labcomm_decoder_sample_ref_register(
+    struct labcomm_decoder *decoder\nonumber
+    const struct labcomm_signature *signature);
+
+int labcomm_encoder_sample_ref_register(
+    struct labcomm_encoder *encoder\nonumber
+    const struct labcomm_signature *signature);
+\end{verbatim}
+
+The value of an unregistered sample reference will be decoded as \verb+null+.
+
+\subsection{User defined types}
+
+User defined types are declared with the \verb+typedef+ reserved word,
+and can then be used in type and sample declarations.
 
 \begin{verbatim}
   typedef struct {
@@ -151,7 +268,8 @@ only arrays of arrays.
 
 The LabComm system consists of a compiler for generating code from the data
 descriptions, and libraries providing LabComm communication facilities in,
-currently, C, Java, Python, and C\#.
+currently, C, Java, Python, C\#, and RAPID\footnote{excluding variable
+size samples, as RAPID has limited support for dynamic memory allocation}.
 
 
 \subsection{The LabComm compiler}
@@ -160,12 +278,12 @@ The LabComm compiler generates code for the declared samples, including marshall
 demarshalling code, in the supported target languages.
 
 The compiler itself is implemented in Java using the JastAdd~\cite{jastadd} compiler compiler.
-\pagebreak
+
 \subsection{The LabComm library}
 
 The LabComm libraries contain functionality for the end-to-end transmission
 of samples. They are divided into two layers, where the upper layer implements
-the general encoding and decoding of samples, and the lower one deals with 
+the general encoding and decoding of samples, and the lower one deals with
 the transmission of the encoded byte stream on a particular transport layer.
 
 Thus, the LabComm communication stack looks like this:
@@ -184,7 +302,7 @@ Thus, the LabComm communication stack looks like this:
 \end{figure}
 \subsubsection{LabComm actions}
 
-(similar to ioctl()) 
+(similar to ioctl())
 The encoder/writer and decoder/reader interfaces consist of a set of actions
 
 One example of this is that there is a a separate writer action for
@@ -193,7 +311,7 @@ from encoded samples, e.g., to allow handshaking during channel setup.
 
 User actions allow the application or a higher level
 protocol to communicate with the underlying transport layer through the LabComm
-encoder. 
+encoder.
 
 One example is reliable communication, which is controlled from the application
 but needs to be implemented for each transport at at the reader/writer level.
@@ -203,7 +321,7 @@ but needs to be implemented for each transport at at the reader/writer level.
 
 \begin{itemize}
 \item a protocol for two-way connections
-\item intrinsically supporting reliable communication 
+\item intrinsically supporting reliable communication
 \item providing semantic service-descriptions
 \end{itemize}
 
@@ -222,89 +340,193 @@ But
 
 With the following `example.lc` file:
 
-\begin{verbatim}
-sample struct {
-  int sequence;
-  struct {
-    boolean last;
-    string data;
-  } line[_];
-} log_message;
-sample float data;
-\end{verbatim}
-
-and this \verb+example_encoder.c+ file 
+\lstinputlisting[basicstyle=\footnotesize\ttfamily]{../examples/wiki_example/example.lc}
+and this \verb+example_encoder.c+ file
+\lstinputlisting[basicstyle=\footnotesize\ttfamily,language=C]{../examples/wiki_example/example_encoder.c}
 
-\begin{verbatim}
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <labcomm_fd_reader.h>
-#include <labcomm_fd_writer.h>
-#include "example.h"
-
-int main(int argc, char *argv[]) {
-  int fd;
-  struct labcomm_encoder *encoder;
-  int i, j;
-
-  fd = open("example.encoded", O_WRONLY|O_CREAT|O_TRUNC, 0644);
-  encoder = labcomm_encoder_new(labcomm_fd_writer, &fd);
-  labcomm_encoder_register_example_log_message(encoder);
-  labcomm_encoder_register_example_data(encoder);
-  for (i = 0 ; i < argc ; i++) {
-    example_log_message message;
-
-    message.sequence = i + 1;
-    message.line.n_0 = i;
-    message.line.a = malloc(message.line.n_0*sizeof(message.line));
-    for (j = 0 ; j < i ; j++) {
-      message.line.a[j].last = (j == message.line.n_0 - 1);
-      message.line.a[j].data = argv[j + 1];
-    }
-    labcomm_encode_example_log_message(encoder, &message);
-    free(message.line.a);
-  }
-  for (i = 0 ; i < argc ; i++) {
-    float f = i;
-    labcomm_encode_example_data(encoder, &f);
-  }
-}
-\end{verbatim}
+\newpage
 
 Running \verb+./example_encoder one two+, will yield the following result in example.encoded:
-
-
 \begin{verbatim}
-00000000  02 40 0b 6c 6f 67 5f 6d  65 73 73 61 67 65 11 02  |.@.log_message..|
-00000010  08 73 65 71 75 65 6e 63  65 23 04 6c 69 6e 65 10  |.sequence#.line.|
-00000020  01 00 11 02 04 6c 61 73  74 20 04 64 61 74 61 27  |.....last .data'|
-00000030  02 41 04 64 61 74 61 25  40 00 00 00 01 00 40 00  |.A.data%@.....@.|
-00000040  00 00 02 01 01 03 6f 6e  65 40 00 00 00 03 02 00  |......one@......|
-00000050  03 6f 6e 65 01 03 74 77  6f 41 00 00 00 00 41 3f  |.one..twoA....A?|
-00000060  80 00 00 41 40 00 00 00                           |...A@...|
-00000068
+00000000  01 0c 0b 4c 61 62 43 6f  6d 6d 32 30 31 34 02 30 |...LabComm2014.0|
+00000010  40 0b 6c 6f 67 5f 6d 65  73 73 61 67 65 22 11 02 |@.log_message"..|
+00000020  08 73 65 71 75 65 6e 63  65 23 04 6c 69 6e 65 10 |.sequence#.line.|
+00000030  01 00 11 02 04 6c 61 73  74 20 04 64 61 74 61 27 |.....last .data'|
+00000040  02 08 41 04 64 61 74 61  01 25 40 04 00 00 00 01 |..A.data.%@.....|
+00000050  00 40 09 00 00 00 02 01  01 03 6f 6e 65 40 0e 00 |.@........one@..|
+00000060  00 00 03 02 00 03 6f 6e  65 01 03 74 77 6f 41 04 |......one..twoA.|
+00000070  00 00 00 00 41 04 3f 80  00 00 41 04 40 00 00 00 |....A.?...A.@...|
+00000080
 \end{verbatim}
 
 i.e.,
 \begin{verbatim}
-<sample_decl> <user_id: 0x40> <string: <len: 11> <"log_message"> 
-  <struct_decl: 
+<version> <length: 12> <string: <len: 11> <"LabComm2014">>
+<sample_decl> <length: 48 
+              <user_id: 0x40> 
+              <string: <len: 11> <"log_message">
+  <signature_length: 34>
+  <struct_decl:
     <number_of_fields: 2>
     <string: <len: 8> <"sequence"> <type: <integer_type>>
-    <string: <len: 4> <"line">> <type: <array_decl 
-      <number_indices: 1> <variable_index> 
-      <type: <struct_decl: 
+    <string: <len: 4> <"line">> <type: <array_decl
+      <number_indices: 1> <variable_index>
+      <type: <struct_decl:
         <number_of_fields:2>
-        <string: <len: 4> <"last">> <type: <boolean_type>> 
+        <string: <len: 4> <"last">> <type: <boolean_type>>
         <string: <len: 4> <"data">> <type: <string_type>>
       >>
    >
 >
-<sample_decl> <user_id: 0x41>
-   ...
+<sample_decl> <length: 8> <user_id: 0x41> <string: <len: 4> <"data">>
+  <signature_length: 1> <float_type>
+<sample_data> <user_id: 40> <length: 4>  <packed_sample_data>
+<sample_data> <user_id: 40> <length: 9>  <packed_sample_data>
+<sample_data> <user_id: 40> <length: 14>  <packed_sample_data>
+\end{verbatim}
+
+\section{Type and sample declarations}
+
+LabComm has two constructs for declaring sample types, \emph{sample
+declarations} and \emph{type declarations}. A sample declaration is used
+for the concrete sample types that may be transmitted, and is always
+encoded as a \emph{flattened} signature. That means that a sample
+containing user types, like
+
+\begin{verbatim}
+typedef struct {
+  int x;
+  int y;
+} point;
+
+sample struct {
+  point start;
+  point end;
+} line;
+\end{verbatim}
+
+is flattened to 
+
+\begin{verbatim}
+sample struct {
+  struct {
+    int x;
+    int y;
+  } start;
+  struct {
+    int x;
+    int y;
+  } end;
+} line;
 \end{verbatim}
 
+Sample declarations are always sent, and is the fundamental identity of
+a type in LabComm. 
+
+Type declarations is the hierarchical counterpart to sample
+declarations: here, fields of user types are encoded as a reference to
+the type instead of being flattened. As the flattened sample decl is the
+fundamental identity of a type, type declarations can be regarded as
+meta-data, describing the internal structure of a sample. They are
+intended to be read by higher-level software and human system developers
+and integrators.
+
+Sample declarations and type declarations have separate name-spaces in
+the sense that the numbers assigned to them by a labcomm encoder 
+come from two independent number series. To identify which
+\verb+TYPE_DECL+ a particular \verb+SAMPLE_DECL+ corresponds to, the
+\verb+TYPE_BINDING+ packet is used.
+
+For sample types that do not depend on any typedefs, no \verb+TYPE_DECL+
+is sent, and the \verb+TYPE_BINDING+ binds the sample id to the special
+value zero to indicate that the type definition is identical to the
+flattened signature.
+
+\subsection{Example}
+
+The labcomm declaration
+\lstinputlisting[basicstyle=\footnotesize\ttfamily]{../examples/user_types/test.lc}
+can be is encoded as
+\begin{lstlisting}[basicstyle=\footnotesize\ttfamily]
+TYPE_DECL 0x40 "coord" <int> val
+TYPE_DECL 0x41 "point" <struct> <2 fields> 
+                                "x" <type: 0x40> 
+                                "y" <type: 0x40>
+TYPE_DECL 0x42 "line" <struct> <2 fields> 
+                                "start" <type: 0x41> 
+                                "end" <type: 0x41>
+TYPE_DECL 0x43 "foo" <struct> <3 fields> 
+                                "a" <int> 
+                                "b" <int> 
+                                "c" <boolean>
+TYPE_DECL 0x44 "twolines" <struct> <3 fields> 
+                                "l1" <type:0x42> 
+                                "l2" <type:0x42> 
+                                "f" <type:0x43>
+
+SAMPLE_DECL 0x40 "twolines" <flat signature>
+
+TYPE_BINDING 0x40 0x44
+\end{lstlisting}
+
+Note that the id 0x40 is used both for the \verb+TYPE_DECL+ of
+\verb+coord+ and the \verb+SAMPLE_DECL+ of \verb+twoline+, and that the
+\verb+TYPE_BINDING+ binds the sample id \verb+0x40+ to the type id
+\verb+0x44+.
+
+\subsection{Run-time behaviour}
+
+When a sample type is registered on an encoder, a \verb+SAMPLE_DECL+
+(i.e., the flat signature) is always generated on that encoder channel.
+
+If the sample depends on user types (i.e., typedefs), \verb+TYPE_DECL+
+packets are encoded, recursively, for the dependent types and a 
+corresponding \verb+TYPE_BINDING+ is encoded.
+
+If a \verb+TYPE_DECL+ is included via multiple sample types, or
+dependency paths, an encoder may choose to only encode it once, but is
+not required to do so. However, if multiple \verb+TYPE_DECL+ packets are
+sent for the same \verb+typedef+, the encoder must use the same
+\verb+type_id+.
+
+\subsection{Decoding in-band type descriptions}
+
+In LabComm, the in-band data descriptions are equivalent to \footnote{in
+the sense that they contain all information needed to recreate} the data
+description source (i.e., the ``.lc-file'').
+%
+As the type declarations (a.k.a. \emph{signatures}) are written before
+sample data on every channel, they can be used to interpret data with
+an unknown (by the receiver) format.
+
+The libraries provide functionality to subscribe to (i.e., register a
+\emph{handler} for) sample and type declarations.
+
+On the low level, the handler receives an instance of the signature
+data structure corresponding to the received declaration.
+
+
+For higher-level processing, the Java library provides the
+\verb+ASTbuilder+ class, which builds an abstract syntax tree in
+the internal representation of the LabComm compiler.
+That enables the user to use the complete functionality of the
+LabComm compiler, e.g. code generation,  on declarations received in a
+LabComm stream. 
+
+
+In combination with on-the-fly compilation and class-loading (or
+linking) that makes it possible to dynamically create handlers for
+previously unknown data types. Thereby, it enables dynamic configuration
+of LabComm endpoints in static languages without the overhead of
+interpreting signatures (at the one-time cost of code generation and
+compilation).
+
+
+
+
+
+
+
 \section{Ideas/Discussion}:
 
 The labcomm language is more expressive than its target languages regarding data types.
@@ -317,28 +539,261 @@ Java primitive types. However, it is unlikely that the entire range is actually
 way of supporting the common cases is to include run-time checks for overflow in the Java encoders
 and decoders.
 
+\section{Related work}
+  
+Two in-band self-descibing communication protocols are Apache
+Avro\cite{avro} and EDN, the extensible data notation developed for
+Clojure and Datomic\cite{EDN}.
+
+EDN encodes \emph{values} as UTF-8 strings. The documentation says
+``edn is a system for the conveyance of values. It is not a type system,
+and has no schemas.'' That said, it is \emph{extensible} in the sense
+that it has a special \emph{dispatch charachter}, \verb+#+, which can  
+be used to add a \emph{tag} to a value. A tag indicates a semantic
+interpretation of a value, and that allows the reader to support
+handlers for specific tags, enabling functionality similar to that of
+labcomm.
+
+\subsection{Apache Avro}
+
+Apache Avro is similar to LabComm in that it has a textual language
+for declaring data, a binary protocol for transmitting data, and code
+generation for several languages.
+
+Avro is a larger system, including RPC \emph{protocols}, support for
+using different \emph{codecs} for data compression, and \emph{schema
+resolution} to support handling schema evolution and transparent 
+interoperability between different versions of a schema.
+
+\subsubsection*{Data types} 
+
+In the table, the Avro type names are listed, and matched to the
+corresponding LabComm type:
+
+\begin{tabular}{|l|c|c|}
+\hline
+  Type &            Labcomm  &               Avro \\
+  \hline Primitive types \\ \hline
+
+int    &         4 bytes     &           varint  \\
+long   &         8 bytes     &           varint  \\
+float  &         4 bytes     &           4 bytes \\
+long   &         8 bytes     &           8 bytes \\
+string &         varint + utf8[]   &     varint + utf8[] \\ 
+bytes  &         varint + byte[]   &     varint + byte[]\\
+
+  \hline Complex types  \\ \hline
+
+struct/record &  concat of fields     &  concat of fields \\ 
+arrays        &  varIdx[] : elements  &  block[]          \\
+map           &    n/a                &  block[]          \\
+union         &   n/a                 & (varint idx) : value \\
+fixed         &   byte[n]             &  the number of bytes declared in
+the schema\\
+\hline
+\end{tabular}
+
+  where 
+
+\begin{verbatim}  
+  block ::= (varint count) : elem[count]      [*1]
+  count == 0 --> no more blocks
+
+
+[*1] for arrays, count == 0 --> end of array
+     if count < 0, there are |count| elements
+     preceded by a varint block_size to allow
+     fast skipping
+\end{verbatim}  
+
+In maps, keys are strings, and values  according to the schema.
+
+In unions, the index indicates the kind of value and the
+value is encoded according to the schema.
+
+Note that the Avro data type \verb+bytes+ corresponds to the
+LabComm declaration \verb+byte[_]+, i.e. a varaible length byte array.
+
+\subsubsection*{the wire protocol}
+
+\begin{tabular}{|l|c|c|}
+  \hline
+  What & LabComm & Avro \\ \hline
+  Data description & Binary signature & JSON schema \\
+  Signature sent only once pre connection& posible & possible \\
+  Signature sent with each sample & possible & possible \\
+  Data encoding & binary & binary \\
+  \hline
+\end{tabular}
+
+
+Both avro and labcomm use varints when encoding data, similar in that
+they both send a sequence of bytes containing 7 bit chunks (with the
+eight bit signalling more chunks to come), but they differ in range,
+endianness and signedness.
+
+\begin{verbatim}
+                LabComm                 Avro
+                unsigned 32 bit         signed zig-zag coding
+                most significant chunk  least significant chunk
+                first                   first
+
+                0   ->  00               0  ->  00
+                1   ->  01              -1  ->  01
+                2   ->  02               1  ->  02
+                    ...                 -2  ->  03
+                                         2  ->  04
+                                            ...
+                127 ->  7f              -64 ->  7f
+                128 ->  81 00            64 ->  80 01
+                129 ->  81 01           -65 ->  81 01
+                130 ->  81 02            65 ->  82 01
+                    ...                     ...   
+\end{verbatim}
+
+\paragraph{Avro Object Container Files} can be seen as a counterpart
+  to a LabComm channel: 
+Avro includes a simple object container file format. A file has a
+schema, and all objects stored in the file must be written according to
+that schema, using binary encoding. Objects are stored in blocks that
+may be compressed. Syncronization markers are used between blocks to
+permit efficient splitting of files, and enable detection of 
+corrupt blocks.
+
+
+The major difference is the sync markers that LabComm does not have, as
+LabComm assumes that, while the transport may drop packets, there will
+be no bit errors in a received packet. If data integrity is required,
+that is delegated to the reader and writer for the particular transport.
+
+\subsubsection{Representation of hierarchical data types}
+
+For a type that contains fields of other user types, like
+\begin{verbatim}
+typedef struct {
+  int x;
+  int y;
+} Point;
+
+sample struct {
+  Point start;
+  Point end;
+} line;
+\end{verbatim}
+
+LabComm encodes both the flattened signature and the 
+typedef which allows the hierarchical type structure to be
+reconstructed.
+%
+The avro encoding is quite similar. 
+The \verb+Line+ example, corresponds to the two schemas
+\begin{verbatim}
+{"namespace": "example.avro",
+ "type": "record",
+ "name": "Point",
+ "fields": [
+     {"name": "x", "type": "int"},
+     {"name": "y", "type": "int"}
+ ]
+}
+\end{verbatim}
+and
+\begin{verbatim}
+{"namespace": "example.avro",
+ "type": "record",
+ "name": "Line",
+ "fields": [
+     {"name": "start", "type": "Point"},
+     {"name": "end",   "type": "Point"}
+ ]
+}
+\end{verbatim}
+which is encoded in an Object Container File as
+\begin{verbatim}
+{"type":"record",
+ "name":"Line",
+ "namespace":"example.avro",
+ "fields":[{"name":"start",
+            "type":{"type":"record",
+                    "name":"Point",
+                    "fields":[{"name":"x","type":"int"},
+                              {"name":"y","type":"int"}]}},
+           {"name":"end",
+            "type":"Point"}
+ ]
+}
+\end{verbatim}
+\subsubsection{Fetures not in LabComm} 
+
+Avro has a set of features with no counterpart in LabComm. They include
+
+\paragraph{Codecs.}
+
+Avro has multiple codecs (for compression of the data):
+
+    \begin{verbatim}
+    Required Codecs:
+    - null : The "null" codec simply passes through data uncompressed.
+
+    - deflate : The "deflate" codec writes the data block using the deflate
+                algorithm as specified in RFC 1951, and typically implemented using the
+                zlib library. Note that this format (unlike the "zlib format" in RFC
+                1950) does not have a checksum.
+
+    Optional Codecs
+
+    - snappy:   The "snappy" codec uses Google's Snappy compression library. Each
+                compressed block is followed by the 4-byte, big-endian CRC32 checksum of
+                the uncompressed data in the block.
+
+    \end{verbatim}
+
+  \paragraph{Schema Resolution.} The main objective of LabComm is to
+    ensure correct operation at run-time. Therefore, a LabComm decoder
+    requires the signatures for each handled sample to match exactly.
+
+    Avro, on the other hand, supports the evolution of schemas and
+    provides support for reading data where the ordering of fields
+    differ (but names and types are the same), numerical types differ
+    but can be
+    \emph{promoted} (E.g., \verb+int+ can be promoted to \verb+long+,
+    \verb+float+, or \verb+double+.), and record fields have been added
+    or removed (but are nullable or have default values).
+
+    \paragraph{Schema fingerprints.} Avro defines a \emph{Parsing
+    Canonical Form} to define when two JSON schemas are ``the same''.
+    To reduce the overhead when, e.g., tagging data with the schema
+    there is support for creating a \emph{fingerprint} using 64/128/256
+    bit hashing, in combination with a centralized repository for
+    fingerprint/schema pairs.
+
+\bibliography{refs}{}
+\bibliographystyle{plain}
+
 \appendix
 \newpage
+
 \section{The LabComm language}
 \label{sec:LanguageGrammar}
 
 \subsection{Abstract syntax}
 \begin{verbatim}
-Program ::= Decl*;
+Specification ::= Decl*;
 
-abstract Decl ::= Type <Name:String>;
+abstract Decl ::= DataType <Name:String>;
 TypeDecl : Decl;
 SampleDecl : Decl;
 
-Field ::= Type <Name:String>;
+Field ::= DataType <Name:String>;
 
-abstract Type;
-VoidType          : Type;
-PrimType          : Type ::= <Name:String> <Token:int>;
-UserType          : Type ::= <Name:String>;
-StructType        : Type ::= Field*;
-ParseArrayType    : Type ::= Type Dim*;
-abstract ArrayType : Type ::= Type Exp*;
+abstract DataType;
+VoidType          : DataType;
+SampleRefType     : DataType;
+PrimType          : DataType ::= <Name:String> <Token:int>;
+UserType          : DataType ::= <Name:String>;
+StructType        : DataType ::= Field*;
+ParseArrayType    : DataType ::= DataType Dim*;
+abstract ArrayType : DataType ::= DataType Exp*;
 VariableArrayType : ArrayType;
 FixedArrayType    : ArrayType;
 
@@ -353,58 +808,104 @@ VariableSize : Exp;
 \section{The LabComm protocol}
 \label{sec:ProtocolGrammar}
 
+Each LabComm2014 packet has the layout
 \begin{verbatim}
-<packet> := ( <type_decl> | <sample_decl> | <sample_data> )*
-<type_decl> := 0x01 ''(packed)'' <user_id> <string> <type>
-<sample_decl> := 0x02 ''(packed)''<user_id> <string> <type>
-<user_id> := 0x40..0xffffffff  ''(packed)''
-<string> := <string_length> <char>*
-<string_length> := 0x00..0xffffffff  ''(packed)''
-<char> := any UTF-8 char
-<type> := ( <basic_type> | <user_id> | <array_decl> | <struct_decl> )
-<basic_type> := ( <boolean_type> | <byte_type> | <short_type> |
-                  <integer_type> | <long_type> | <float_type> | 
-                  <double_type> | <string_type> )
-<boolean_type> := 0x20 ''(packed)''
-<byte_type> := 0x21 ''(packed)''
-<short_type> := 0x22 ''(packed)''
-<integer_type> := 0x23 ''(packed)''
-<long_type> := 0x24 ''(packed)''
-<float_type> := 0x25 ''(packed)''
-<double_type> := 0x26 ''(packed)''
-<string_type> := 0x27 ''(packed)''
-<array_decl> := 0x10 ''(packed)'' <number_of_indices> <indices> <type>
-<number_of_indices> := 0x00..0xffffffff  ''(packed)''
-<indices> := ( <variable_index> | <fixed_index> )*
-<variable_index> := 0x00  ''(packed)''
-<fixed_index> := 0x01..0xffffffff  ''(packed)''
-<struct_decl> := 0x11 ''(packed)'' <number_of_fields> <field>*
-<number_of_fields> := 0x00..0xffffffff  ''(packed)''
-<field> := <string> <type>
-<sample_data> := <user_id> <packed_sample_data>
-<packed_sample_data> := is sent in network order, sizes are as follows:
-
-||Type           ||Encoding/Size                                         ||
-||---------------||------------------------------------------------------||
-||boolean        ||  8 bits                                              ||
-||byte           ||  8 bits                                              ||
-||short          || 16 bits                                              ||
-||integer        || 32 bits                                              ||
-||long           || 64 bits                                              ||
-||float          || 32 bits                                              ||
-||double         || 64 bits                                              ||
-||string         || length ''(packed)'', followed by UTF8 encoded string ||
-||array          || each variable index ''(packed)'',                    ||
-||               || followed by encoded elements                         ||
-||struct         || concatenation of encoding of each element            ||
-||               || in declaration order                                 ||
+<id> <length> <data...>
 \end{verbatim}
-
-Type fields, user IDs, number of indices and lengths are sent in a packed, or
-variable length, form.  An integer is sent as a sequence of bytes where the
-lower seven bits contain a chunk of the actual number and the high bit
-indicates if more chunks follow. The sequence of chunks are sent with the least
-significant chunk first.  (The numbers encoded in this form are indicated above
-with \textit{(packed)}.)
-
+where \verb+length+ is the number of bytes of the \verb+data+ part
+(i.e., excluding the \verb+id+ and \verb+length+ fields), and 
+the \verb+id+ gives the layout of the \verb+data+ part as defined 
+in \ref{sec:ConcreteGrammar}
+\subsection{Data encoding}
+LabComm primitive types are encoded as fixed width values, sent in
+network order.  Type fields, user IDs, number of indices and lengths are
+sent in a variable length (\emph{varint}) form.  A varint integer value
+is sent as a sequence of bytes where the lower seven bits contain a
+chunk of the actual number and the high bit indicates if more chunks
+follow. The sequence of chunks are sent with the least significant chunk
+first.  
+
+The built-in data types are encoded as follows:
+\begin{lstlisting}[basicstyle=\footnotesize\ttfamily]
+||Type       ||Encoding/Size                                      ||
+||-----------||---------------------------------------------------||
+||boolean    ||  8 bits                                           ||
+||byte       ||  8 bits                                           ||
+||short      || 16 bits                                           ||
+||integer    || 32 bits                                           ||
+||long       || 64 bits                                           ||
+||float      || 32 bits                                           ||
+||double     || 64 bits                                           ||
+||sample_ref || 32 bits                                           ||
+||string     || length (varint), followed by UTF8 encoded string  ||
+||array      || each variable index (varint),                     ||
+||           || followed by encoded elements                      ||
+||struct     || concatenation of encoding of each element         ||
+||           || in declaration order                              ||
+\end{lstlisting}
+\pagebreak
+\subsection{Protocol grammar}
+\label{sec:ConcreteGrammar}
+\begin{lstlisting}[basicstyle=\footnotesize\ttfamily]
+<packet>          := <id> <length> ( <version>      | 
+                                     <type_decl>    | 
+                                     <sample_decl>  |
+                                     <sample_ref>   |
+                                     <type_binding> |
+                                     <sample_data> )
+<version>         := <string>
+<sample_decl>     := <sample_id> <string> <signature>
+<sample_ref>      := <sample_id> <string> <signature>
+<type_decl>       := <type_id> <string> <signature>
+<type_binding>    := <sample_id> <type_id>
+<user_id>         := 0x40..0xffffffff  
+<sample_id> : <user_id>
+<type_id>   : <user_id>
+<string>          := <string_length> <char>*
+<string_length>   := 0x00..0xffffffff  
+<char>            := any UTF-8 char
+<signature>       := <length> <type>
+<type>            := <length> ( <basic_type>  | 
+                                <array_decl>  | 
+                                <struct_decl> | 
+                                <type_id> )
+<basic_type>      := ( <void_type> | <boolean_type> | <byte_type> | <short_type> |
+                     <integer_type> | <long_type> | <float_type> |
+                     <double_type> | <string_type> | <sample_ref_type>)
+
+<void_type>       := <struct_decl> 0 //void is encoded as empty struct
+<boolean_type>    := 0x20 
+<byte_type>       := 0x21 
+<short_type>      := 0x22 
+<integer_type>    := 0x23 
+<long_type>       := 0x24 
+<float_type>      := 0x25 
+<double_type>     := 0x26 
+<string_type>     := 0x27 
+<sample_ref_type> := 0x28 
+<array_decl>      := 0x10  <nbr_of_indices> <indices> <type>
+<nbr_of_indices>  := 0x00..0xffffffff  
+<indices>         := ( <variable_index> | <fixed_index> )*
+<variable_index>  := 0x00  
+<fixed_index>     := 0x01..0xffffffff  
+<struct_decl>     := 0x11  <nbr_of_fields> <field>*
+<nbr_of_fields>   := 0x00..0xffffffff  
+<field>           := <string> <type>
+<sample_data>     := packed sample data sent in network order, with
+                  primitive type elements encoded according to
+                  the sizes above
+\end{lstlisting}
+where the \verb+<id>+ in \verb+<packet>+ signals the type of payload,
+and may be either a \verb+<sample_id>+ or a system packet id.
+The labcomm sytem packet ids are:
+\begin{lstlisting}[basicstyle=\footnotesize\ttfamily]
+version:      0x01 
+sample_decl:  0x02 
+sample_ref:   0x03 
+type_decl:    0x04 
+type_binding: 0x05          
+\end{lstlisting}
+Note that since the signature transmitted in a \verb+<sample_def>+ is
+flattened, the \verb+<type>+ transmitted in a \verb+<sample_def>+ may
+not contain any \verb+<type_id>+ fields.
 \end{document}

examples/Makefile

+SUBDIRS=duck_typing dynamic jgrafchart robot simple tcp \
+	twoway user_types wiki_example
+
 .PHONY: all
+
 all:
-	echo To be done...
+	echo More to be done...
 	$(MAKE) -C twoway all
 
+UNAME_S=$(shell uname -s)
+
 .PHONY: test
 test:
 	echo More to be done...
+ifeq ($(UNAME_S),Darwin)
+	$(MAKE) -C user_types all 
+else	
 	cd simple ; sh compile.sh && sh run.sh
-	$(MAKE) -C twoway test 
+	$(MAKE) -C  wiki_example test
+	$(MAKE) -C user_types all 
+endif	
 	$(MAKE) -C duck_typing test
+	$(MAKE) -C twoway test 
+
+.PHONY: clean
+clean: $(SUBDIRS:%=clean-%)
+clean-%:
+	$(MAKE) -C $* clean 
 
-.PHONY: clean distclean
-clean distclean:
-	echo To be done...
-	$(MAKE) -C twoway clean
-	$(MAKE) -C duck_typing clean
+.PHONY: distclean
+distclean: $(SUBDIRS:%=distclean-%)
+distclean-%:
+	$(MAKE) -C $* distclean 

examples/duck_typing/.gitignore

+gen

examples/duck_typing/Makefile

-LABCOMM_JAR=../../compiler/labComm.jar
+LABCOMM_JAR=../../compiler/labcomm2014_compiler.jar
 LABCOMM=java -jar $(LABCOMM_JAR) 
 
 all: gen/animal.py

examples/duck_typing/animal.lc

@@ -6,4 +6,4 @@ sample animal cow;
 sample animal dog;
 sample struct {
   string says;
-} duck;
\ No newline at end of file
+} duck;

examples/duck_typing/duck_typing.py

 #!/usr/bin/python
 
-import labcomm
+import labcomm2014
 import animal
 import StringIO
 
@@ -10,7 +10,7 @@ class Animal:
 
 if __name__ == '__main__':
     buf = StringIO.StringIO()
-    encoder = labcomm.Encoder(labcomm.StreamWriter(buf))
+    encoder = labcomm2014.Encoder(labcomm2014.StreamWriter(buf))
     encoder.add_decl(animal.cow.signature)
     encoder.add_decl(animal.dog.signature)
     encoder.add_decl(animal.duck.signature)
@@ -22,12 +22,12 @@ if __name__ == '__main__':
     theAnimal.says = 'Quack'
     encoder.encode(theAnimal, animal.duck.signature)
     buf.seek(0)
-    decoder = labcomm.Decoder(labcomm.StreamReader(buf))
+    decoder = labcomm2014.Decoder(labcomm2014.StreamReader(buf))
     try:
         while True:
             value,decl = decoder.decode()
             if value:
-                print decl.name, 'says', value.says
+                print decl.name, 'says', value
                 pass
             pass
         pass

examples/dynamic/Makefile

+all:
+	sh dynamic.sh
+	sh dynamic_type.sh
+	sh test.sh
+	sh test_type.sh
+
+clean:
+	-rm test/*.class
+	-rm encoded_data
+	-rm dynamic_out
+
+distclean: clean

examples/dynamic/dynamic.sh

 #dummy script to test the on-the-fly compilation
 
-java -jar ../../compiler/labComm.jar --java=gen --javapackage=gen simple.lc
+java -jar ../../compiler/labcomm2014_compiler.jar --java=gen --javapackage=gen simple.lc
 
-javac -cp .:gen:../../lib/java/labcomm.jar gen/*.java
+javac -cp .:gen:../../lib/java/labcomm2014.jar gen/*.java
 
 # compile static encoder and decoder 
-javac -cp .:gen:../../lib/java/labcomm.jar test/StaticEncoder.java
-javac -cp .:gen:../../lib/java/labcomm.jar test/StaticDecoder.java 
+javac -cp .:gen:../../lib/java/labcomm2014.jar test/StaticEncoder.java
+javac -cp .:gen:../../lib/java/labcomm2014.jar test/StaticDecoder.java 
 
 # compile dynamic part 
-javac -cp .:../../compiler/labComm.jar:../../lib/java/labcomm.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test/DynamicPart.java 
+javac -cp .:../../compiler/labcomm2014_compiler.jar:../../lib/java/labcomm2014.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test/DynamicPart.java 
 
 javac test/HandlerContext.java
 
 # run static encoder 
-java -cp .:gen:../../lib/java//labcomm.jar test.StaticEncoder encoded_data
+java -cp .:gen:../../lib/java//labcomm2014.jar test.StaticEncoder encoded_data
 
 # run dynamic part 
-java -cp .:../../compiler/labComm.jar:../../lib/java/labcomm.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test.DynamicPart simple.lc handlers2.txt encoded_data dynamic_out
+java -cp .:../../compiler/labcomm2014_compiler.jar:../../lib/java/labcomm2014.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test.DynamicPart simple.lc handlers2.txt encoded_data dynamic_out
 
 
 # run static decoder 
-java -cp .:gen:../../lib/java//labcomm.jar test.StaticDecoder dynamic_out
+java -cp .:gen:../../lib/java//labcomm2014.jar test.StaticDecoder dynamic_out

examples/dynamic/dynamic_type.sh

 #dummy script to test the on-the-fly compilation
 
-java -jar ../../compiler/labComm.jar --java=gen --javapackage=gen simple.lc
+java -jar ../../compiler/labcomm2014_compiler.jar --java=gen --javapackage=gen simple.lc
 
-javac -cp .:gen:../../lib/java/labcomm.jar gen/*.java
+javac -cp .:gen:../../lib/java/labcomm2014.jar gen/*.java
 
 # compile static encoder and decoder 
-javac -cp .:gen:../../lib/java/labcomm.jar test/StaticEncoder.java
-javac -cp .:gen:../../lib/java/labcomm.jar test/StaticDecoder.java 
+javac -cp .:gen:../../lib/java/labcomm2014.jar test/StaticEncoder.java
+javac -cp .:gen:../../lib/java/labcomm2014.jar test/StaticDecoder.java 
 
 # compile dynamic part 
-javac -cp .:../../compiler/labComm.jar:../../lib/java/labcomm.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test/DynamicPart.java 
+javac -cp .:../../compiler/labcomm2014_compiler.jar:../../lib/java/labcomm2014.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test/DynamicPart.java 
 
 javac test/HandlerContext.java
 
 # run static encoder 
-java -cp .:gen:../../lib/java//labcomm.jar test.StaticEncoder encoded_data
+java -cp .:gen:../../lib/java//labcomm2014.jar test.StaticEncoder encoded_data
 
 # run dynamic part 
-java -cp .:../../compiler/labComm.jar:../../lib/java/labcomm.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test.DynamicPart simple_type.lc handlers_type.txt encoded_data dynamic_out
+java -cp .:../../compiler/labcomm2014_compiler.jar:../../lib/java/labcomm2014.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test.DynamicPart simple_type.lc handlers_type.txt encoded_data dynamic_out
 
 
 # run static decoder 
-java -cp .:gen:../../lib/java//labcomm.jar test.StaticDecoder dynamic_out
+java -cp .:gen:../../lib/java//labcomm2014.jar test.StaticDecoder dynamic_out

examples/dynamic/static.sh

@@ -2,12 +2,12 @@
 
 # run labcomm compilation
 
-java -jar ../../compiler/labComm.jar --java=gen --javapackage=gen simple.lc
+java -jar ../../compiler/labcomm2014_compiler.jar --java=gen --javapackage=gen simple.lc
 
 # compile example programs
-javac -cp .:gen:../../lib/java/labcomm.jar test/StaticEncoder.java
-javac -cp .:gen:../../lib/java/labcomm.jar test/StaticDecoder.java
+javac -cp .:gen:../../lib/java/labcomm2014.jar test/StaticEncoder.java
+javac -cp .:gen:../../lib/java/labcomm2014.jar test/StaticDecoder.java
 
 # run example programs
-java -cp .:gen:../../lib/java//labcomm.jar test.StaticEncoder encoded_data
-java -cp .:gen:../../lib/java//labcomm.jar test.StaticDecoder encoded_data
+java -cp .:gen:../../lib/java//labcomm2014.jar test.StaticEncoder encoded_data
+java -cp .:gen:../../lib/java//labcomm2014.jar test.StaticDecoder encoded_data

examples/dynamic/test.sh

 #dummy script to test the on-the-fly compilation
 
-javac -cp .:../../compiler/labComm.jar:../../lib/java/labcomm.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test/TestLabcommGen.java 
+javac -cp .:../../compiler/labcomm2014_compiler.jar:../../lib/java/labcomm2014.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test/TestLabcommGen.java 
 
 javac test/HandlerContext.java
 
-java -cp .:../../compiler/labComm.jar:../../lib/java/labcomm.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test.TestLabcommGen simple.lc handlers2.txt encoded_data
+java -cp .:../../compiler/labcomm2014_compiler.jar:../../lib/java/labcomm2014.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar  test.TestLabcommGen simple.lc handlers2.txt encoded_data

examples/dynamic/test/DynamicPart.java

@@ -18,13 +18,13 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.Map;
 
-import se.lth.control.labcomm.LabCommDecoder;
-import se.lth.control.labcomm.LabCommDecoderChannel;
-import se.lth.control.labcomm.LabCommEncoder;
-import se.lth.control.labcomm.LabCommEncoderChannel;
-import AST.LabCommParser;
-import AST.LabCommScanner;
-import AST.Program;
+import se.lth.control.labcomm2014.Decoder;
+import se.lth.control.labcomm2014.DecoderChannel;
+import se.lth.control.labcomm2014.Encoder;
+import se.lth.control.labcomm2014.EncoderChannel;
+import se.lth.control.labcomm2014.compiler.LabCommParser;
+import se.lth.control.labcomm2014.compiler.LabCommScanner;
+import se.lth.control.labcomm2014.compiler.Program;
 import beaver.Parser.Exception;
 
 public class DynamicPart {
@@ -158,7 +158,7 @@ public class DynamicPart {
 	}
 
 	public static InRAMCompiler generateCode(String lcDecl, HashMap<String, String> handlers) {
-		Program ast = null;
+		Specification ast = null;
 		InputStream in = new ByteArrayInputStream(lcDecl.getBytes());
 		LabCommScanner scanner = new LabCommScanner(in);
 		LabCommParser parser = new LabCommParser();
@@ -166,7 +166,7 @@ public class DynamicPart {
 
 		InRAMCompiler irc = null;
 		try {
-			Program p = (Program)parser.parse(scanner);
+			Specification p = (Specification)parser.parse(scanner);
 			p.errorCheck(errors);
 			if (errors.isEmpty()) {
 				ast = p;
@@ -197,10 +197,10 @@ public class DynamicPart {
 	 * @param handlers - a map <name, source> of handlers for the types in ast
 	 * @return an InRAMCompiler object containing the generated clases
 	 */
-	private static InRAMCompiler handleAst(Program lcAST, HashMap<String, String> handlers) {
+	private static InRAMCompiler handleAst(Specification lcAST, HashMap<String, String> handlers) {
 		Map<String, String> genCode = new HashMap<String, String>();
 		try {
-			lcAST.J_gen(genCode, "labcomm.generated");
+			lcAST.J_gen(genCode, "labcomm.generated", 2014);
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
@@ -272,7 +272,7 @@ public class DynamicPart {
 	private void decodeTest(InRAMCompiler irc, HandlerContext ctxt, String tmpFile, String... sampleNames) {
 		try {
 			FileInputStream in = new FileInputStream(tmpFile);
-			LabCommDecoderChannel dec = new LabCommDecoderChannel(in);
+			DecoderChannel dec = new DecoderChannel(in);
 			Class handlerClass =  irc.load(handlerClassName);
 			Constructor hcc = handlerClass.getDeclaredConstructor(Object.class);
 			Object handler = hcc.newInstance(ctxt);
@@ -282,7 +282,7 @@ public class DynamicPart {
 				Class sampleClass = irc.load(sampleName);
 				Class handlerInterface = irc.load(sampleName+"$Handler");
 
-				Method reg = sampleClass.getDeclaredMethod("register", LabCommDecoder.class, handlerInterface);
+				Method reg = sampleClass.getDeclaredMethod("register", Decoder.class, handlerInterface);
 				reg.invoke(sampleClass, dec, handler);
 			}
 
@@ -324,20 +324,20 @@ public class DynamicPart {
 
 
 			FileOutputStream out = new FileOutputStream(tmpFile);
-			LabCommEncoderChannel enc = new LabCommEncoderChannel(out);
+			EncoderChannel enc = new EncoderChannel(out);
 
 			/* register and send foo */
-			Method regFoo = fc.getDeclaredMethod("register", LabCommEncoder.class);
+			Method regFoo = fc.getDeclaredMethod("register", Encoder.class);
 			regFoo.invoke(fc, enc);
 
-			Method doEncodeFoo = fc.getDeclaredMethod("encode", LabCommEncoder.class, ft);
+			Method doEncodeFoo = fc.getDeclaredMethod("encode", Encoder.class, ft);
 			doEncodeFoo.invoke(fc, enc, fv);
 
 			/* register and send bar (NB! uses primitive type int) */
-			Method regBar = bc.getDeclaredMethod("register", LabCommEncoder.class);
+			Method regBar = bc.getDeclaredMethod("register", Encoder.class);
 			regBar.invoke(bc, enc);
 
-			Method doEncodeBar = bc.getDeclaredMethod("encode", LabCommEncoder.class, Integer.TYPE);
+			Method doEncodeBar = bc.getDeclaredMethod("encode", Encoder.class, Integer.TYPE);
 			doEncodeBar.invoke(bc, enc, ctxt.bar);
 
 			out.close();

examples/dynamic/test/StaticDecoder.java

@@ -6,16 +6,16 @@ import gen.bar;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
-import se.lth.control.labcomm.LabCommDecoderChannel;
+import se.lth.control.labcomm2014.DecoderChannel;
 
 
 public class StaticDecoder implements foo.Handler, bar.Handler
 {
 
-  LabCommDecoderChannel decoder;
+  DecoderChannel decoder;
 
   public StaticDecoder(InputStream in) throws Exception {
-    decoder = new LabCommDecoderChannel(in);
+    decoder = new DecoderChannel(in);
     foo.register(decoder, this);
     bar.register(decoder, this);
 

examples/dynamic/test/StaticEncoder.java

@@ -2,19 +2,19 @@ package test;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.OutputStream;
-import se.lth.control.labcomm.LabCommEncoderChannel;
+import se.lth.control.labcomm2014.EncoderChannel;
 
 import gen.foo;
 import gen.bar;
 
 public class StaticEncoder {
 
-  LabCommEncoderChannel encoder;
+  EncoderChannel encoder;
 
   public StaticEncoder(OutputStream out) 
     throws Exception 
   {
-    encoder = new LabCommEncoderChannel(out);
+    encoder = new EncoderChannel(out);
     foo.register(encoder);
     bar.register(encoder);
   }

examples/dynamic/test/TestLabCommCompiler.java

@@ -16,18 +16,18 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.Map;
 
-import se.lth.control.labcomm.LabCommDecoder;
-import se.lth.control.labcomm.LabCommDecoderChannel;
-import se.lth.control.labcomm.LabCommEncoder;
-import se.lth.control.labcomm.LabCommEncoderChannel;
-import AST.LabCommParser;
-import AST.LabCommScanner;
-import AST.Program;
+import se.lth.control.labcomm2014.Decoder;
+import se.lth.control.labcomm2014.DecoderChannel;
+import se.lth.control.labcomm2014.Encoder;
+import se.lth.control.labcomm2014.EncoderChannel;
+import AST.Parser;
+import AST.Scanner;
+import AST.Specification;
 import beaver.Parser.Exception;
 
 
 
-public class TestLabCommCompiler {
+public class TestCompiler {
 
 	private static final String BAR = "bar";
 	private static final String FOO = "foo";
@@ -73,7 +73,7 @@ public class TestLabCommCompiler {
 	private static void decodeTest(InRAMCompiler irc, String tmpFile) {
 		try {
 			FileInputStream in = new FileInputStream(tmpFile);
-			LabCommDecoderChannel dec = new LabCommDecoderChannel(in);
+			DecoderChannel dec = new DecoderChannel(in);
 	
 			Class fc = irc.load(FOO);
 			Class hc = irc.load("gen_"+FOO+"Handler");
@@ -81,7 +81,7 @@ public class TestLabCommCompiler {
 
 			Object h = hc.newInstance(); 
 		
-			Method reg = fc.getDeclaredMethod("register", LabCommDecoder.class, hi);
+			Method reg = fc.getDeclaredMethod("register", Decoder.class, hi);
 			reg.invoke(fc, dec, h);
 			
 			dec.runOne();
@@ -106,11 +106,11 @@ public class TestLabCommCompiler {
 			z.setInt(f, 12);
 			
 			FileOutputStream out = new FileOutputStream(tmpFile);
-			LabCommEncoderChannel enc = new LabCommEncoderChannel(out);
-			Method reg = fc.getDeclaredMethod("register", LabCommEncoder.class);
+			EncoderChannel enc = new EncoderChannel(out);
+			Method reg = fc.getDeclaredMethod("register", Encoder.class);
 			reg.invoke(fc, enc);
 			
-			Method doEncode = fc.getDeclaredMethod("encode", LabCommEncoder.class, fc);
+			Method doEncode = fc.getDeclaredMethod("encode", Encoder.class, fc);
 			doEncode.invoke(fc, enc, f);
 			
 			out.close();
@@ -121,15 +121,15 @@ public class TestLabCommCompiler {
 
 	}
 	public static InRAMCompiler generateCode(String lcDecl, HashMap<String, String> handlers) {
-		Program ast = null;
+		Specification ast = null;
 		InputStream in = new ByteArrayInputStream(lcDecl.getBytes());
-		LabCommScanner scanner = new LabCommScanner(in);
-		LabCommParser parser = new LabCommParser();
+		Scanner scanner = new Scanner(in);
+		Parser parser = new Parser();
 		Collection errors = new LinkedList();
 
 		InRAMCompiler irc = null;
 		try {
-			Program p = (Program)parser.parse(scanner);
+			Specification p = (Specification)parser.parse(scanner);
 			p.errorCheck(errors);
 			if (errors.isEmpty()) {
 				ast = p;
@@ -207,10 +207,10 @@ public class TestLabCommCompiler {
 	 * @param handlers - a map <name, source> of handlers for the types in ast
 	 * @return an InRAMCompiler object containing the generated clases
 	 */
-	private static InRAMCompiler handleAst(Program lcAST, HashMap<String, String> handlers) {
+	private static InRAMCompiler handleAst(Specification lcAST, HashMap<String, String> handlers) {
 		Map<String, String> genCode = new HashMap<String, String>();
 		try {
-			lcAST.J_gen(genCode, "labcomm.generated");
+			lcAST.J_gen(genCode, "labcomm.generated", 2013);
 		} catch (IOException e) {
 			e.printStackTrace();
 		}

examples/dynamic/test/TestLabcommGen.java

@@ -18,13 +18,13 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.Map;
 
-import se.lth.control.labcomm.LabCommDecoder;
-import se.lth.control.labcomm.LabCommDecoderChannel;
-import se.lth.control.labcomm.LabCommEncoder;
-import se.lth.control.labcomm.LabCommEncoderChannel;
-import AST.LabCommParser;
-import AST.LabCommScanner;
-import AST.Program;
+import se.lth.control.labcomm2014.Decoder;
+import se.lth.control.labcomm2014.DecoderChannel;
+import se.lth.control.labcomm2014.Encoder;
+import se.lth.control.labcomm2014.EncoderChannel;
+import se.lth.control.labcomm2014.compiler.LabCommParser;
+import se.lth.control.labcomm2014.compiler.LabCommScanner;
+import se.lth.control.labcomm2014.compiler.Program;
 import beaver.Parser.Exception;
 
 public class TestLabcommGen {
@@ -161,7 +161,7 @@ public class TestLabcommGen {
 	}
 
 	public static InRAMCompiler generateCode(String lcDecl, HashMap<String, String> handlers) {
-		Program ast = null;
+		Specification ast = null;
 		InputStream in = new ByteArrayInputStream(lcDecl.getBytes());
 		LabCommScanner scanner = new LabCommScanner(in);
 		LabCommParser parser = new LabCommParser();
@@ -169,7 +169,7 @@ public class TestLabcommGen {
 
 		InRAMCompiler irc = null;
 		try {
-			Program p = (Program)parser.parse(scanner);
+			Specification p = (Specification)parser.parse(scanner);
 			p.errorCheck(errors);
 			if (errors.isEmpty()) {
 				ast = p;
@@ -200,10 +200,10 @@ public class TestLabcommGen {
 	 * @param handlers - a map <name, source> of handlers for the types in ast
 	 * @return an InRAMCompiler object containing the generated clases
 	 */
-	private static InRAMCompiler handleAst(Program lcAST, HashMap<String, String> handlers) {
+	private static InRAMCompiler handleAst(Specification lcAST, HashMap<String, String> handlers) {
 		Map<String, String> genCode = new HashMap<String, String>();
 		try {
-			lcAST.J_gen(genCode, "labcomm.generated");
+			lcAST.J_gen(genCode, "labcomm.generated", 2014);
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
@@ -284,10 +284,10 @@ public class TestLabcommGen {
 	 * @param handlers - a map <name, source> of handlers for the types in ast
 	 * @return an InRAMCompiler object containing the generated clases
 	 */
-	private static InRAMCompiler handleAstSeparate(Program lcAST, HashMap<String, String> handlers) {
+	private static InRAMCompiler handleAstSeparate(Specification lcAST, HashMap<String, String> handlers) {
 		Map<String, String> genCode = new HashMap<String, String>();
 		try {
-			lcAST.J_gen(genCode, "labcomm.generated");
+			lcAST.J_gen(genCode, "labcomm.generated", 2013);
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
@@ -332,7 +332,7 @@ public class TestLabcommGen {
 	private static void decodeTest(InRAMCompiler irc, String tmpFile, String... sampleNames) {
 		try {
 			FileInputStream in = new FileInputStream(tmpFile);
-			LabCommDecoderChannel dec = new LabCommDecoderChannel(in);
+			DecoderChannel dec = new DecoderChannel(in);
 			Class handlerClass =  irc.load(handlerClassName);
 			Constructor hcc = handlerClass.getDeclaredConstructor(Object.class);
 //			Object handler = handlerClass.newInstance(); 
@@ -344,7 +344,7 @@ public class TestLabcommGen {
 				Class sampleClass = irc.load(sampleName);
 				Class handlerInterface = irc.load(sampleName+"$Handler");
 
-				Method reg = sampleClass.getDeclaredMethod("register", LabCommDecoder.class, handlerInterface);
+				Method reg = sampleClass.getDeclaredMethod("register", Decoder.class, handlerInterface);
 				reg.invoke(sampleClass, dec, handler);
 			}
 
@@ -389,20 +389,20 @@ public class TestLabcommGen {
 
 
 			FileOutputStream out = new FileOutputStream(tmpFile);
-			LabCommEncoderChannel enc = new LabCommEncoderChannel(out);
+			EncoderChannel enc = new EncoderChannel(out);
 
 			/* register and send foo */
-			Method regFoo = fc.getDeclaredMethod("register", LabCommEncoder.class);
+			Method regFoo = fc.getDeclaredMethod("register", Encoder.class);
 			regFoo.invoke(fc, enc);
 
-			Method doEncodeFoo = fc.getDeclaredMethod("encode", LabCommEncoder.class, ft);
+			Method doEncodeFoo = fc.getDeclaredMethod("encode", Encoder.class, ft);
 			doEncodeFoo.invoke(fc, enc, f);
 
 			/* register and send bar (NB! uses primitive type int) */
-			Method regBar = bc.getDeclaredMethod("register", LabCommEncoder.class);
+			Method regBar = bc.getDeclaredMethod("register", Encoder.class);
 			regBar.invoke(bc, enc);
 
-			Method doEncodeBar = bc.getDeclaredMethod("encode", LabCommEncoder.class, Integer.TYPE);
+			Method doEncodeBar = bc.getDeclaredMethod("encode", Encoder.class, Integer.TYPE);
 			doEncodeBar.invoke(bc, enc, 42);
 
 			out.close();
@@ -475,7 +475,7 @@ public class TestLabcommGen {
 	private static void decodeTestSeparate(InRAMCompiler irc, String tmpFile, String... sampleNames) {
 		try {
 			FileInputStream in = new FileInputStream(tmpFile);
-			LabCommDecoderChannel dec = new LabCommDecoderChannel(in);
+			DecoderChannel dec = new DecoderChannel(in);
 			for (String sampleName : sampleNames) {
 				System.out.println("registering handler for "+sampleName);
 				Class sampleClass = irc.load(sampleName);
@@ -484,7 +484,7 @@ public class TestLabcommGen {
 
 				Object handler = handlerClass.newInstance(); 
 
-				Method reg = sampleClass.getDeclaredMethod("register", LabCommDecoder.class, handlerInterface);
+				Method reg = sampleClass.getDeclaredMethod("register", Decoder.class, handlerInterface);
 				reg.invoke(sampleClass, dec, handler);
 			}