20141009 passes first tests, lots of cleanup still to do.

compiler/C_CodeGen.jrag

@@ -1188,12 +1188,12 @@ aspect C_Signature {
       }
     }
     env.println("};");
+    C_emitSizeofValue(env);
     env.println("struct labcomm"+env.verStr+"_signature labcomm"+env.verStr+"_signature_" + 
 		env.prefix + getName() + " = {");
     env.indent();
-    env.println("LABCOMM_SAMPLE, \"" + getName() + "\",");
-    env.println("(int (*)(struct labcomm"+env.verStr+"_signature *, void *))labcomm"+env.verStr+"_sizeof_" + 
-		env.prefix + getName() + ",");
+    env.println("\"" + getName() + "\",");
+    env.println("sizeof_" + env.prefix + getName() + ",");
     env.println("sizeof(signature_bytes_" + env.prefix + getName() + "),");
     env.println("signature_bytes_" + env.prefix + getName() + ",");
     env.println("0");
@@ -1260,6 +1260,22 @@ aspect C_Sizeof {
     return getType().C_fixedSizeof();
   }
 
+  public void Decl.C_emitSizeof(C_env env) {
+  }
+
+  public void SampleDecl.C_emitSizeof(C_env env) {
+    env = env.nestStruct("(*v)");
+    env.println("int labcomm"+env.verStr+"_sizeof_" + env.prefix + getName() +
+		"(" + env.prefix + getName() + " *v)");
+    env.println("{");
+    env.indent();
+    env.println("return labcomm_internal_sizeof(" +
+                "&labcomm" + env.verStr+"_signature_" + env.prefix + getName() +
+                ", v);");
+    env.unindent();
+    env.println("}");
+  }
+
   public int Type.C_fixedSizeof() {
     throw new Error(this.getClass().getName() + 
 		    ".C_fixedSizeof()" + 
@@ -1308,18 +1324,17 @@ aspect C_Sizeof {
     return getType().C_fixedSizeof() * elements;
   }
 
-  public void Decl.C_emitSizeof(C_env env) {
+  public void Decl.C_emitSizeofValue(C_env env) {
   }
 
-  public void SampleDecl.C_emitSizeof(C_env env) {
+  public void SampleDecl.C_emitSizeofValue(C_env env) {
     env = env.nestStruct("(*v)");
-    env.println("int labcomm"+env.verStr+"_sizeof_" + env.prefix + getName() +
-		"(" + env.prefix + getName() + " *v)");
+    env.println("static int sizeof_" + env.prefix + getName() + "(void *vv)");
     env.println("{");
     env.indent();
-    env.println("int result = labcomm"+env.verStr+"_size_packed32(labcomm"+env.verStr+"_signature_" + 
-                env.prefix + getName() +".index);");
+    env.println("int result = 0;");
     if (C_isDynamic()) {
+      env.println(env.prefix + getName() + " *v = vv;");
       getType().C_emitSizeof(env);
     } else {
       env.println("result += " + C_fixedSizeof() + ";");
@@ -1338,7 +1353,8 @@ aspect C_Sizeof {
   public void PrimType.C_emitSizeof(C_env env) {
     switch (getToken()) {
       case LABCOMM_STRING: { 
-	env.println("result += 0 + strlen(" + env.qualid + ");"); 
+        env.print("{ int length = strlen(" + env.qualid + "); ");
+	env.println("result += labcomm_size_packed32(length) + length; }"); 
       } break;
       default: { 
 	throw new Error(this.getClass().getName() + 

compiler/Python_CodeGen.jrag

@@ -83,6 +83,7 @@ aspect Python_CodeGen {
     env.println("import labcomm");
     env.println();
     Python_genTypes(env);
+/* Typedefs curenntly disabled
     env.println("typedef = [");
     env.indent();
     for (int i = 0 ; i < getNumDecl() ; i++) {
@@ -90,6 +91,7 @@ aspect Python_CodeGen {
     }
     env.unindent();
     env.println("]");
+*/
     env.println("sample = [");
     env.indent();
     for (int i = 0 ; i < getNumDecl() ; i++) {
@@ -105,11 +107,7 @@ aspect PythonTypes {
   
   public void Program.Python_genTypes(Python_env env) {
     for (int i = 0 ; i < getNumDecl() ; i++) {
-      env.println("class " + getDecl(i).getName() + "(object):");
-      env.indent();
       getDecl(i).Python_genSignature(env);
-      env.unindent();
-      env.println();
     }
   }
 
@@ -120,19 +118,29 @@ aspect PythonTypes {
   }
 
   public void TypeDecl.Python_genSignature(Python_env env) {
+/*
+    env.println("class " + getName() + "(object):");
+    env.indent();
     env.println("signature = labcomm.typedef('" + getName() + "',");
     env.indent();
     getType().Python_genSignature(env);
     env.unindent();
     env.println(")");
+    env.unindent();
+    env.println();
+*/
   }
 
   public void SampleDecl.Python_genSignature(Python_env env) {
+    env.println("class " + getName() + "(object):");
+    env.indent();
     env.println("signature = labcomm.sample('" + getName() + "', ");
     env.indent();
     getType().Python_genSignature(env);
     env.unindent();
     env.println(")");
+    env.unindent();
+    env.println();
   }
 
   public void UserType.Python_genSignature(Python_env env) {

lib/c/labcomm.c

@@ -260,3 +260,18 @@ int labcomm_get_local_index(struct labcomm_signature *signature)
   }
   return signature->index;
 }
+
+int labcomm_internal_sizeof(struct labcomm_signature *signature,
+                            void *v)
+{
+  int length = signature->encoded_size(v);
+  fprintf(stderr, "SIZEOF(%s) = %d %d %d\n",
+          signature->name, 
+          labcomm_size_packed32(signature->index),
+          labcomm_size_packed32(length),
+          length);
+  return (labcomm_size_packed32(signature->index) +
+          labcomm_size_packed32(length) +
+          length);
+}
+

lib/c/labcomm.h

@@ -43,9 +43,8 @@ struct labcomm_decoder;
  * Signature entry
  */
 struct labcomm_signature {
-  int type;
   char *name;
-  int (*encoded_size)(struct labcomm_signature *, void *); // void * == encoded_sample *
+  int (*encoded_size)(void *); /* void* refers to sample_data */
   int size;
   unsigned char *signature; 
   int index;

lib/c/labcomm_decoder.c

@@ -187,7 +187,7 @@ static int decode_sample(struct labcomm_decoder *d, int kind)
     .error = 0,
   };
   struct labcomm_signature signature, *local_signature;
-  int remote_index, local_index, err;
+  int remote_index, local_index, err, length;
   
   local_signature = NULL;
   local_index = 0;
@@ -195,7 +195,8 @@ static int decode_sample(struct labcomm_decoder *d, int kind)
   labcomm_writer_start(&writer, writer.action_context, 0, NULL, NULL);
   remote_index = labcomm_read_packed32(d->reader);
   signature.name = labcomm_read_string(d->reader);
-  signature.type = kind;
+  length = labcomm_read_packed32(d->reader);
+  fprintf(stderr, "SIGNATURE_LENGTH=%d\n", length);
   collect_flat_signature(d, &writer);
   labcomm_writer_end(&writer, writer.action_context);
   err = writer_ioctl(&writer, 
@@ -228,7 +229,6 @@ static int decode_sample(struct labcomm_decoder *d, int kind)
       s = LABCOMM_SIGNATURE_ARRAY_REF(d->memory, 
 				      d->local,  struct sample_entry, i);
       if (s->signature &&
-	  s->signature->type == signature.type &&
 	  s->signature->size == signature.size &&
 	  strcmp(s->signature->name, signature.name) == 0 &&
 	  memcmp((void*)s->signature->signature, (void*)signature.signature,
@@ -302,7 +302,7 @@ static void reader_alloc(struct labcomm_decoder *d)
 
 int labcomm_decoder_decode_one(struct labcomm_decoder *d)
 {
-  int result, remote_index;
+  int result, remote_index, length;
 
   reader_alloc(d);
   remote_index = labcomm_read_packed32(d->reader);
@@ -310,6 +310,12 @@ int labcomm_decoder_decode_one(struct labcomm_decoder *d)
     result = d->reader->error;
     goto out;
   }
+  length = labcomm_read_packed32(d->reader);
+  if (d->reader->error < 0) {
+    result = d->reader->error;
+    goto out;
+  }
+  fprintf(stderr, "LENGTH=%d\n", length);
   if (remote_index == LABCOMM_SAMPLE) {
     result = decode_sample(d, remote_index); 
   } else {

lib/c/labcomm_encoder.c

@@ -75,11 +75,10 @@ int labcomm_internal_encoder_register(
   labcomm_encoder_function encode)
 {
   int result = -EINVAL;
-  int index, *done, err, i;
+  int index, *done, err, i, length;
 
   index = labcomm_get_local_index(signature);
   labcomm_scheduler_writer_lock(e->scheduler);
-  if (signature->type != LABCOMM_SAMPLE) { goto out; }
   if (index <= 0) { goto out; }
   done = LABCOMM_SIGNATURE_ARRAY_REF(e->memory, e->registered, int, index);
   if (*done) { goto out; }
@@ -88,9 +87,26 @@ int labcomm_internal_encoder_register(
 			     index, signature, NULL);
   if (err == -EALREADY) { result = 0; goto out; }
   if (err != 0) { result = err; goto out; }
-  labcomm_write_packed32(e->writer, signature->type);
+  labcomm_write_packed32(e->writer, LABCOMM_SAMPLE);
+  length = (labcomm_size_packed32(index) +
+            labcomm_size_string(signature->name) +
+            labcomm_size_packed32(signature->size) +
+            signature->size);
+  {
+    fprintf(stderr, "LENGTH(%s, %d)=%d %d %d %d -> %d\n", 
+            signature->name, index,
+            labcomm_size_packed32(index),
+            labcomm_size_string(signature->name),
+            labcomm_size_packed32(signature->size),
+            signature->size,
+            length);
+
+  }
+
+  labcomm_write_packed32(e->writer, length);
   labcomm_write_packed32(e->writer, index);
   labcomm_write_string(e->writer, signature->name);
+  labcomm_write_packed32(e->writer, signature->size);
   for (i = 0 ; i < signature->size ; i++) {
     if (e->writer->pos >= e->writer->count) {
       labcomm_writer_flush(e->writer, e->writer->action_context);
@@ -111,16 +127,25 @@ int labcomm_internal_encode(
   labcomm_encoder_function encode,
   void *value)
 {
-  int result;
-  int index;
+  int result, index, length;
 
   index = labcomm_get_local_index(signature);
+  /* FIXME: try to gt rid of labcomm_size_packed32(index) in 
+            labcomm_sizeof_* (since that calculation is currently
+            wrong [length field not accounted for]) */
+  length = (signature->encoded_size(value));
+  {
+    fprintf(stderr, "LENGTH(%s, %d)=%d\n", 
+            signature->name, index,
+            length);
+  }
   labcomm_scheduler_writer_lock(e->scheduler);
   result = labcomm_writer_start(e->writer, e->writer->action_context, 
 				index, signature, value);
   if (result == -EALREADY) { result = 0; goto no_end; }
   if (result != 0) { goto out; }
   result = labcomm_write_packed32(e->writer, index);
+  result = labcomm_write_packed32(e->writer, length);
   if (result != 0) { goto out; }
   result = encode(e->writer, value);
 out:

lib/c/labcomm_private.h

@@ -377,6 +377,9 @@ int labcomm_internal_encoder_ioctl(struct labcomm_encoder *encoder,
 				   struct labcomm_signature *signature,
 				   uint32_t ioctl_action, va_list args);
 
+int labcomm_internal_sizeof(struct labcomm_signature *signature,
+                            void *v);
+
 #if __BYTE_ORDER == __LITTLE_ENDIAN
 
 #define LABCOMM_ENCODE(name, type)					\
@@ -445,7 +448,7 @@ static inline int labcomm_write_string(struct labcomm_writer *w, char *s)
 {
   int length, i, err; 
 
-  length = strlen((char*)s);
+  length = strlen(s);
   err = labcomm_write_packed32(w, length);
   if (err != 0) { return err; }
   for (i = 0 ; i < length ; i++) {
@@ -473,6 +476,13 @@ static inline int labcomm_size_packed32(unsigned int data)
 
 }
 
+static inline int labcomm_size_string(char *s)
+{
+  int length = strlen(s);
+  
+  return labcomm_size_packed32(length) + length;
+}
+
 /*
  * Macros for handling arrays indexed by signature index
  */

lib/c/test/test_labcomm.c

@@ -193,19 +193,19 @@ int main(void)
   labcomm_encoder_register_test_sample_test_var(encoder);
   err = test_decode_one(decoder);
   fprintf(stderr, "decode of register -> index %d\n", err);
-  test_encode_decode(encoder, decoder, 7, 1, 1);
+  test_encode_decode(encoder, decoder, 8, 1, 1);
   if (decoder_var.a[0] != encoder_var.a[0]) {
     fprintf(stderr, "Failed to decode correct value %d != %d\n", 
 	    encoder_var.a[0], decoder_var.a[0]);
     exit(1);
   }
-  test_encode_decode(encoder, decoder, 19, 2, 2);
-  test_encode_decode(encoder, decoder, 3, 0, 0);
+  test_encode_decode(encoder, decoder, 20, 2, 2);
+  test_encode_decode(encoder, decoder, 4, 0, 0);
   for (i = 1 ; i <= 4 ; i++) {
-    test_encode_decode(encoder, decoder, 2+i, 0, (1<<(7*i))-1);
-    test_encode_decode(encoder, decoder, 3+i, 0, (1<<(7*i)));
+    test_encode_decode(encoder, decoder, 3+i, 0, (1<<(7*i))-1);
+    test_encode_decode(encoder, decoder, 4+i, 0, (1<<(7*i)));
   }
-  test_encode_decode(encoder, decoder, 7, 0, 4294967295);
+  test_encode_decode(encoder, decoder, 8, 0, 4294967295);
   return 0;
 }
 

lib/c/test/test_labcomm_generated_encoding.c

@@ -190,23 +190,25 @@ int main(void)
     labcomm_pthread_scheduler_new(labcomm_default_memory));
 
   labcomm_encoder_ioctl(encoder, IOCTL_WRITER_RESET);
+  /* Register twice to make sure that only one registration gets encoded */
   labcomm_encoder_register_generated_encoding_V(encoder);
   labcomm_encoder_register_generated_encoding_V(encoder);
-  EXPECT({ 0x02, -1, 0x01, 'V', 0x11, 0x00 });
+  EXPECT({ 0x02, 0x06, -1, 0x01, 'V', 0x02, 0x11, 0x00 });
 
   labcomm_encoder_ioctl(encoder, IOCTL_WRITER_RESET);
+  /* Register twice to make sure that only one registration gets encoded */
   labcomm_encoder_register_generated_encoding_B(encoder);
   labcomm_encoder_register_generated_encoding_B(encoder);
-  EXPECT({0x02, -1, 0x01, 'B', 0x21});
+  EXPECT({0x02, 0x05, -1, 0x01, 'B',0x01, 0x21});
 
   labcomm_encoder_ioctl(encoder, IOCTL_WRITER_RESET);
   // was: labcomm_encode_generated_encoding_V(encoder, &V);
   labcomm_encode_generated_encoding_V(encoder);
-  EXPECT({-1});
+  EXPECT({-1, 0x00 });
 
   labcomm_encoder_ioctl(encoder, IOCTL_WRITER_RESET);
   labcomm_encode_generated_encoding_B(encoder, &B);
-  EXPECT({-1, 1});
+  EXPECT({-1, 0x01, 1});
 
   return 0;
 }

lib/csharp/se/lth/control/labcomm/LabCommDecoderChannel.cs

@@ -24,13 +24,22 @@ namespace se.lth.control.labcomm {
       bool done = false;
       while (!done) {
 	int tag = decodePacked32();
+        int length = decodePacked32();
+        Console.Error.WriteLine(" tag=" + tag + "length=" + length);
 	switch (tag) {
         case LabComm.SAMPLE: {
           int index = decodePacked32();
           String name = decodeString();
-	  MemoryStream signature = new MemoryStream();
-	  collectFlatSignature(new LabCommEncoderChannel(signature, false));
-	  registry.add(index, name, signature.ToArray());
+          int signature_length = decodePacked32();
+//	  MemoryStream signature = new MemoryStream();
+          byte[] signature = new byte[signature_length];
+//	  collectFlatSignature(new LabCommEncoderChannel(signature, false));
+          ReadBytes(signature, signature_length);
+          Console.Error.WriteLine("REMOTE:" + name + " " + 
+                                   signature_length + " " + 
+                                   signature);
+//	  registry.add(index, name, signature.ToArray());
+	  registry.add(index, name, signature);
         } break;
         default: {
           LabCommDecoderRegistry.Entry e = registry.get(tag);

lib/csharp/se/lth/control/labcomm/LabCommEncoderChannel.cs

@@ -11,11 +11,15 @@ namespace se.lth.control.labcomm {
     private MemoryStream bytes = new MemoryStream();
     private LabCommEncoderRegistry registry = new LabCommEncoderRegistry();
     byte[] buf = new byte[8];
+    private int current_tag; 
 
     public LabCommEncoderChannel(Stream writer, bool emitVersion) {
       this.writer = writer;
       if (emitVersion) {
 	encodeString(LabComm.VERSION);
+        bytes.WriteTo(writer);
+        bytes.SetLength(0);
+        writer.Flush();
       }
     }
 
@@ -24,26 +28,51 @@ namespace se.lth.control.labcomm {
 
     public void register(LabCommDispatcher dispatcher) {
       int index = registry.add(dispatcher);
-      encodePacked32(LabComm.SAMPLE);
+      begin(LabComm.SAMPLE);
       encodePacked32(index);
       encodeString(dispatcher.getName());
       byte[] signature = dispatcher.getSignature();
+      encodePacked32(signature.Length);
       for (int i = 0 ; i < signature.Length ; i++) {
 	encodeByte(signature[i]);
       }
       end(null);
     }
 
+    private void begin(int tag) {
+      current_tag = tag;
+      bytes.SetLength(0);
+      Console.Error.WriteLine("BEGIN CURRENT=" + current_tag + " TAG=" + tag + 
+                              "LENGTH=" + bytes.Length);
+    }
+
     public void begin(Type c) {
-      encodePacked32(registry.getTag(c));
+      begin(registry.getTag(c));
     }
 
     public void end(Type c) {
+      Console.Error.WriteLine("END CURRENT=" + current_tag + 
+                              "LENGTH=" + bytes.Length);
+      WritePacked32(writer, current_tag);
+      WritePacked32(writer, bytes.Length);
       bytes.WriteTo(writer);
       bytes.SetLength(0);
       writer.Flush();
     }
 
+    private void WritePacked32(Stream s, Int64 value) {
+      Console.Error.WriteLine("PACKED=" + value);
+      Int64 v = value & 0xffffffff;
+      int i;
+  
+      for (i = 0 ; i == 0 || v != 0 ; i++, v = (v >> 7)) {
+        buf[i] = (byte)(v & 0x7f | (i!=0?0x80:0x00));
+      }
+      for (i = i - 1 ; i >= 0 ; i--) {
+        s.WriteByte(buf[i]);
+      }      
+    }
+
     private void WriteInt(Int64 value, int length) {
       for (int i = length - 1 ; i >= 0 ; i--) {
 	buf[i] = (byte)(value & 0xff);
@@ -96,16 +125,7 @@ namespace se.lth.control.labcomm {
     }
 
     public void encodePacked32(Int64 value) {
-      byte[] tmp = new byte[5];
-      Int64 v = value & 0xffffffff;
-      int i;
-  
-      for (i = 0 ; i == 0 || v != 0 ; i++, v = (v >> 7)) {
-        tmp[i] = (byte)(v & 0x7f);
-      }
-      for (i = i - 1 ; i >= 0 ; i--) {
-        encodeByte((byte)(tmp[i] | (i!=0?0x80:0x00)));
-      }
+      WritePacked32(bytes, value);
     }
   }
 }

lib/java/se/lth/control/labcomm/LabCommDecoderChannel.java

@@ -4,6 +4,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.InputStream;
 import java.io.IOException;
+import java.io.EOFException;
 
 public class LabCommDecoderChannel implements LabCommDecoder {
 
@@ -25,13 +26,18 @@ public class LabCommDecoderChannel implements LabCommDecoder {
     boolean done = false;
     while (!done) {
       int tag = decodePacked32();
+      int length = decodePacked32();
       switch (tag) {
 	case LabComm.SAMPLE: {
 	  int index = decodePacked32();
 	  String name = decodeString();
-	  ByteArrayOutputStream signature = new ByteArrayOutputStream();
-	  collectFlatSignature(new LabCommEncoderChannel(signature, false));
-	  registry.add(index, name, signature.toByteArray());
+          int signature_length = decodePacked32();
+//	  ByteArrayOutputStream signature = new ByteArrayOutputStream();
+          byte[] signature = new byte[signature_length];
+//	  collectFlatSignature(new LabCommEncoderChannel(signature, false));
+          ReadBytes(signature, signature_length);
+//	  registry.add(index, name, signature.toByteArray());
+	  registry.add(index, name, signature);
 	} break;
 	default: {
 	  LabCommDecoderRegistry.Entry e = registry.get(tag);
@@ -100,6 +106,19 @@ public class LabCommDecoderChannel implements LabCommDecoder {
     registry.add(dispatcher, handler);
   }
 
+  private void ReadBytes(byte[] result, int length) throws IOException {
+      int offset = 0;
+      while (offset < length) {
+	int count = in.read(result, offset, length - offset);
+	if (count <= 0) {
+	  throw new EOFException(
+	    "End of stream reached with " +
+            (length - offset) + " bytes left to read");
+        }
+	offset += count;
+      }
+    }
+
   public boolean decodeBoolean() throws IOException {
     return in.readBoolean();
   }

lib/java/se/lth/control/labcomm/LabCommEncoderChannel.java

@@ -11,6 +11,7 @@ public class LabCommEncoderChannel implements LabCommEncoder {
   private ByteArrayOutputStream bytes;
   private DataOutputStream data;
   private LabCommEncoderRegistry registry;
+  private int current_tag; 
 
   public LabCommEncoderChannel(LabCommWriter writer, 
 			       boolean emitVersion) throws IOException {
@@ -20,6 +21,9 @@ public class LabCommEncoderChannel implements LabCommEncoder {
     registry = new LabCommEncoderRegistry();
     if (emitVersion) {
       encodeString(LabComm.VERSION);
+      data.flush();
+      writer.write(bytes.toByteArray());
+      bytes.reset();
     }
   }
 
@@ -38,28 +42,53 @@ public class LabCommEncoderChannel implements LabCommEncoder {
 
   public void register(LabCommDispatcher dispatcher) throws IOException {
     int index = registry.add(dispatcher);
-    encodePacked32(LabComm.SAMPLE);
+    begin(LabComm.SAMPLE);
     encodePacked32(index);
     encodeString(dispatcher.getName());
     byte[] signature = dispatcher.getSignature();
+    encodePacked32(signature.length);
     for (int i = 0 ; i < signature.length ; i++) {
       encodeByte(signature[i]);
     }
     end(null);
   }
 
+  private void begin(int tag) {
+    current_tag = tag;
+    bytes.reset();
+    System.err.println("BEGIN CURRENT=" + current_tag + " TAG=" + tag + 
+                       "LENGTH=" + bytes.size());
+  }
+
   public void begin(Class<? extends LabCommSample> c) throws IOException {
-    encodePacked32(registry.getTag(c));
+    begin(registry.getTag(c));
   }
 
   public void end(Class<? extends LabCommSample> c) throws IOException {
     data.flush();
-    //XXX when writer was a stream, it was probably a bit more GC efficient:
-    //bytes.writeTo(writer);
+    System.err.println("END CURRENT=" + current_tag + " " + 
+                       "LENGTH=" + bytes.size());
+    WritePacked32(writer, current_tag);
+    WritePacked32(writer, bytes.size());
     writer.write(bytes.toByteArray());
     bytes.reset();
   }
 
+  private void WritePacked32(LabCommWriter s, long value) throws IOException {
+    byte[] tmp1 = new byte[5];
+    byte[] tmp2 = new byte[1];
+    long v = value & 0xffffffff;
+    int i;
+    
+    for (i = 0 ; i == 0 || v != 0 ; i++, v = (v >> 7)) {
+      tmp1[i] = (byte)(v & 0x7f | (i!=0?0x80:0x00));
+    }
+    for (i = i - 1 ; i >= 0 ; i--) {
+      tmp2[0] = tmp1[i];
+      writer.write(tmp2);
+    }      
+  }
+
   public void encodeBoolean(boolean value) throws IOException{
     data.writeBoolean(value);