git.net

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[GitHub] mbeckerle closed pull request #65: Added AIS layer with tests. Added tests matching to layer wiki page.


mbeckerle closed pull request #65: Added AIS layer with tests. Added tests matching to layer wiki page.
URL: https://github.com/apache/incubator-daffodil/pull/65
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/AIS_PAYLOAD_ARMORING.scala b/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/AISPayloadArmoring.scala
similarity index 93%
rename from daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/AIS_PAYLOAD_ARMORING.scala
rename to daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/AISPayloadArmoring.scala
index 115904442..b885b33e1 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/AIS_PAYLOAD_ARMORING.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/AISPayloadArmoring.scala
@@ -26,11 +26,11 @@ import org.apache.daffodil.schema.annotation.props.gen.BitOrder
  *
  * http://catb.org/gpsd/AIVDM.html#_aivdm_aivdo_payload_armoring
  *
- * To convert a string of length N, You will get 6N bits.
+ * To convert a string of length N bytes, You will get 6N bits.
  *
  * The decoder can be used for unit testing, but the point of this class
  * is to make the encoder available for use in un-doing the AIS Payload
- * armoring.
+ * armoring when parsing, and performing this armoring when unparsing.
  *
  * When encoding from 8-bit say, ascii, or iso-8859-1, this can only encode
  * things that stay within the 64 allowed characters.
diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd
index 8ed5b82ba..1d1635c33 100644
--- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd
+++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd
@@ -765,7 +765,7 @@
            </xsd:documentation>
          </xsd:annotation>
        </xsd:enumeration>      
-       <xsd:enumeration value="ais">
+       <xsd:enumeration value="aisPayloadArmor">
          <xsd:annotation>
            <xsd:documentation>
              Automatic Identification System - ITU-R M.1371-1
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/AISTransformer.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/AISTransformer.scala
new file mode 100644
index 000000000..82b2502c7
--- /dev/null
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/AISTransformer.scala
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.layers
+
+import org.apache.daffodil.schema.annotation.props.gen.LayerLengthKind
+import org.apache.daffodil.schema.annotation.props.gen.LayerLengthUnits
+import org.apache.daffodil.util.Maybe
+import org.apache.daffodil.processors.TermRuntimeData
+import org.apache.daffodil.processors.LayerLengthInBytesEv
+import org.apache.daffodil.processors.LayerBoundaryMarkEv
+import org.apache.daffodil.processors.LayerCharsetEv
+import org.apache.daffodil.processors.parsers.PState
+import org.apache.daffodil.processors.unparsers.UState
+import org.apache.daffodil.processors.charset.AIS_PAYLOAD_ARMORING
+import java.nio._
+import java.nio.charset._
+import java.io._
+import org.apache.commons.io.IOUtils
+import org.apache.daffodil.exceptions.Assert
+import org.apache.daffodil.io.BoundaryMarkLimitingStream
+import org.apache.daffodil.io.LayerBoundaryMarkInsertingJavaOutputStream
+
+object AISPayloadArmoringTransformer {
+  val iso8859 = StandardCharsets.ISO_8859_1
+}
+
+class AISPayloadArmoringTransformer()
+  extends LayerTransformer() {
+  import AISPayloadArmoringTransformer._
+
+  /**
+   * Decoding AIS payload armoring is encoding the ASCII text into the
+   * underlying binary data.
+   */
+  override def wrapLayerDecoder(jis: java.io.InputStream) = {
+    new AISPayloadArmoringInputStream(jis)
+  }
+
+  override def wrapLimitingStream(jis: java.io.InputStream, state: PState) = {
+    val layerBoundaryMark = ","
+    val s = BoundaryMarkLimitingStream(jis, layerBoundaryMark, iso8859)
+    s
+  }
+
+  override protected def wrapLayerEncoder(jos: java.io.OutputStream): java.io.OutputStream = {
+    new AISPayloadArmoringOutputStream(jos)
+  }
+
+  override protected def wrapLimitingStream(jos: java.io.OutputStream, state: UState): java.io.OutputStream = {
+    val layerBoundaryMark = ","
+    val newJOS = new LayerBoundaryMarkInsertingJavaOutputStream(jos, layerBoundaryMark, iso8859)
+    newJOS
+  }
+}
+
+class AISPayloadArmoringInputStream(jis: InputStream)
+  extends InputStream {
+  import AISPayloadArmoringTransformer._
+
+  private lazy val enc = AIS_PAYLOAD_ARMORING.newEncoder()
+
+  private lazy val bais = {
+    val armoredText = IOUtils.toString(jis, iso8859)
+    val cb = CharBuffer.wrap(armoredText)
+
+    val numBytes = (6 * armoredText.length) / 8
+    val ba = new Array[Byte](numBytes + 1)
+    val bb = ByteBuffer.wrap(ba)
+    val cr = enc.encode(cb, bb, true)
+    //
+    // We made bb have one extra byte in it.
+    // So this should end on UNDERFLOW meaning it could take more characters.
+    //
+    Assert.invariant(cr.isUnderflow())
+
+    val bais = new ByteArrayInputStream(ba, 0, numBytes)
+    bais
+  }
+
+  override def read(): Int = {
+    bais.read()
+  }
+}
+
+class AISPayloadArmoringOutputStream(jos: java.io.OutputStream)
+  extends OutputStream {
+  import AISPayloadArmoringTransformer._
+
+  private lazy val dec = AIS_PAYLOAD_ARMORING.newDecoder()
+
+  private val baos = new ByteArrayOutputStream()
+
+  private var closed = false
+
+  override def close(): Unit = {
+    if (!closed) {
+      val ba = baos.toByteArray()
+      val bb = ByteBuffer.wrap(ba)
+      // must call our own decode. This is not a regular java CharsetDecoder, its Daffodil's BitsCharsetDecoder
+      val cb = dec.decode(bb)
+      IOUtils.write(cb.toString, jos, iso8859)
+      jos.close()
+      closed = true
+    }
+  }
+
+  override def write(bInt: Int): Unit = {
+    baos.write(bInt)
+  }
+
+}
+
+object AISPayloadArmoringTransformerFactory
+  extends LayerTransformerFactory("aisPayloadArmor") {
+
+  override def newInstance(maybeLayerCharsetEv: Maybe[LayerCharsetEv],
+    maybeLayerLengthKind: Maybe[LayerLengthKind],
+    maybeLayerLengthInBytesEv: Maybe[LayerLengthInBytesEv],
+    maybeLayerLengthUnits: Maybe[LayerLengthUnits],
+    maybeLayerBoundaryMarkEv: Maybe[LayerBoundaryMarkEv],
+    trd: TermRuntimeData): LayerTransformer = {
+
+    val xformer = new AISPayloadArmoringTransformer()
+    xformer
+  }
+}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
index a4a6bc26c..a5f7a6c85 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
@@ -101,6 +101,7 @@ object LayerTransformerFactory {
   register(GZIPTransformerFactory)
   register(IMFLineFoldedTransformerFactory)
   register(ICalendarLineFoldedTransformerFactory)
+  register(AISPayloadArmoringTransformerFactory)
 }
 
 /**
diff --git a/daffodil-runtime1/src/test/scala/org/apache/daffodil/layers/TestAISStreams.scala b/daffodil-runtime1/src/test/scala/org/apache/daffodil/layers/TestAISStreams.scala
new file mode 100644
index 000000000..45a0888de
--- /dev/null
+++ b/daffodil-runtime1/src/test/scala/org/apache/daffodil/layers/TestAISStreams.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.layers
+
+import junit.framework.Assert._
+import java.io._
+import org.junit.Test
+import org.apache.daffodil.io.RegexLimitingStream
+import java.nio.charset.StandardCharsets
+import org.apache.daffodil.util.Misc
+import org.apache.commons.io.IOUtils
+import org.apache.daffodil.io.LayerBoundaryMarkInsertingJavaOutputStream
+
+/**
+ * AISPayloadArmoring Stream unit tests
+ *
+ * Results verified against this site: http://www.bosunsmate.org/ais/#bitvector
+ * However, that site has the data string wrong. The proper dataString
+ * is below.
+ */
+class TestAISPayloadArmoringStreams {
+
+  val iso8859 = StandardCharsets.ISO_8859_1
+
+  /**
+   * Shows that the regex will limit length and then AISPayloadArmoring decode does what
+   * it is allowed to decode.
+   */
+  @Test def testAISPayloadArmoringDecode() = {
+    val dataString = "14eGL:@000o8oQ'LMjOchmG@08HK,"
+    val bba = new ByteArrayInputStream(dataString.getBytes(iso8859))
+    //
+    // regex is ",0*"
+    //
+    val rls = new RegexLimitingStream(bba, ",", ",", iso8859)
+    val aas = new AISPayloadArmoringInputStream(rls)
+
+    val baos = new ByteArrayOutputStream()
+    var c: Int = -1
+    while ({
+      c = aas.read()
+      c != -1
+    }) {
+      baos.write(c)
+    }
+    baos.close()
+    val result = baos.toByteArray()
+    val expected = Misc.bits2Bytes("000001 000100 101101 010111 011100 001010 010000 000000 000000 000000 110111 001000 110111 100001 101000 011100 011101 110010 011111 101011 110000 110101 010111 010000 000000 001000 011000 011011 ")
+    assertEquals(expected.length, result.length)
+    val pairs = expected zip result
+    pairs.foreach {
+      case (exp, act) =>
+        assertEquals(exp, act)
+    }
+  }
+
+  @Test def testAISPayloadArmoringEncode() = {
+    val dataBytes = Misc.bits2Bytes("000001 000100 101101 010111 011100 001010 010000 000000 000000 000000 110111 001000 110111 100001 101000 011100 011101 110010 011111 101011 110000 110101 010111 010000 000000 001000 011000 011011 ")
+    val bais = new ByteArrayInputStream(dataBytes)
+    val baos = new ByteArrayOutputStream()
+    val lbmijos = new LayerBoundaryMarkInsertingJavaOutputStream(baos, ",", iso8859)
+    val aas = new AISPayloadArmoringOutputStream(lbmijos)
+    IOUtils.copy(bais, aas)
+    aas.close()
+    val result = baos.toByteArray()
+    val expected = "14eGL:@000o8oQ'LMjOchmG@08HK,".getBytes(iso8859)
+    assertEquals(expected.length, result.length)
+    (expected zip result).foreach {
+      case (exp, act) =>
+        assertEquals(exp, act)
+    }
+  }
+
+}
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/layers/ais.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/layers/ais.tdml
new file mode 100644
index 000000000..247ed93ab
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/layers/ais.tdml
@@ -0,0 +1,139 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<tdml:testSuite xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+  xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"; xmlns:xs="http://www.w3.org/2001/XMLSchema"; xmlns:fn="http://www.w3.org/2005/xpath-functions";
+  xmlns:ex="http://example.com"; xmlns:tns="http://example.com"; defaultRoundTrip="true">
+
+<!-- 
+Examples:
+!AIVDM,1,1,,A,13aEOK?P00PD2wVMdLDRhgvL289?,0*26
+!AIVDM,1,1,,B,16S`2cPP00a3UF6EKT@2:?vOr0S2,0*00
+!AIVDM,2,1,9,B,53nFBv01SJ<thHp6220H4heHTf2222222222221?50:454o<`9QSlUDp,0*09
+!AIVDM,2,2,9,B,888888888888880,2*2E
+
+see http://catb.org/gpsd/AIVDM.html
+
+Example:
+!AIVDM,1,1,,B,177KQJ5000G?tO`K>RA1wUbN0TKH,0*5C
+
+In the above, after the glop field's trailing boundaryMark comma, the 0 is n fill bits 
+required to pad the data payload to a 6-bit boundary. So it is 0 to 5. 
+
+The *5C is a NMEA 0183 data integrity checksum, preceded by "*" - computed on 
+the whole message including the AIVDM, but excluding the "!"
+
+Seems another role for layers is computing checksums.
+
+Example Multifragment sentence:
+!AIVDM,2,1,3,B,55P5TL01VIaAL@7WKO@mBplU@<PDhh000000001S;AJ::4A80?4i@E53,0*3E
+!AIVDM,2,2,3,B,1@0000000000000,2*55
+
+ -->
+  <tdml:defineSchema name="ais1" elementFormDefault="unqualified">
+
+    <dfdl:defineFormat name="aisText">
+      <dfdl:format ref="ex:GeneralFormat" lengthKind="delimited" />
+    </dfdl:defineFormat>
+    <dfdl:defineFormat name="aisEncoding">
+      <dfdl:format ref="ex:GeneralFormat" layerTransform="aisPayloadArmor" layerLengthUnits="bytes" />
+    </dfdl:defineFormat>
+    <dfdl:defineFormat name="aisBinary">
+      <dfdl:format ref="ex:GeneralFormat" lengthKind="explicit" lengthUnits="bits" alignment="1" alignmentUnits="bits"
+        representation="binary" byteOrder="bigEndian" bitOrder="mostSignificantBitFirst" binaryBooleanTrueRep="1" binaryBooleanFalseRep="0" />
+    </dfdl:defineFormat>
+
+    <dfdl:format ref="ex:aisBinary" />
+
+    <xs:element name="AIVDM" dfdl:initiator="!AIVDM" dfdl:ref="ex:aisText" dfdl:lengthKind="implicit">
+      <xs:complexType>
+        <xs:sequence dfdl:separator="," dfdl:separatorPosition="prefix" dfdl:ref="ex:aisText">
+          <xs:element name="numSentences" type="xs:unsignedInt" dfdl:ref="ex:aisText" />
+          <!-- could add facets minValue is 1 -->
+          <xs:element name="sentenceNumber" type="xs:unsignedInt" dfdl:ref="ex:aisText" />
+          <!-- could add facets minValue is 1 -->
+          <xs:element name="sequentialMessageID" type="xs:string" dfdl:ref="ex:aisText" />
+          <xs:element name="AISChannel" type="xs:string" dfdl:ref="ex:aisText" />
+          <!-- could add enum. One of A, B, 1, 2 -->
+          <xs:sequence>
+            <xs:sequence dfdl:ref="aisEncoding">
+              <xs:sequence>
+                <!-- now using default format which is aisBinary -->
+                <xs:element name="messageType" type="xs:unsignedInt" dfdl:length="6" />
+                <xs:element name="repeatIndicator" type="xs:unsignedInt" dfdl:length="2" />
+                <xs:element name="userID" type="xs:unsignedInt" dfdl:length="30" />
+                <xs:element name="navigationStatus" type="xs:unsignedInt" dfdl:length="4" />
+                <xs:element name="rateOfTurn" type="xs:unsignedInt" dfdl:length="8" />
+                <xs:element name="speedOverGround" type="xs:unsignedInt" dfdl:length="10" />
+                <xs:element name="positionAccuracy" type="xs:boolean" dfdl:length="1" />
+                <xs:element name="longitude" type="xs:int" dfdl:length="28" />
+                <xs:element name="latitude" type="xs:int" dfdl:length="27" />
+                <xs:element name="courseOverGround" type="xs:unsignedInt" dfdl:length="12" />
+                <xs:element name="trueHeading" type="xs:unsignedInt" dfdl:length="9" />
+                <xs:element name="timeStamp" type="xs:unsignedInt" dfdl:length="6" />
+                <xs:element name="regional" type="xs:unsignedInt" dfdl:length="2" />
+                <xs:element name="spare" type="xs:unsignedInt" dfdl:length="3" />
+                <xs:element name="RAIM" type="xs:unsignedInt" dfdl:length="1" />
+                <xs:element name="radioStatus" type="xs:unsignedInt" dfdl:length="19" />
+              </xs:sequence>
+            </xs:sequence>
+            <xs:element name="bitsPad" type="xs:unsignedInt" dfdl:ref="ex:aisText" dfdl:lengthKind="explicit" dfdl:length="1" />
+              <!--  must be 0 to 5 -->
+            <xs:element name="checksum" type="xs:string" dfdl:ref="ex:aisText" dfdl:initiator="*" />
+          </xs:sequence>
+        </xs:sequence>
+      </xs:complexType>
+    </xs:element>
+
+  </tdml:defineSchema>
+
+
+  <tdml:parserTestCase name="ais1" root="AIVDM" model="ais1" roundTrip="true">
+    <tdml:document>
+      <tdml:documentPart type="text"><![CDATA[!AIVDM,1,1,,A,13aEOK?P00PD2wVMdLDRhgvL289?,0*26]]></tdml:documentPart>
+    </tdml:document>
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <ex:AIVDM>
+          <numSentences>1</numSentences>
+          <sentenceNumber>1</sentenceNumber>
+          <sequentialMessageID />
+          <AISChannel>A</AISChannel>
+          <messageType>1</messageType>
+          <repeatIndicator>0</repeatIndicator>
+          <userID>244670316</userID>
+          <navigationStatus>15</navigationStatus>
+          <rateOfTurn>128</rateOfTurn>
+          <speedOverGround>0</speedOverGround>
+          <positionAccuracy>true</positionAccuracy>
+          <longitude>2627571</longitude>
+          <latitude>31136850</latitude>
+          <courseOverGround>706</courseOverGround>
+          <trueHeading>511</trueHeading>
+          <timeStamp>14</timeStamp>
+          <regional>0</regional>
+          <spare>0</spare>
+          <RAIM>1</RAIM>
+          <radioStatus>33359</radioStatus>
+          <bitsPad>0</bitsPad>
+          <checksum>26</checksum>
+        </ex:AIVDM>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+  </tdml:parserTestCase>
+
+</tdml:testSuite>
\ No newline at end of file
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/layers/layers.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/layers/layers.tdml
index c5027c7b9..3fdfb58c6 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/layers/layers.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/layers/layers.tdml
@@ -109,7 +109,8 @@
       <xs:complexType>
         <xs:sequence dfdl:ref="folded">
           <xs:sequence>
-            <xs:element name="marker" dfdl:initiator="boundary=" type="xs:string" dfdl:terminator="%CR;%LF;" />
+            <xs:element name="marker" type="xs:string"
+              dfdl:initiator="boundary=" dfdl:terminator="%CR;%LF;" />
             <xs:element name="contents" dfdl:lengthKind="implicit" dfdl:initiator="{ fn:concat('--', ../marker, '%CR;%LF;') }">
               <xs:complexType>
                 <xs:sequence>
@@ -254,4 +255,134 @@ xxx]]></tdml:documentPart>
     </tdml:errors>
   </tdml:parserTestCase>
 
+  <tdml:defineSchema name="s4" elementFormDefault="unqualified">
+
+    <dfdl:defineFormat name="general">
+      <dfdl:format ref="ex:GeneralFormat" lengthKind="delimited" outputNewLine="%CR;%LF;" layerEncoding="iso-8859-1"
+        layerLengthUnits='bytes' />
+    </dfdl:defineFormat>
+
+    <dfdl:defineFormat name="base64">
+      <dfdl:format ref="ex:general" layerTransform="base64_MIME" layerLengthKind="boundaryMark" />
+    </dfdl:defineFormat>
+
+    <dfdl:defineFormat name="gzip">
+      <dfdl:format ref="ex:general" layerTransform="gzip" layerLengthKind="explicit" />
+    </dfdl:defineFormat>
+
+    <dfdl:format ref="ex:general" />
+
+    <xs:element name="file" type="ex:fileType" />
+
+    <!-- broke this up to provide some resuable types and groups here -->
+
+    <xs:complexType name="fileType">
+      <!--
+           first we have the base64 details
+       -->
+      <xs:sequence dfdl:ref="ex:base64" dfdl:layerBoundaryMark="--END--">
+        <xs:sequence>
+          <!--
+              now the gzip details, including the 4-byte gzLength element that stores how long
+              the gzipped data is.
+           -->
+          <xs:element name="gzLength" type="xs:int" dfdl:representation="binary" dfdl:lengthKind="implicit"
+            dfdl:outputValueCalc="{ dfdl:contentLength( ../data, 'bytes') }" />
+          <!--
+             this 'data' element is needed only because we have to measure how big it is when unparsing.
+             If we were only worried about parsing, we woundn't need to have this extra 'data' element wrapped around
+             the contents.
+           -->
+          <xs:element name="data" dfdl:lengthKind="implicit">
+            <xs:complexType>
+              <!--
+                 now the gzipped layered sequence itself
+               -->
+              <xs:sequence dfdl:ref="ex:gzip" dfdl:layerLength="{ ../gzLength }">
+                <!--
+                  finally, inside that, we have the original fileTypeGroup group reference.
+                  -->
+                <xs:group ref="ex:fileTypeGroup" />
+              </xs:sequence>
+            </xs:complexType>
+          </xs:element>
+        </xs:sequence>
+      </xs:sequence>
+    </xs:complexType>
+
+    <xs:group name="fileTypeGroup">
+      <xs:sequence dfdl:separator="%NL;" dfdl:separatorPosition="postfix">
+        <xs:element name="header" minOccurs="0" maxOccurs="1" dfdl:occursCountKind="implicit">
+          <xs:complexType>
+            <xs:sequence dfdl:separator=",">
+              <xs:element name="title" type="xs:string" maxOccurs="unbounded" />
+            </xs:sequence>
+          </xs:complexType>
+        </xs:element>
+        <xs:element name="record" maxOccurs="unbounded">
+          <xs:complexType>
+            <xs:sequence dfdl:separator=",">
+              <xs:element name="item" type="xs:string" maxOccurs="unbounded" dfdl:occursCount="{ fn:count(../../header/title) }"
+                dfdl:occursCountKind="expression" />
+            </xs:sequence>
+          </xs:complexType>
+        </xs:element>
+      </xs:sequence>
+    </xs:group>
+
+  </tdml:defineSchema>
+
+  <tdml:unparserTestCase name="layers4" root="file" model="s4" roundTrip="true">
+    <tdml:document>
+    <!-- 
+    This data is gzipped, then base64 encoded:
+     
+last,first,middle,DOB
+smith,robert,brandon,1988-03-24
+johnson,john,henry,1986-01-23
+jones,arya,cat,1986-02-19 
+
+We gzip that, and then we must prepend that with the length (as a binary 4-byte int) before we base64 encode.
+  -->
+      <tdml:documentPart type="text" 
+      replaceDFDLEntities="true"
+      ><![CDATA[AAAAcx+LCAAAAAAAAAAtyUEKgCAQheG94E1mIDWittG+M0xpaNQIo5tuX0Kb98P7LioVjiTf3sn7%CR;
+K8CyzlqVO9UIkrcgFTYh9pnBTOOInUPba3XmyOX7WiEGlqfxgJ1B6xpzKEDyEOxUf7JoJq1e/RI4%CR;
+wXIAAAA=--END--]]></tdml:documentPart>
+    </tdml:document>
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <ex:file>
+          <gzLength>115</gzLength>
+          <data>
+            <header>
+              <title>last</title>
+              <title>first</title>
+              <title>middle</title>
+              <title>DOB</title>
+            </header>
+            <record>
+              <item>smith</item>
+              <item>robert</item>
+              <item>brandon</item>
+              <item>1988-03-24</item>
+            </record>
+            <record>
+              <item>johnson</item>
+              <item>john</item>
+              <item>henry</item>
+              <item>1986-01-23</item>
+            </record>
+            <record>
+              <item>jones</item>
+              <item>arya</item>
+              <item>cat</item>
+              <item>1986-02-19</item>
+            </record>
+          </data>
+        </ex:file>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+  </tdml:unparserTestCase>
+  
 </tdml:testSuite>
\ No newline at end of file
diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/layers/TestAIS.scala b/daffodil-test/src/test/scala/org/apache/daffodil/layers/TestAIS.scala
new file mode 100644
index 000000000..c81942d28
--- /dev/null
+++ b/daffodil-test/src/test/scala/org/apache/daffodil/layers/TestAIS.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.layers
+
+/* This section00 is for testing general features of DFDL that are
+ * not related to any specific requirement
+ */
+
+import org.junit.Test
+import org.apache.daffodil.tdml.Runner
+import org.junit.AfterClass
+
+object TestAISPayloadArmoring {
+  lazy val testDir = "/org/apache/daffodil/layers/"
+  lazy val runner = Runner(testDir, "ais.tdml")
+
+  @AfterClass def shutDown() {
+    runner.reset
+  }
+}
+
+class TestAISPayloadArmoring {
+
+  import TestAISPayloadArmoring._
+
+  @Test def test_ais1() { runner.runOneTest("ais1") }
+
+}
diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/layers/TestLayers.scala b/daffodil-test/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
index 0f9752db7..29f24b956 100644
--- a/daffodil-test/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
+++ b/daffodil-test/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
@@ -42,5 +42,6 @@ class TestLayers {
   @Test def test_layers2() { runner.runOneTest("layers2") }
   @Test def test_layers3() { runner.runOneTest("layers3") }
   @Test def test_layersErr1() { runner.runOneTest("layersErr1") }
+  @Test def test_layers4() { runner.runOneTest("layers4") }
 
 }


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@xxxxxxxxxxxxxxxx


With regards,
Apache Git Services