#!/usr/bin/env python3

"""DECnet protocol implementation

Classes for packet layouts.
"""

import sys
import struct
import time
from collections import OrderedDict

from .common import *
from . import logging

SvnFileRev = "$LastChangedRevision: 487 $"

# Exceptions related to packet definitions
class InvalidField (DNAException):
    """Invalid field descriptor."""
    
class ReadOnlyError (AttributeError): "Attempt to change a read-only attribute"

try:
    int.from_bytes
except AttributeError:
    raise ImportError ("Python 3.3 or later required")

# Checking for a bug in Python <= 3.2
if type (memoryview (b"ab")[0]) is not int:
    raise ImportError ("Python 3.3 or later required")

def fieldnum (fn):
    # Return the number part of "fieldnnn" as an integer
    return int (fn[5:])

def checkrowargs (ftype, name, args):
    "Verify that args (output from makecoderow) has the proper length"
    # Note we don't handle arg = defaultvar or *args yet (not needed
    # at the moment)
    for m, extra in (ftype.encode, 0), (ftype.decode, 1):
        # Check that encode/decode is correctly defined
        if hasattr (m, "__isabstractmethod__"):
            raise TypeError ("{}.{} is abstract".format (ftype.__name__,
                                                         m.__name__))
        maxcount = m.__code__.co_argcount - extra
        if name:
            # encode signature is encode (self, arg, ...)
            maxcount -= 1
        else:
            # encode signature is encode (ftype, packet, arg, ...)
            maxcount -= 2
        mincount = maxcount
        if m.__defaults__:
            mincount -= len (m.__defaults__)
        if m.__code__.co_flags & 4:
            # *x type argument give, so no max
            maxcount = 999999
        if not mincount <= len (args) <= maxcount:
            raise TypeError ("Wrong argument count {} for {} {}, expecting {} to {}".format (len (args), ftype.__name__, name, mincount, maxcount))
    
class FieldGroup (Field):
    """Abstract base class for packet elements that turn into several
    named fields.  These include BM, TLV, and NICE items.
    """
    __slots__ = ()

    def encode (self, packet):
        # Note that for these elements, the packet encoder passes an
        # additional argument (the packet object).
        assert False, "Subclass must supply encode"

    @classmethod
    def decode (self, buf, packet):
        # Note that for these elements, the packet decoder passes an
        # additional argument (the packet object).
        assert False, "Subclass must supply decode"
    
class I (Field, bytes):
    __slots__ = ()

    def encode (self, maxlen):
        vl = len (self)
        if vl > maxlen:
            logging.debug ("Value too long for {} byte field", maxlen)
            raise FieldOverflow
        return byte (vl) + self

    @classmethod
    def decode (cls, buf, maxlen):
        if not buf:
            logging.debug ("No data left for image field")
            raise MissingData
        flen = buf[0]
        if flen > maxlen:
            logging.debug ("Image field length {} longer than max length {}",
                           flen, maxlen)
            raise FieldOverflow
        v = buf[1:flen + 1]
        if len (v) != flen:
            logging.debug ("Not {} bytes left for image field", flen)
            raise MissingData
        return cls (v), buf[flen + 1:]

    def __format__ (self, format):
        return "-".join ("{:02x}".format (i) for i in self)

class A (Field, str):
    __slots__ = ()

    def encode (self, maxlen):
        v = bytes (self, encoding = "latin1")
        vl = len (v)
        if vl > maxlen:
            logging.debug ("Value too long for {} byte field", maxlen)
            raise FieldOverflow
        return byte (vl) + v

    @classmethod
    def decode (cls, buf, maxlen):
        if not buf:
            logging.debug ("No data left for image field")
            raise MissingData
        flen = buf[0]
        if flen > maxlen:
            logging.debug ("Image field length {} longer than max length {}",
                           flen, maxlen)
            raise FieldOverflow
        v = buf[1:flen + 1]
        if len (v) != flen:
            logging.debug ("Not {} bytes left for image field", flen)
            raise MissingData
        return cls (str (v, encoding = "latin1")), buf[flen + 1:]

class RES (FieldGroup):
    """A reserved field (ignored on input, zeroes on output).  We
    pretend this is a group field because it doesn't have a name.
    """
    __slots__ = ()

    @classmethod
    def encode (cls, packet, flen, pad):
        return pad

    @classmethod
    def decode (cls, buf, packet, flen, pad):
        if len (buf) < flen:
            logging.debug ("Not {} bytes left for reserved field", flen)
            raise MissingData
        return buf[flen:]
        
    @classmethod
    def makecoderow (cls, flen):
        # We calculate the pad (encoded value) at compile time because
        # it never changes, so there is no reason to recreate that bytes
        # object for every packet.
        return cls, None, (flen, bytes (flen)), (), False

class B (Field, int):
    "An unsigned integer of fixed length"
    __slots__ = ()

    def encode (self, flen):
        return self.to_bytes (flen, LE)

    @classmethod
    def decode (cls, buf, flen):
        if len (buf) < flen:
            logging.debug ("Not {} bytes left for integer field", flen)
            raise MissingData
        return cls (int.from_bytes (buf[:flen], LE)), buf[flen:]
        
class SIGNED (Field, int):
    "A signed integer of fixed length"
    __slots__ = ()

    def encode (self, flen):
        return self.to_bytes (flen, LE, signed = True)

    @classmethod
    def decode (cls, buf, flen):
        if len (buf) < flen:
            logging.debug ("Not {} bytes left for integer field", flen)
            raise MissingData
        return cls (int.from_bytes (buf[:flen], LE, signed = True)), buf[flen:]

class CTR (B):
    "Counter: like unsigned integer but capped at the max value for the size"
    __slots__ = ()

    def encode (self, flen):
        if self <= maxint[flen]:
            return super ().encode (flen)
        return maxint[flen].to_bytes (flen, LE)

class BM (FieldGroup):
    __slots__ = ()

    @classmethod
    def encode (cls, packet, flen, elements):
        """Encode a bitmap field.  "elements" is a sequence of
        tuples: name, starting bit position, bit count.
        """
        field = 0
        for name, start, bits, etype in elements:
            val = getattr (packet, name, 0)
            # If not of the correct type already, convert to (little
            # endian) integer.
            if not isinstance (val, etype):
                if val is None:
                    val = etype ()
                else:
                    val = etype (val)
            if val >> bits:
                logging.debug ("Field {} value {} too large for {} bit field",
                               name, val, bits)
                raise FieldOverflow
            field |= val << start
        return field.to_bytes (flen, LE)

    @classmethod
    def decode (cls, buf, packet, flen, elements):
        """Decode a bitmap field.  "elements" is a sequence of
        tuples: name, starting bit position, bit count type.  The
        fields are decoded according to ftype (which is int if not
        otherwise specified in the layout).  Returns the remaining
        buffer.
        """
        if len (buf) < flen:
            logging.debug ("Not {} bytes left for bit mapped field", flen)
            raise MissingData
        obj = cls ()
        field = int.from_bytes (buf[:flen], LE)
        for name, start, bits, etype in elements:
            val = etype ((field >> start) & ((1 << bits) - 1))
            try:
                setattr (packet, name, val)
            except ReadOnlyError:
                logging.debug ("Field required value mismatch: {}", name)
                raise WrongValue from None
            except ValueError:
                logging.debug ("Invalid field value: {}", name)
                raise WrongValue from None
        return buf[flen:]

    @classmethod
    def makecoderow (cls, *args):
        elements = list ()
        names = set ()
        # Find the field length in bytes
        topbit = -1
        fields = args
        for name, start, bits, *etype in args:
            if etype:
                etype = etype[0]
            else:
                etype = int
            topbit = max (topbit, start + bits - 1)
            elements.append ((name, start, bits, etype))
            names.add (name)
        flen = (topbit + 8) // 8
        return cls, None, (flen, elements), names, False
    
class EX (Field, int):
    "Extensible field"
    __slots__ = ()

    def encode (self, maxlen):
        val = int (self)
        retval = [ ]
        while val >> 7:
            retval.append (byte ((val & 0x7f) | 0x80))
            val >>= 7
        retval.append (byte (val))
        if len (retval) > maxlen:
            logging.debug ("Extensible field is longer than {} bytes", maxlen)
            raise FieldOverflow
        return b''.join (retval)

    @classmethod
    def decode (cls, buf, maxlen):
        val = 0
        for i in range (maxlen):
            if i >= len (buf):
                logging.debug ("EX field extends beyond end of data")
                raise MissingData
            b = buf[i]
            val |= (b & 0x7f) << (7 * i)
            if b < 0x80:
                break
            if i == maxlen - 1:
                logging.debug ("Extensible field longer than {}", maxlen)
                raise FieldOverflow
        return cls (val), buf[i + 1:]
            
class I_tlv (I):
    """A byte string inside a TLV item (shown as "I" in the specs)"""
    __slots__ = ()

    def encode (self, flen):
        return self

    @classmethod
    def decode (cls, buf, flen):
        return cls (buf), b""

class BV (I):
    "A fixed length byte string"
    __slots__ = ()

    def encode (self, flen):
        retval = makebytes (self)
        l = len (retval)
        if l < flen:
            retval += bytes (flen - l)
        elif l > flen:
            logging.debug ("Value too long for {} byte field", flen)
            raise FieldOverflow
        return retval
        
    @classmethod
    def decode (cls, buf, flen):
        if len (buf) < flen:
            logging.debug ("Not {} bytes left for bit string field", flen)
            raise MissingData
        return cls (buf[:flen]), buf[flen:]

class PAYLOAD (Field):
    "The remainder of the buffer"
    lastfield = True
    __slots__ = ("buf",)
    
    # Ideally PAYLOAD would be a subclass of memoryview, but subclassing
    # memoryview isn't allows at the moment so fake it.
    def __init__ (self, buf):
        buf = makebytes (buf)
        self.buf = buf

    def __bytes__ (self):
        return self.buf

    def encode (self):
        return self.buf
    
    @classmethod
    def decode (cls, buf):
        # We return the bufer itself, because often it's a memoryview
        # and we don't want to convert that.  If it has to be encoded
        # later on, the "checktype" method will take care of making it a
        # PAYLOAD object at that point.
        return buf, b""

# This is how we write an entry for payload in a layout list.  It
# supplies the type and the standard field name "payload".
Payload = (PAYLOAD, "payload")

class LIST (Field, list):
    "A list of items of a specified type"
    __slots__ = ()

    def encode (self, etype, count = None, *eargs):
        # Count is unused but present to match the decode signature
        return b"".join ((etype.checktype ("list", e)).encode (*eargs)
                         for e in self)

    @classmethod
    def decode (cls, buf, etype, count = None, *eargs):
        ret = cls ()
        if count is None:
            while buf:
                e, buf = etype.decode (buf, *eargs)
                ret.append (e)
        else:
            for i in range (count):
                e, buf = etype.decode (buf, *eargs)
                ret.append (e)
        return ret, buf
    
class TLV (FieldGroup):
    __slots__ = ()
    lastfield = True

    @classmethod
    def encode (cls, packet, tlen, llen, wild, codedict):
        retval = [ ]
        for k, v in codedict.items ():
            ftype, fname, fargs = v
            if fname:
                # Simple field
                v = getattr (packet, fname, None)
                if v is not None:
                    v = ftype.checktype (fname, v)
                    v = v.encode (*fargs)
            else:
                v = ftype.encode (packet, *fargs)
            if v is not None:
                retval.append (k.to_bytes (tlen, LE))
                retval.append (len (v).to_bytes (llen, LE))
                retval.append (v)
        return b''.join (retval)

    @classmethod
    def decode (cls, buf, packet, tlen, llen, wild, codedict):
        """Decode the remainder of the buffer as a sequence of TLV
        (tag, length, value) fields where tlen and llen are the length
        of the tag and length fields.  Each value field is decoded
        according to the decode rules given by the codedict entry
        keyed by the tag value.
        """
        pos = 0
        blen = len (buf)
        while pos < blen:
            left = blen - pos
            if left < tlen + llen:
                if packet.tolerant:
                    return b''
                logging.debug ("Incomplete TLV at end of buffer")
                raise MissingData
            tag = int.from_bytes (buf[pos:pos + tlen], LE)
            pos += tlen + llen
            vlen = int.from_bytes (buf[pos - llen:pos], LE)
            if pos + vlen > blen:
                logging.debug ("TLV {} Value field extends beyond end of buffer", tag)
                raise MissingData
            try:
                ftype, fname, fargs = codedict[tag]
            except KeyError:
                if wild:
                    ftype = I_tlv
                    fname = "field{}".format (tag)
                    packet._xfields = True
                    fargs = (llen,)
                else:
                    logging.debug ("Unknown TLV tag {}", tag)
                    raise InvalidTag from None
            if fname:
                # Simple field
                v, buf2 = ftype.decode (buf[pos:pos + vlen], *fargs)
                try:
                    setattr (packet, fname, v)
                except ReadOnlyError:
                    logging.debug ("Field required value mismatch: {}", fname)
                    raise WrongValue from None
                except ValueError:
                    logging.debug ("Invalid field value: {}", fname)
                    raise WrongValue from None
            else:
                buf2 = ftype.decode (buf[pos:pos + vlen], packet, *fargs)
            if buf2:
                if not packet.tolerant:
                    logging.debug ("TLV {} Value field not fully parsed, left = {}",
                                   tag, len (buf2))
                    raise ExtraData
            pos += vlen
        return None
    
    @classmethod
    def makecoderow (cls, *args):
        names = set ()
        tlen, llen, wild, *layout = args
        codedict = dict ()
        for k, ftype, *fargs in layout:
            if ftype is I:
                ftype = I_tlv
            if not issubclass (ftype, Field):
                raise InvalidField ("Invalid field type {}".format (ftype.__name__))
            ftype, fname, fargs, fnames, x  = ftype.makecoderow (*fargs)
            checkrowargs (ftype, fname, fargs)
            dups = names & fnames
            if dups:
                dups = ", ".join (n for n in dups)
                raise InvalidField ("Duplicate fields {} in layout".format (n))
            if ftype.lastfield:
                raise TypeError ("Invalid type {} inside TLV".format (ftype))
            names.update (fnames)
            codedict[k] = (ftype, fname, fargs)
        return cls, None, (tlen, llen, wild, codedict), names, wild
    
class indexer (type):
    """Metaclass that builds an index of the classes it creates, for use
    by packet code dependent class lookup.

    An indexed class is derived from a base class that has attribute
    "classindexkey".  It may be a class method, or a string.  If a class
    method, it is called with the new class as its argument to get the
    class index.  If a string, it will be used as the name of an
    attribute of the new class in which to find the index.  If the new
    class does not have that attribute but one of its base classes does,
    that value is used, but only if the index is not already in the
    index dictionary.  If successful and the result is not None, the
    class is then entered into the dictionary given by class attribute
    "classindex".

    Note that the base class (where "classindexkey" is defined) is not
    itself entered in its index.  But if it is a subclass of an earlier
    indexed class, it is entered there.  This allows the creation of
    multiple levels of indexing, where the first level finds a new class
    which in turn can be used to find a class in the second level.

    The Packet class can use this index mechanism to create different
    related packet formats that have a common header, and are recognized
    by the value of certain fields.  The Packet.decode method will
    automatically identify the correct class; see below for the
    details.
    """
    def __new__ (cls, name, bases, classdict):
        result = type.__new__ (cls, name, bases, classdict)
        try:
            # Look up attribute classindexkey in the base classes of the
            # new class.
            key = super (result, result).classindexkey
        except AttributeError:
            # Plain class, we're done.
            return result
        # Found it, but is it None to say "not actually indexed"?
        if not key:
            return result
        # Indexed class, find the index dictionary
        classindex = super (result, result).classindex
        if callable (key):
            idx = key ()
        else:
            # Attribute name.  Accept it (enter this class) if it
            # has a value for that name.  If not, use the value in
            # its base classes, if any, but only if there isn't
            # already an entry for that key.
            #
            # The result of this rule is that you can subclass
            # indexed classes and those subclasses will not replace
            # the base as the class index entries.  But you can form
            # indexed classes by inheriting from a root class and a
            # second base class that provides the key value; see
            # nice_coding.EventEntityBase and its subclasses for an
            # example.
            idx = classdict.get (key, None)
            if idx is None:
                idx2 = getattr (result, key, None)
                if idx2 not in classindex:
                    idx = idx2
        if idx is not None:
            classindex[idx] = result
        return result

class Indexed (metaclass = indexer):
    __slots__ = ()
    classindexkey = None
    
    @classmethod
    def defaultclass (cls, idx):
        """Return a default class if the class index doesn't list the
        supplied index value.  This method may return a particular
        default class, or it may generate a new class, or (as this
        method does) return None to indicate there isn't a class.
        """
        return None

    @classmethod
    def findclass (cls, idx):
        """Return the class whose index value matches the supplied one.

        If the supplied class has an index value and that matches what's
        requested, return this class as the preferred answer.  Otherwise
        return the class with matching index, if there is one.  If not,
        return what the defaultclass method returns for this index.
        """
        key = cls.classindexkey
        if callable (key):
            clsidx = key ()
        else:
            clsidx = getattr (cls, key, None)
        if clsidx == idx:
            return cls
        return cls.classindex.get (idx, None) or cls.defaultclass (idx)

class packet_encoding_meta (indexer):
    """Metaclass for "Packet" that will process the "_layout"
    for the packet into the necessary encoding and decoding
    tables.

    The layout is specified in class variable "_layout".
    The metaclass uses the layout definition to build an
    encode/decode table, which becomes class variable
    "_codetable".

    All fields mentioned in the layout, except those that are given
    values by class attributes, are mentioned in __slots__ so they
    become valid instance attributes.  However, if a TLV field group
    marked as "wild", or a NICE field group, is present in the layout,
    then generated field names may appear when decoding a packet with
    unknown elements, and in that case the __slots__ attribute is
    omitted from the class so any attribute name will be allowed.
    """
    def __new__ (cls, name, bases, classdict):
        packetbase = None
        for b in bases:
            # By the rules for __slots__, we allow just one base class
            # that defines a layout
            if isinstance (b, cls):
                assert packetbase is None, "Multiple Packet base classes"
                packetbase = b
        if packetbase is None:
            # Not a subclass of Packet, we're done
            return indexer.__new__ (cls, name, bases, classdict)
        allslots = set (packetbase._allslots)
        codetable = list (packetbase._codetable)
        # Remember if we have a field that must come last
        last = codetable and codetable[-1][0].lastfield
        if hasattr (packetbase, "__slots__"):
            # Base packet class has slots, we'll have them also unless
            # this layout is wild.
            slots = set ()
        else:
            # Base packet class has no slots, we don't either.
            slots = None
        layout = classdict.get ("_layout", ())
        classnames = frozenset (classdict)
        for ftype, fname, *args in layout:
            # Process the rows of the layout table
            if last:
                # Something after a "last" field
                raise InvalidField ("Extra field {} {}".format (ftype.__name__, fname))
            if not issubclass (ftype, Field):
                raise InvalidField ("Invalid field type {}".format (ftype.__name__))
            if fname:
                # Simple field.
                if fname in allslots:
                    raise InvalidField ("Duplicate field {} in layout".format (fname))
                if fname in classnames:
                    # Fixed value.  Make sure the class attribute has
                    # the correct type.  This isn't really required
                    # (since encode will force the correct type) but it
                    # avoids doing that fixup at runtime.
                    classdict[fname] = ftype.checktype (fname, classdict[fname])
            ftype, fname, args, newslots, wild = ftype.makecoderow (fname, *args)
            checkrowargs (ftype, fname, args)
            last = ftype.lastfield
            codetable.append ((ftype, fname, args))
            # Any attributes defined as class attributes will not be
            # created as instance attributes.
            newslots = set (newslots)
            newslots -= classnames
            allslots |= newslots
            slots |= newslots
            if wild:
                slots.add ("__dict__")
        # Must end up with some layout
        addslots = classdict.get ("_addslots", None)
        if not codetable and addslots is None:
            raise InvalidField ("Required attribute '_layout' "\
                                " not defined in class '{}'".format (name))
        # Add any extra slots requested by the class, then set __slots__
        if addslots:
            slots.update (addslots)
        classdict["__slots__"] = tuple (slots)
        classdict["_codetable"] = tuple (codetable)
        classdict["_allslots"] = tuple (allslots)
        return indexer.__new__ (cls, name, bases, classdict)
            
class Packet (Field, Indexed, metaclass = packet_encoding_meta):
    """Base class for DECnet packets.

    The packet layout is given by class variable "layout",
    which has to be set by the derived class definition.
    See below for detailed documentation.

    A packet object is essentially a struct, with elements of specified
    type in a specified order.  Usually it is used by itself, but it can
    also be used as an element in a (larger) packet object.  This is
    used in some places to handle common parts of a packet.

    The _layout class attribute is a sequence of tuples.  Each starts
    with the class name for this field (a subclass of Field), followed
    by a description for that field.  The format of the description
    depends on the field code:

    BM: description is a sequence of tuples, which together make up the
    bit field elements of the protocol field.  Each tuple consists of
    name, start bit position, bit count, and optionally the field type.
    If omitted, the type is unsigned integer.  The bit fields must be
    listed together and given in ascending order of bit position.  The
    size of the field is taken to be the minimal number of bytes needed
    to hold all the bit fields.

    I, A, B, EX: description is name and length.  For I, A, and EX,
    length means the maximum length.  A means the value is interpreted
    as text (str type); for the others, the value is type "bytes".

    SIGNED is like B except that the value is interpreted as a signed
    rather than an unsigned integer. 

    CTR is like B except that when encoding a value too large for the
    specified field size, the maximum value (all ones of that length) is
    supplied.  This matches the behavior of counters which "saturate" at
    the max value.

    BV is a fixed length byte string. Description is field name and
    length.
    
    RES is a reserved field.  Description is the field length.  Reserved
    fields are ignored on receipt and sent as bytes of zero.

    TLV: description is the size of the type field, size of the length
    field, wildcard flag, followed by a sequence of value codes.  Each
    value code consists of the value type code for that value, the value
    type, and value description as for any other Packet field.  If the
    wildcard flag is True, unrecognized type fields are accepted in
    decode, and turn into "fieldnnn" attributes containing the field
    value as a byte string.  If False, unrecognized type fields are an
    error.  

    PAYLOAD is the rest of the packet.  By convention it is written as
    the layout item "Payload" which is a shorthand for the tuple
    (PAYLOAD, "payload"), i.e., the rest of the packet deliverd to the 
    field named "payload".

    The code table is used by the "encode" and "decode" methods of the
    class being defined.  This generally means those methods as defined
    in the Packet base class.  The way a given field is encoded is
    defined by the encode and decode methods of the field class, which
    makes it easy to add new types or specialized encodings for standard
    types.
    """
    __slots__ = _allslots = ( "src", "decoded_from" )
    _codetable = ()
    _xfields = False
    
    # A subclass can override this to be True, in which case some
    # format errors are suppressed.  This is useful to accommodate
    # non-conforming packets seen in the wild.
    tolerant = False

    def __new__ (cls, buf = None, *args, **kwargs):
        """Create a Packet object.
        """
        if cls == __class__:
            # Instantiating this class (the base class), reject
            raise TypeError ("Can't instantiate object of "\
                             "class {}".format (cls.__name__))
        if buf:
            ret, buf = cls.decode (buf)
            if buf:
                logging.debug ("Unexpected data for {} after parse: {}",
                               cls.__name__, buf)
                raise ExtraData
            return ret
        return super (__class__, cls).__new__ (cls)
    
    def __init__ (self, buf = None, copy = None, **kwargs):
        """Initialize a Packet object.

        If "buf" is supplied, that buffer is decoded.  Otherwise, if
        "copy" is specified, its instance attributes are initialized
        from that object, to the extent that the copied-from object has
        the corresponding attributes.  In either case, if other keyword
        arguments are supplied, they initialize attributes of those
        names.
        """
        super ().__init__ ()
        if buf:
            pass    # handled in __new__ method
        elif copy:
            for attr in self._allslots:
                v = getattr (copy, attr, None)
                if v is not None:
                    setattr (self, attr, v)
        if kwargs:
            for k, v in kwargs.items ():
                setattr (self, k, v)

    def __setattr__ (self, field, val):
        """Set an attribute.  If the attribute being set is the name
        of a class attribute, and that attribute is not None, the value
        being set must match that class attribute's value.  This enforces
        fixed field values when decoding incoming packets.
        """
        try:
            super ().__setattr__ (field, val)
        except AttributeError as a:
            prev = getattr (self, field, None)
            if prev is not None:
                if prev != val:
                    raise WrongValue ("Cannot change attribute {} " \
                                      "from {} to {}" \
                                      .format (field, prev, val)) from None
            else:
                raise

    def encode (self):
        """Encode the packet according to the current attributes.  The
        resulting packet data is returned.
        """
        data = [ ]
        for ftype, fname, args in self._codetable:
            try:
                if fname:
                    # Simple field, get its value
                    val = getattr (self, fname, None)
                    # Check type and/or supply default
                    val = ftype.checktype (fname, val)
                    if val is not None:
                        data.append (val.encode (*args))
                else:
                    # Composite like TLV
                    data.append (ftype.encode (self, *args))
            except Exception:
                logging.exception ("Error encoding {} {}",
                                   ftype.__name__, fname)
                raise
        return b''.join (data)

    @classmethod
    def decode (cls, buf, *decodeargs):
        """Decode a packet buffer and return a pair of the resulting
        Packet instance and the remaining buffer.

        If more data is present than accounted for in the layout
        definition, the remainder is returned.  This is useful for
        variable layout packets; in that case the class layout is used
        to define the header layout, and anything beyond the header
        is processed separately.

        If any layout fields have values set in the packet class, those
        values are required values and mismatches will raise an
        Exception that is a subclass of DecodeError.
        """
        ret = cls ()
        buf = makebytes (buf)
        # Save the buffer in case we want to redo the decode for a
        # indexed subclass.
        buf2 = buf
        # Start filling in the object data as instructed by the code
        # table.
        ret.decoded_from = buf
        for ftype, fname, args in ret._codetable:
            if fname:
                val, buf = ftype.decode (buf, *args)
                try:
                    setattr (ret, fname, val)
                except ReadOnlyError:
                    logging.debug ("Field required value mismatch: {}", fname)
                    raise WrongValue from None
                except ValueError:
                    logging.debug ("Invalid field value: {}", fname)
                    raise WrongValue from None
            else:
                buf = ftype.decode (buf, ret, *args)
        while True:
            ret.check ()
            # See if this is an indexed class
            key = ret.classindexkey
            if not key:
                break
            if callable (key):
                idx = ret.instanceindexkey ()
            else:
                idx = getattr (ret, key, None)
            if idx is None:
                break
            cls2 = ret.findclass (idx)
            if cls2 and not isinstance (ret, cls2):
                # We want a different class; create an instance of
                # that one and redo the decode with it.
                ret, buf = cls2.decode (buf2, *decodeargs)
            else:
                break
        return ret, buf
    
    def __bytes__ (self):
        """Convert to bytes.  We encode the data each time, since this
        doesn't happen often enough to bother with the rather hairy
        process of caching the answer.
        """
        return self.encode ()

    def __len__ (self):
        """Return the packet length, i.e., the length of the encoded
        packet data.  Note that this builds the encoding, so this is not
        all that efficient and should be used sparingly.
        """
        return len (bytes (self))

    def __bool__ (self):
        return True
    
    def __iter__ (self):
        """Return an iterator over the packet contents.
        """
        return iter (bytes (self))

    def check (self):
        """Override this method to implement additional checks after
        individual field parse.  It should raise an exception if there
        is a problem, or return if all is well.
        """
        pass

    def format (self, exclude = { "decoded_from" }):
        # By default we omit the "decoded_from" field because that
        # rarely contains anything useful and can make the string
        # absurdly large.
        ret = list ()
        for a in self._allslots:
            if a in exclude:
                continue
            v = getattr (self, a, None)
            if v is not None:
                ret.append ("{}={}".format (a, v))
        return "{}({})".format (self.__class__.__name__, ", ".join (ret))

    def __str__ (self):
        return self.format ()
    
    __repr__ = __str__

    def __eq__ (self, other):
        return bytes (self) == bytes (other)

    def __ne__ (self, other):
        return bytes (self) != bytes (other)

    def xfields (self, sortlist = False):
        """Return a list of the "fieldnnn" attributes of this object,
        sorted in numerical order if requested.
        """
        if not self._xfields:
            return [ ]
        try:
            ret = [ n for n in self.__dict__ if n.startswith ("field") ]
        except AttributeError:
            return [ ]
        if sortlist and ret:
            ret.sort (key = fieldnum)
        return ret

    @staticmethod
    def fieldlabel (fn, desc = None):
        """Convert a field name to a user-friendly label string.
        """
        if desc:
            return desc
        if fn.startswith ("field"):
            return "Parameter #{}".format (fn[5:])
        fn = fn.replace ("_", " ")
        fn = fn[0].upper () + fn[1:]
        return fn