commit 61bf09d8c40c476aad8491c16de6ba5f5900cb55 Author: m0sk13 <85328702+m0sk13@users.noreply.github.com> Date: Fri Oct 17 20:02:29 2025 +0800 initial commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..b6c037e --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# Amazon Rekognition People Label Detection via KVS +This is the README for the whole people detection pipeline designed for real-time video analysis. + +There are three Lambda functions placed inside this repository: +### 1. Kinesis Video Stream to Rekognition (`cctv-people-rekogniton`) +This function utilizes Amazon Rekognition's stream processor, which is configured to monitor an incoming Amazon Kinesis Video Stream for people label detection. + +Note that this Lambda function is intended to process **a single Kinesis video stream only**. For a multi-camera setup, multiple deployments of this Lambda function is necessary. + +This is built on top of AWS KVS Consumer Library for Python. The original repository can be found here: https://github.com/aws-samples/amazon-kinesis-video-streams-consumer-library-for-python/ + +### 2. SNS to DynamoDB and OpenSearch (`cctv-people-sns-dynamodb`) +This function writes a new entry to DynamoDB and Opensearch tables whenever there is a detection coming from Rekognition via SNS. + +### 3. DynamoDB to Supabase (`cctv-people-dynamodb-supabase`) +This function acts an extension of the previous function, writing a new entry to Supabase whenever there is a new entry in DynamoDB. + +## Prerequites +The following must already be set up before deploying this pipeline: +1. A working Kinesis video stream as input +2. An S3 Bucket where all frames with detection go +3. A SNS topic that will receive the notification coming from Rekognition +4. A DynamoDB table +5. An OpenSearch Service collection + +It is advised to have all above components in the same region for easy configuration. + +## Setup +### Part I. Kinesis Video Stream to Rekognition (`\cctv-people-rekogniton`) + +1. Create an IAM role for the Rekognition stream processor. This role must contain the following AWS-managed policies: + - AmazonRekognitionServiceRole + - AmazonS3FullAccess + +2. Provide the following details in `lambda_function.py`: + - Kinesis video stream name and ARN + - Stream processor name (must be unique to the KVS stream) + - S3 bucket name + - IAM role ARN + - SNS topic ARN + +3. Package the folder as a ZIP file. +4. Create an AWS Lambda function and upload the ZIP file. +5. Add an EventBridge trigger with the schedule expression of `rate(5 minutes)` + +### Part II. SNS to DynamoDB and OpenSearch (`\cctv-people-sns-dynamodb`) + +1. Create a Lambda function and upload the `lambda_function.py` file. +2. Add a trigger to receive notifications from the SNS topic. +3. Add the following AWS-managed permissions to this Lambda function: + - AmazonDynamoDBFullAccess_v2 + - AmazonOpenSearchIngestionFullAccess + - AmazonOpenSearchServiceFullAccess + - AmazonS3FullAccess +4. Add an inline policy for OpenSearch Serverless: +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Statement1", + "Effect": "Allow", + "Action": "aoss:*", + "Resource": "*" + } + ] +} +``` +5. [insert Grafana instructions here] + +### 3. DynamoDB to Supabase (`\cctv-people-dynamodb-supabase`) +1. On a local machine, prepare a folder for Supabase installation. +2. Open a terminal pointed at this directory and run the following command: +``` +pip3 install supabase --platform manylinux2014_x86_64 --python-version 3.12 --only-binary=:all: -t . +``` +3. Place the `lambda_function.py` inside this folder. +4. Package the folder as a ZIP file. +5. Create an AWS Lambda function and upload the ZIP file. +6. Add the following AWS-managed permission to the IAM role for this Lambda: + - AmazonDynamoDBFullAccess_v2 +7. Enable stream on the DynamoDB table. Set view type to **New image**. +8. Add trigger to the Lambda function to connect to the DynamoDB stream. + diff --git a/lambda/cctv-people-dynamodb-supabase/lambda_function.py b/lambda/cctv-people-dynamodb-supabase/lambda_function.py new file mode 100644 index 0000000..638de63 --- /dev/null +++ b/lambda/cctv-people-dynamodb-supabase/lambda_function.py @@ -0,0 +1,26 @@ +import os +import json +from supabase import create_client, Client + +url: str = "https://ruthqdjqcvazlregemdb.supabase.co" +key: str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InJ1dGhxZGpxY3ZhemxyZWdlbWRiIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NTk0MTA2NTEsImV4cCI6MjA3NDk4NjY1MX0.peoFylhJQeMbPDze_6VTfbJUGqvaUCFsIH-9A0Aa3EM" +supabase: Client = create_client(url, key) + +def lambda_handler(event, context): + if event['Records'][0]['eventName'] == 'INSERT': + new_item = event['Records'][0]['dynamodb']['NewImage'] + response = ( + supabase.table("cctv-people") + .insert({ + 'camera_id': new_item['camera_id']['S'], + 'label_id': new_item['label_id']['S'], + 'detect_timestamp': new_item['detect_timestamp']['S'], + 'frame_timestamp': new_item['frame_timestamp']['S'], + 'frame_url': new_item['frame_url']['S'], + 'label_name': new_item['label_name']['S'], + 'confidence': new_item['confidence']['N'], + 'rekognition_response': json.loads(new_item['rekognition_response']['S']) + }) + .execute() + ) + print(response) \ No newline at end of file diff --git a/lambda/cctv-people-rekognition/ebmlite/__init__.py b/lambda/cctv-people-rekognition/ebmlite/__init__.py new file mode 100644 index 0000000..63bf32a --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/__init__.py @@ -0,0 +1,4 @@ +from .core import * +from .core import SCHEMA_PATH, SCHEMATA, __all__ + +name = "ebmlite" diff --git a/lambda/cctv-people-rekognition/ebmlite/core.py b/lambda/cctv-people-rekognition/ebmlite/core.py new file mode 100644 index 0000000..edd0722 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/core.py @@ -0,0 +1,1598 @@ +"""''' +EBMLite: A lightweight EBML parsing library. It is designed to crawl through +EBML files quickly and efficiently, and that's about it. + +@todo: Complete EBML encoding. Specifically, make 'master' elements write + directly to the stream, rather than build bytearrays, so huge 'master' + elements can be handled. It appears that the official spec may prohibit + (or at least counter-indicate) multiple root elements. Possible + compromise until proper fix: handle root 'master' elements differently + than deeper ones, more like the current `Document`. +@todo: Validation. Enforce the hierarchy defined in each schema. +@todo: Optimize 'infinite' master elements (i.e `size` is `None`). See notes + in `MasterElement` class' method definitions. +@todo: Improved `MasterElement.__eq__()` method, possibly doing a recursive + crawl of both elements and comparing the actual contents, or iterating + over chunks of the raw binary data. Current implementation doesn't check + element contents, just ID and payload size (for speed). +@todo: Document-wide caching, for future handling of streamed data. Affects + the longer-term streaming to-do (listed below) and optimization of + 'infinite' elements (listed above). +@todo: Clean up and standardize usage of the term 'size' versus 'length.' +@todo: General documentation (more detailed than the README) and examples. +@todo: Document the best way to load schemata in a PyInstaller executable. + +@todo: (longer term) Consider making schema loading automatic based on the EBML + DocType, DocTypeVersion, and DocTypeReadVersion. Would mean a refactoring + of how schemata are loaded. +@todo: (longer term) Refactor to support streaming data. This will require + modifying the indexing and iterating methods of `Document`. Also affects + the document-wide caching to-do item, listed above. +@todo: (longer term) Support the official Schema definition format. Start by + adopting some of the attributes, specifically ``minOccurs`` and + ``maxOccurs`` (they serve the function provided by the current + ``mandatory`` and ``multiple`` attributes). Add ``range`` later. + Eventually, recognize official schemata when loading, like the system + currently handles legacy ``python-ebml`` schemata. +""" +__author__ = "David Randall Stokes, Connor Flanigan" +__copyright__ = "Copyright 2022, Mide Technology Corporation" +__credits__ = "David Randall Stokes, Connor Flanigan, Becker Awqatty, Derek Witt" + +__all__ = ['BinaryElement', 'DateElement', 'Document', 'Element', + 'FloatElement', 'IntegerElement', 'MasterElement', 'Schema', + 'StringElement', 'UIntegerElement', 'UnicodeElement', + 'UnknownElement', 'VoidElement', 'loadSchema', 'parseSchema'] + +from ast import literal_eval +from datetime import datetime +import errno +import importlib +from io import BytesIO, StringIO, IOBase +import os.path +from pathlib import Path +import re +import sys +import types +from xml.etree import ElementTree as ET + +from .decoding import readElementID, readElementSize +from .decoding import readFloat, readInt, readUInt, readDate +from .decoding import readString, readUnicode +from . import encoding +from . import schemata + +# Dictionaries in Python 3.7+ are explicitly insert-ordered in all +# implementations. If older, continue to use `collections.OrderedDict`. +if sys.hexversion < 0x03070000: + from collections import OrderedDict as Dict +else: + Dict = dict + +# Additionally, `importlib.resources.files` is new to 3.9 as well; this is +# part of a work-around. +if sys.hexversion < 0x03090000: + importlib_resources = None +else: + import importlib.resources as importlib_resources + +# ============================================================================== +# +# ============================================================================== + +# SCHEMA_PATH: A list of paths for schema XML files, similar to `sys.path`. +# When `loadSchema()` is used, it will search these paths, in order, to find +# the schema file. +SCHEMA_PATH = ['', + os.path.realpath(os.path.dirname(schemata.__file__))] + +SCHEMA_PATH.extend(p for p in os.environ.get('EBMLITE_SCHEMA_PATH', '').split(os.path.pathsep) + if p not in SCHEMA_PATH) + +# SCHEMATA: A dictionary of loaded schemata, keyed by filename. Used by +# `loadSchema()`. In most cases, SCHEMATA should not be otherwise modified. +SCHEMATA = {} + + +# ============================================================================== +# +# ============================================================================== + +class Element(object): + """ Base class for all EBML elements. Each data type has its own subclass, + and these subclasses get subclassed when a Schema is read. + + @cvar id: The element's EBML ID. + @cvar name: The element's name. + @cvar schema: The `Schema` to which this element belongs. + @cvar multiple: Can this element be appear multiple times? Note: + Currently only enforced for encoding. + @cvar mandatory: Must this element appear in all EBML files using + this element's schema? Note: Not currently enforced. + @cvar children: A list of valid child element types. Only applicable to + `Document` and `Master` subclasses. Note: Not currently enforced. + @cvar dtype: The element's native Python data type. + @cvar precache: If `True`, the Element's value is read when the Element + is parsed. if `False`, the value is lazy-loaded when needed. + Numeric element types default to `True`. Can be used to reduce + the number of file seeks, potentially speeding things up. + @cvar length: An explicit length (in bytes) of the element when + encoding. `None` will use standard EBML variable-length encoding. + """ + __slots__ = ("stream", "offset", "size", "sizeLength", "payloadOffset", "_value") + + # Parent `Schema` + schema = None + + # Python native data type. + dtype = bytearray + + # Should this element's value be read/cached when the element is parsed? + precache = False + + # Do valid EBML documents require this element? + mandatory = False + + # Does a valid EBML document permit more than one of the element? + multiple = False + + # Explicit length for this Element subclass, used for encoding. + length = None + + # For python-ebml compatibility; not currently used. + children = None + + def parse(self, stream, size): + """ Type-specific helper function for parsing the element's payload. + It is assumed the file pointer is at the start of the payload. + """ + # Document-wide caching could be implemented here. + return bytearray(stream.read(size)) + + def __init__(self, stream=None, offset=0, size=0, payloadOffset=0): + """ Constructor. Instantiate a new Element from a file. In most cases, + elements should be created when a `Document` is loaded, rather + than instantiated explicitly. + + @keyword stream: A file-like object containing EBML data. + @keyword offset: The element's starting location in the file. + @keyword size: The size of the whole element. + @keyword payloadOffset: The starting location of the element's + payload (i.e. immediately after the element's header). + """ + self.stream = stream + self.offset = offset + self.size = size + self.payloadOffset = payloadOffset + self._value = None + + def __repr__(self): + return "<%s (ID:0x%02X), offset %s, size %s>" % \ + (self.__class__.__name__, self.id, self.offset, self.size) + + def __eq__(self, other): + """ Equality check. Elements are considered equal if they are the same + type and have the same ID, size, offset, and schema. Note: element + value is not considered! Check for value equality explicitly + (e.g. ``el1.value == el2.value``). + """ + if other is self: + return True + try: + return (self.dtype == other.dtype + and self.id == other.id + and self.offset == other.offset + and self.size == other.size + and self.schema == other.schema) + except AttributeError: + return False + + @property + def value(self): + """ Parse and cache the element's value. """ + if self._value is not None: + return self._value + self.stream.seek(self.payloadOffset) + self._value = self.parse(self.stream, self.size) + return self._value + + def getRaw(self): + """ Get the element's raw binary data, including EBML headers. + """ + self.stream.seek(self.offset) + return self.stream.read(self.size + (self.payloadOffset - self.offset)) + + def getRawValue(self): + """ Get the raw binary of the element's value. + """ + self.stream.seek(self.payloadOffset) + return self.stream.read(self.size) + + # ========================================================================== + # Caching (experimental) + # ========================================================================== + + def gc(self, recurse=False): + """ Clear any cached values. To save memory and/or force values to be + re-read from the file. Returns the number of cached values cleared. + """ + if self._value is None: + return 0 + + self._value = None + return 1 + + # ========================================================================== + # Encoding + # ========================================================================== + + @classmethod + def encodePayload(cls, data, length=None): + """ Type-specific payload encoder. """ + return encoding.encodeBinary(data, length) + + @classmethod + def encode(cls, value, length=None, lengthSize=None, infinite=False): + """ Encode an EBML element. + + @param value: The value to encode, or a list of values to encode. + If a list is provided, each item will be encoded as its own + element. + @keyword length: An explicit length for the encoded data, + overriding the variable length encoding. For producing + byte-aligned structures. + @keyword lengthSize: An explicit length for the encoded element + size, overriding the variable length encoding. + @return: A bytearray containing the encoded EBML data. + """ + if infinite and not issubclass(cls, MasterElement): + raise ValueError("Only Master elements can have 'infinite' lengths") + length = cls.length if length is None else length + if isinstance(value, (list, tuple)): + if not cls.multiple: + raise ValueError("Multiple %s elements per parent not permitted" + % cls.name) + result = bytearray() + for v in value: + result.extend(cls.encode(v, length, lengthSize, infinite)) + return result + payload = cls.encodePayload(value, length=length) + length = None if infinite else (length or len(payload)) + encId = encoding.encodeId(cls.id) + return encId + encoding.encodeSize(length, lengthSize) + payload + + def dump(self): + """ Dump this element's value as nested dictionaries, keyed by + element name. For non-master elements, this just returns the + element's value; this method exists to maintain uniformity. + """ + return self.value + + +# ============================================================================== + + +class IntegerElement(Element): + """ Base class for an EBML signed integer element. Schema-specific + subclasses are generated when a `Schema` is loaded. + """ + __slots__ = ("stream", "offset", "size", "sizeLength", "payloadOffset", "_value") + dtype = int + precache = True + + def __eq__(self, other): + if not super(IntegerElement, self).__eq__(other): + return False + return self.value == other.value + + def parse(self, stream, size): + """ Type-specific helper function for parsing the element's payload. + It is assumed the file pointer is at the start of the payload. + """ + return readInt(stream, size) + + @classmethod + def encodePayload(cls, data, length=None): + """ Type-specific payload encoder for signed integer elements. """ + return encoding.encodeInt(data, length) + + +# ============================================================================== + + +class UIntegerElement(IntegerElement): + """ Base class for an EBML unsigned integer element. Schema-specific + subclasses are generated when a `Schema` is loaded. + """ + __slots__ = ("stream", "offset", "size", "sizeLength", "payloadOffset", "_value") + dtype = int + precache = True + + def parse(self, stream, size): + """ Type-specific helper function for parsing the element's payload. + It is assumed the file pointer is at the start of the payload. + """ + return readUInt(stream, size) + + @classmethod + def encodePayload(cls, data, length=None): + """ Type-specific payload encoder for unsigned integer elements. """ + return encoding.encodeUInt(data, length) + + +# ============================================================================== + + +class FloatElement(Element): + """ Base class for an EBML floating point element. Schema-specific + subclasses are generated when a `Schema` is loaded. + """ + __slots__ = ("stream", "offset", "size", "sizeLength", "payloadOffset", "_value") + dtype = float + precache = True + + def __eq__(self, other): + if not super(FloatElement, self).__eq__(other): + return False + return self.value == other.value + + def parse(self, stream, size): + """ Type-specific helper function for parsing the element's payload. + It is assumed the file pointer is at the start of the payload. + """ + return readFloat(stream, size) + + @classmethod + def encodePayload(cls, data, length=None): + """ Type-specific payload encoder for floating point elements. """ + return encoding.encodeFloat(data, length) + + +# ============================================================================== + + +class StringElement(Element): + """ Base class for an EBML ASCII string element. Schema-specific + subclasses are generated when a `Schema` is loaded. + """ + __slots__ = ("stream", "offset", "size", "sizeLength", "payloadOffset", "_value") + dtype = str + + def __eq__(self, other): + if not super(StringElement, self).__eq__(other): + return False + return self.value == other.value + + def __len__(self): + return self.size + + def parse(self, stream, size): + """ Type-specific helper function for parsing the element's payload. + It is assumed the file pointer is at the start of the payload. + """ + return readString(stream, size) + + @classmethod + def encodePayload(cls, data, length=None): + """ Type-specific payload encoder for ASCII string elements. """ + return encoding.encodeString(data, length) + + +# ============================================================================== + + +class UnicodeElement(StringElement): + """ Base class for an EBML UTF-8 string element. Schema-specific subclasses + are generated when a `Schema` is loaded. + """ + __slots__ = ("stream", "offset", "size", "sizeLength", "payloadOffset", "_value") + dtype = str + + def __len__(self): + # Value may be multiple bytes per character + return len(self.value) + + def parse(self, stream, size): + """ Type-specific helper function for parsing the element's payload. + It is assumed the file pointer is at the start of the payload. + """ + return readUnicode(stream, size) + + @classmethod + def encodePayload(cls, data, length=None): + """ Type-specific payload encoder for Unicode string elements. """ + return encoding.encodeUnicode(data, length) + + +# ============================================================================== + + +class DateElement(IntegerElement): + """ Base class for an EBML 'date' element. Schema-specific subclasses are + generated when a `Schema` is loaded. + """ + __slots__ = ("stream", "offset", "size", "sizeLength", "payloadOffset", "_value") + dtype = datetime + + def parse(self, stream, size): + """ Type-specific helper function for parsing the element's payload. + It is assumed the file pointer is at the start of the payload. + """ + return readDate(stream, size) + + @classmethod + def encodePayload(cls, data, length=None): + """ Type-specific payload encoder for date elements. """ + return encoding.encodeDate(data, length) + + +# ============================================================================== + + +class BinaryElement(Element): + """ Base class for an EBML 'binary' element. Schema-specific subclasses + are generated when a `Schema` is loaded. + """ + + __slots__ = ("stream", "offset", "size", "sizeLength", "payloadOffset", "_value") + + def __len__(self): + return self.size + + +# ============================================================================== + + +class VoidElement(BinaryElement): + """ Special case ``Void`` element. Its contents are ignored and not read; + its `value` is always returned as ``0xFF`` times its length. To get + the actual contents, use `getRawValue()`. + """ + __slots__ = ("stream", "offset", "size", "sizeLength", "payloadOffset", "_value") + + def parse(self, stream, size): + return bytearray() + + @classmethod + def encodePayload(cls, data, length=0): + """ Type-specific payload encoder for Void elements. """ + length = 0 if length is None else length + return bytearray(b'\xff' * length) + + +# ============================================================================== + + +class UnknownElement(BinaryElement): + """ Special case ``Unknown`` element, used for elements with IDs not + present in a schema. Unlike other elements, each instance has its own + ID. + """ + __slots__ = ("stream", "offset", "size", "sizeLength", "payloadOffset", "_value", "id", + "schema") + name = "UnknownElement" + precache = False + + def __init__(self, stream=None, offset=0, size=0, payloadOffset=0, eid=None, + schema=None): + """ Constructor. Instantiate a new `UnknownElement` from a file. In + most cases, elements should be created when a `Document` is loaded, + rather than instantiated explicitly. + + @keyword stream: A file-like object containing EBML data. + @keyword offset: The element's starting location in the file. + @keyword size: The size of the whole element. + @keyword payloadOffset: The starting location of the element's + payload (i.e. immediately after the element's header). + @keyword id: The unknown element's ID. Unlike 'normal' elements, + in which ID is a class attribute, each UnknownElement instance + explicitly defines this. + @keyword schema: The schema used to load the element. Specified + explicitly because `UnknownElement`s are not part of any + schema. + """ + super(UnknownElement, self).__init__(stream, offset, size, + payloadOffset) + self.id = eid + self.schema = schema + + def __eq__(self, other): + """ Equality check. Unknown elements are considered equal if they have + the same ID and value. Note that this differs from the criteria + used for other element classes! + """ + if other is self: + return True + try: + return (self.name == other.name + and self.id == other.id + and self.value == other.value) + except AttributeError: + return False + + +# ============================================================================== + + +class MasterElement(Element): + """ Base class for an EBML 'master' element, a container for other + elements. + """ + __slots__ = ("stream", "offset", "sizeLength", "payloadOffset", "_value", + "_size", "_length") + dtype = list + + def parse(self): + """ Type-specific helper function for parsing the element's payload. + """ + # Special case; unlike other elements, value() property doesn't call + # parse(). Used only when pre-caching. + return self.value + + def parseElement(self, stream, nocache=False): + """ Read the next element from a stream, instantiate a `MasterElement` + object, and then return it and the offset of the next element + (this element's position + size). + + @param stream: The source file-like stream. + @keyword nocache: If `True`, the parsed element's `precache` + attribute is ignored, and the element's value will not be + cached. For faster iteration when the element value doesn't + matter (e.g. counting child elements). + @return: The parsed element and the offset of the next element + (i.e. the end of the parsed element). + """ + offset = stream.tell() + eid, idlen = readElementID(stream) + esize, sizelen = readElementSize(stream) + payloadOffset = offset + idlen + sizelen + + try: + etype = self.schema.elements[eid] + el = etype(stream, offset, esize, payloadOffset) + except KeyError: + el = self.schema.UNKNOWN(stream, offset, esize, payloadOffset, + eid=eid, schema=self.schema) + + if el.precache and not nocache: + # Read the value now, avoiding a seek later. + el._value = el.parse(stream, el.size) + + return el, payloadOffset + el.size + + @classmethod + def _isValidChild(cls, elId): + """ Is the given element ID represent a valid sub-element, i.e. + explicitly specified as a child element or a 'global' in the + schema? + """ + if not cls.children: + return False + + try: + return elId in cls._childIds + except AttributeError: + # The set of valid child IDs hasn't been created yet. + cls._childIds = set(cls.children) + if cls.schema is not None: + cls._childIds.update(cls.schema.globals) + return elId in cls._childIds + + @property + def size(self): + """ The element's size. Master elements can be instantiated with this + as `None`; this denotes an 'infinite' EBML element, and its size + will be determined by iterating over its contents until an invalid + child type is found, or the end-of-file is reached. + """ + try: + return self._size + except AttributeError: + # An "infinite" element (size specified in file is all 0xFF) + pos = end = self.payloadOffset + numChildren = 0 + while True: + self.stream.seek(pos) + end = pos + try: + # TODO: Cache parsed elements? + el, pos = self.parseElement(self.stream, nocache=True) + if self._isValidChild(el.id): + numChildren += 1 + else: + break + except TypeError as err: + # Will occur at end of file; message will contain "ord()". + if "ord()" in str(err): + break + # Not the expected EOF TypeError! + raise + + self._size = end - self.payloadOffset + self._length = numChildren + return self._size + + @size.setter + def size(self, esize): + if esize is not None: + # Only create the `_size` attribute for a real value. Don't + # define it if it's `None`, so `size` will get calculated. + self._size = esize + + def __iter__(self, nocache=False): + """ x.__iter__() <==> iter(x) + """ + # TODO: Better support for 'infinite' elements (getting the size of + # an infinite element iterates over it, so there's duplicated effort.) + pos = self.payloadOffset + payloadEnd = pos + self.size + + while pos < payloadEnd: + self.stream.seek(pos) + try: + el, pos = self.parseElement(self.stream, nocache=nocache) + yield el + except TypeError as err: + if "ord()" in str(err): + break + raise + + def __len__(self): + """ x.__len__() <==> len(x) + """ + try: + return self._length + except AttributeError: + if self._value is not None: + self._length = len(self._value) + else: + n = 0 # In case there's nothing to enumerate + for n, _el in enumerate(self.__iter__(nocache=True), 1): + pass + self._length = n + return self._length + + @property + def value(self): + """ Parse and cache the element's value. + """ + if self._value is not None: + return self._value + self._value = list(self) + return self._value + + def __getitem__(self, *args): + # TODO: Parse only the requested item(s), like `Document` + return self.value.__getitem__(*args) + + # ========================================================================== + # Caching (experimental!) + # ========================================================================== + + def gc(self, recurse=False): + """ Clear any cached values. To save memory and/or force values to be + re-read from the file. + """ + cleared = 0 + if self._value is not None: + if recurse: + cleared = sum(ch.gc(recurse) for ch in self._value) + 1 + self._value = None + return cleared + + # ========================================================================== + # Encoding + # ========================================================================== + + @classmethod + def encodePayload(cls, data, length=None): + """ Type-specific payload encoder for 'master' elements. + """ + result = bytearray() + if data is None: + return result + elif isinstance(data, dict): + data = data.items() + elif not isinstance(data, (list, tuple)): + raise TypeError("wrong type for %s payload: %s" % (cls.name, + type(data))) + for k, v in data: + if k not in cls.schema: + raise TypeError("Element type %r not found in schema" % k) + # TODO: Validation of hierarchy, multiplicity, mandate, etc. + result.extend(cls.schema[k].encode(v)) + + return result + + @classmethod + def encode(cls, data, length=None, lengthSize=None, infinite=False): + """ Encode an EBML master element. + + @param data: The data to encode, provided as a dictionary keyed by + element name, a list of two-item name/value tuples, or a list + of either. Note: individual items in a list of name/value + pairs *must* be tuples! + @keyword infinite: If `True`, the element will be written with an + undefined size. When parsed, its end will be determined by the + occurrence of an invalid child element (or end-of-file). + @return: A bytearray containing the encoded EBML binary. + """ + # TODO: Use 'length' to automatically generate `Void` element? + if isinstance(data, list) and len(data) > 0 and isinstance(data[0], list): + # List of lists: special case for 'master' elements. + # Encode as multiple 'master' elements. + result = bytearray() + for v in data: + result.extend(cls.encode(v, length=length, + lengthSize=lengthSize, + infinite=infinite)) + return result + + # TODO: Remove 'infinite' kwarg from `Element.encode()` and handle it + # here, since it only applied to Master elements. + return super(MasterElement, cls).encode(data, length=length, + lengthSize=lengthSize, + infinite=infinite) + + def dump(self): + """ Dump this element's value as nested dictionaries, keyed by + element name. The values of 'multiple' elements return as lists. + Note: The order of 'multiple' elements relative to other elements + will be lost; a file containing elements ``A1 B1 A2 B2 A3 B3`` will + result in``[A1 A2 A3][B1 B2 B3]``. + + @todo: Decide if this should be in the `util` submodule. It is + very specific, and it isn't totally necessary for the core + library. + """ + result = Dict() + for el in self: + if el.multiple: + result.setdefault(el.name, []).append(el.dump()) + else: + result[el.name] = el.dump() + return result + + +# ============================================================================== +# +# ============================================================================== + + +class Document(MasterElement): + """ Base class for an EBML document, containing multiple 'root' elements. + Loading a `Schema` generates a subclass. + """ + + def __init__(self, stream, name=None, size=None, headers=True): + """ Constructor. Instantiate a `Document` from a file-like stream. + In most cases, `Schema.load()` should be used instead of + explicitly instantiating a `Document`. + + @param stream: A stream object (e.g. a file) from which to read + the EBML content. + @keyword name: The name of the document. Defaults to the filename + (if applicable). + @keyword size: The size of the document, in bytes. Use if the + stream is neither a file or a `BytesIO` object. + @keyword headers: If `False`, the file's ``EBML`` header element + (if present) will not appear as a root element in the document. + The contents of the ``EBML`` element will always be read, + regardless, and stored in the Document's `info` attribute. + """ + self._ownsStream = False + if isinstance(stream, (str, bytes, bytearray)): + stream = open(stream, 'rb') + self._ownsStream = True + + if not all((hasattr(stream, 'read'), + hasattr(stream, 'tell'), + hasattr(stream, 'seek'))): + raise TypeError('Object %r does not have the necessary stream methods' % stream) + + self._value = None + self.stream = stream + self.size = size + self.name = name + self.id = None # Not applicable to Documents. + self.offset = self.payloadOffset = self.stream.tell() + + try: + self.filename = stream.name + except AttributeError: + self.filename = "" + + if name is None: + if self.filename: + self.name = os.path.splitext(os.path.basename(self.filename))[0] + else: + self.name = self.__class__.__name__ + + if size is None: + # Note: this doesn't work for cStringIO! + if isinstance(stream, BytesIO): + self.size = len(stream.getvalue()) + elif self.filename and os.path.exists(self.filename): + self.size = os.path.getsize(self.stream.name) + + self.info = {} + + try: + # Attempt to read the first element, which should be an EBML header. + el, pos = self.parseElement(self.stream) + if el.name == "EBML": + # Load 'header' info from the file + self.info = el.dump() + if not headers: + self.payloadOffset = pos + except: + # Failed to read the first element. Don't raise here; do that when + # the Document is actually used. + pass + + def __repr__(self): + """ "x.__repr__() <==> repr(x) """ + if self.name == self.__class__.__name__: + return object.__repr__(self) + return "<%s %r at 0x%08X>" % (self.__class__.__name__, self.name, + id(self)) + + def __enter__(self): + """ Enter context manager for this document. + """ + return self + + def __exit__(self, exc_type, exc_value, traceback): + """ Close this document on exiting context manager. + """ + self.close() + + def close(self): + """ Closes the EBML file. If the `Document` was created using a + file/stream (as opposed to a filename), the source file/stream is + not closed. + """ + if self._ownsStream: + self.stream.close() + + def __len__(self): + """ x.__len__() <==> len(x) + Not recommended for huge documents. + """ + try: + return self._length + except AttributeError: + n = 0 # in case there's nothing to enumerate + for n, _el in enumerate(self.__iter__(nocache=True), 1): + pass + self._length = n + return self._length + + def __iter__(self, nocache=False): + """ Iterate root elements. + """ + # TODO: Cache root elements, prevent unnecessary duplicates. Maybe a + # dict keyed by offset? + pos = self.payloadOffset + while True: + self.stream.seek(pos) + try: + el, pos = self.parseElement(self.stream, nocache=nocache) + yield el + except TypeError as err: + # Occurs at end of file (parsing 0 length string), it's okay. + if "ord()" not in str(err): + # (Apparently) not the TypeError raised at EOF! + raise + break + + @property + def value(self): + """ An iterator for iterating the document's root elements. Same as + `Document.__iter__()`. + """ + # 'value' not really applicable to a document; return an iterator. + return iter(self) + + def __getitem__(self, idx): + """ Get one of the document's root elements by index. + """ + # TODO: Cache parsed root elements, handle indexing dynamically. + if isinstance(idx, int): + if idx < 0: + raise IndexError("Negative indices in a Document not (yet) supported") + n = None + for n, el in enumerate(self): + if n == idx: + return el + if n is None: + # If object being enumerated is empty, `n` is never set. + raise IndexError("Document contained no readable data") + raise IndexError("list index out of range (0-%d)" % n) + elif isinstance(idx, slice): + raise IndexError("Document root slicing not (yet) supported") + else: + raise TypeError("list indices must be integers, not %s" % type(idx)) + + @property + def version(self): + """ The document's type version (i.e. the EBML ``DocTypeVersion``). """ + return self.info.get('DocTypeVersion') + + @property + def type(self): + """ The document's type name (i.e. the EBML ``DocType``). """ + return self.info.get('DocType') + + # ========================================================================== + # Caching (experimental!) + # ========================================================================== + + def gc(self, recurse=False): + # TODO: Implement this if/when caching of root elements is implemented. + return 0 + + # ========================================================================== + # Encoding + # ========================================================================== + + @classmethod + def _createHeaders(cls): + """ Create the default EBML 'header' elements for a Document, using + the default values in the schema. + + @return: A dictionary containing a single key (``EBML``) with a + dictionary as its value. The child dictionary contains + element names and values. + """ + if 'EBML' not in cls.schema: + return {} + + headers = Dict() + for elName, elType in (('EBMLVersion', int), + ('EBMLReadVersion', int), + ('DocType', str), + ('DocTypeVersion', int), + ('DocTypeReadVersion', int)): + if elName in cls.schema: + v = cls.schema._getInfo(cls.schema[elName].id, elType) + if v is not None: + headers[elName] = v + + return Dict(EBML=headers) + + @classmethod + def encode(cls, stream, data, headers=False, **kwargs): + """ Encode an EBML document. + + @param value: The data to encode, provided as a dictionary keyed + by element name, or a list of two-item name/value tuples. + Note: individual items in a list of name/value pairs *must* + be tuples! + @return: A bytearray containing the encoded EBML binary. + """ + if headers is True: + stream.write(cls.encodePayload(cls._createHeaders())) + + if isinstance(data, list): + if len(data) > 0 and isinstance(data[0], list): + # List of lists: special case for Documents. + # Encode as multiple 'root' elements. + raise TypeError('Cannot encode multiple Documents') + else: + for v in data: + stream.write(cls.encodePayload(v)) + else: + stream.write(cls.encodePayload(data)) + + +# ============================================================================== +# +# ============================================================================== + + +class Schema(object): + """ An EBML schema, mapping element IDs to names and data types. Unlike + the document and element types, this is not a base class; all schemata + are actual instances of this class. + + @ivar document: The schema's Document subclass. + @ivar elements: A dictionary mapping element IDs to the schema's + corresponding `Element` subclasses. + @ivar elementsByName: A dictionary mapping element names to the + schema's corresponding `Element` subclasses. + @ivar elementInfo: A dictionary mapping IDs to the raw schema + attribute data. It may have additional items not present in the + created element class' attributes. + + @ivar UNKNOWN: A class/function that handles unknown element IDs. By + default, this is the `UnknownElement` class. Special-case handling + can be done by substituting a different class, or an + element-producing factory function. + + @ivar source: The source from which the Schema was loaded; either a + filename or a file-like stream. + @ivar filename: The absolute path of the source file, if the source + was a file or a filename. + """ + + BASE_CLASSES = { + 'BinaryElement': BinaryElement, + 'DateElement': DateElement, + 'FloatElement': FloatElement, + 'IntegerElement': IntegerElement, + 'MasterElement': MasterElement, + 'StringElement': StringElement, + 'UIntegerElement': UIntegerElement, + 'UnicodeElement': UnicodeElement, + } + + # Mapping of schema type names to the corresponding Element subclasses. + # For python-ebml schema compatibility. + ELEMENT_TYPES = { + 'integer': IntegerElement, + 'uinteger': UIntegerElement, + 'float': FloatElement, + 'string': StringElement, + 'utf-8': UnicodeElement, + 'date': DateElement, + 'binary': BinaryElement, + 'master': MasterElement, + } + + # The handler for unknown element IDs. By default, this is just the + # `UnknownElement` class. Special-case handling of unknown elements can + # be done by substituting a different class, or an element-producing + # factory function. + UNKNOWN = UnknownElement + + def __init__(self, source, name=None): + """ Constructor. Creates a new Schema from a schema description XML. + + @param source: The Schema's source, either a string with the full + path and name of the schema XML file, or a file-like stream. + @keyword name: The schema's name. Defaults to the document type + element's default value (if defined) or the base file name. + """ + self.source = source + self.filename = None + + if isinstance(source, (str, bytes, bytearray)): + self.filename = os.path.realpath(source) + elif hasattr(source, "name"): + self.filename = os.path.realpath(source.name) + + self.elements = {} # Element types, keyed by ID + self.elementsByName = {} # Element types, keyed by element name + self.elementInfo = {} # Raw element schema attributes, keyed by ID + + self.globals = {} # Elements valid for any parent, by ID + self.children = {} # Valid root elements, by ID + + # Parse, using the correct method for the schema format. + schema = ET.parse(source) + root = schema.getroot() + if root.tag == "table": + # Old python-ebml schema: root element is + self._parseLegacySchema(root) + elif root.tag == "Schema": + # new ebmlite schema: root element is + self._parseSchema(root, self) + else: + raise IOError("Could not parse schema; expected root element " + " or
, got <%s>" % root.tag) + + # Special case: `Void` is a standard EBML element, but not its own + # type (it's technically binary). Use the special `VoidElement` type. + if 'Void' in self.elementsByName: + el = self.elementsByName['Void'] + void = type('VoidElement', (VoidElement,), + {'id': el.id, 'name': 'Void', 'schema': self, + 'mandatory': el.mandatory, 'multiple': el.multiple}) + self.elements[el.id] = void + self.elementsByName['Void'] = void + + # Schema name. Defaults to the schema's default EBML 'DocType' + self.name = name or self.type + + # Create the schema's Document subclass. + self.document = type('%sDocument' % self.name.title(), (Document,), + {'schema': self, 'children': self.children}) + + def _parseLegacySchema(self, schema): + """ Parse a legacy python-ebml schema XML file. + """ + for el in schema.findall('element'): + attribs = el.attrib.copy() + + eid = int(attribs['id'], 16) if 'id' in attribs else None + ename = attribs['name'].strip() if 'name' in attribs else None + etype = attribs['type'].strip() if 'type' in attribs else None + + # Use text in the element as its docstring. Note: embedded HTML + # tags (as in the Matroska schema) will cause the text to be + # truncated. + docs = el.text.strip() if isinstance(el.text, (str, bytes, bytearray)) else None + + if etype is None: + raise ValueError('Element "%s" (ID 0x%02X) missing required ' + '"type" attribute' % (ename, eid)) + + if etype not in self.ELEMENT_TYPES: + raise ValueError("Unknown type for element %r (ID 0x%02x): %r" % + (ename, eid, etype)) + + self.addElement(eid, ename, self.ELEMENT_TYPES[etype], attribs, + docs=docs) + + def _parseSchema(self, el, parent=None): + """ Recursively crawl a schema XML definition file. + """ + if el.tag == "Schema": + for chEl in el: + self._parseSchema(chEl, self) + return + + if el.tag not in self.BASE_CLASSES: + if el.tag.endswith('Element'): + raise ValueError('Unknown element type: %s' % el.tag) + + # FUTURE: Add schema-describing metadata (author, origin, + # description, etc.) to XML as non-Element elements. Parse them + # out here. + return + + attribs = el.attrib.copy() + eid = int(attribs['id'], 16) if 'id' in attribs else None + ename = attribs['name'].strip() if 'name' in attribs else None + + # Use text in the element as its docstring. Note: embedded HTML tags + # (as in the Matroska schema) will cause the text to be truncated. + docs = el.text.strip() if isinstance(el.text, (str, bytes, bytearray)) else None + + baseClass = self.BASE_CLASSES[el.tag] + + cls = self.addElement(eid, ename, baseClass, attribs, parent, docs) + + if baseClass is MasterElement: + for chEl in el: + self._parseSchema(chEl, cls) + + def addElement(self, eid, ename, baseClass, attribs={}, parent=None, + docs=None): + """ Create a new `Element` subclass and add it to the schema. + + Duplicate elements are permitted (e.g. if one kind of element can + appear in different master elements), provided their attributes do + not conflict. The first appearance of an element definition in the + schema must contain the required ID, name, and type; successive + appearances only need the ID and/or name. + + @param eid: The element's EBML ID. + @param ename: The element's name. + @keyword multiple: If `True`, an EBML document can contain more + than one of this element. Not currently enforced. + @keyword mandatory: If `True`, a valid EBML document requires one + (or more) of this element. Not currently enforced. + @keyword length: A fixed length to use when writing the element. + `None` will use the minimum length required. + @keyword precache: If `True`, the element's value will be read + when the element is parsed, rather than when the value is + explicitly accessed. Can save time for small elements. + @keyword attribs: A dictionary of raw element attributes, as read + from the schema file. + @keyword parent: The new element's parent element class. + @keyword docs: The new element's docstring (e.g. the defining XML + element's text content). + """ + + def _getBool(d, k, default): + """ Helper function to get a dictionary value cast to bool. """ + try: + return str(d[k]).strip()[0] in 'Tt1' + except (KeyError, TypeError, IndexError, ValueError): + # TODO: Don't fail silently for some exceptions. + pass + return default + + def _getInt(d, k, default): + """ Helper function to get a dictionary value cast to int. """ + try: + return int(literal_eval(d[k].strip())) + except (KeyError, SyntaxError, TypeError, ValueError): + # TODO: Don't fail silently for some exceptions. + pass + return default + + if eid in self.elements or ename in self.elementsByName: + # Already appeared in schema. Duplicates are permitted for + # defining an element that can appear as a child to multiple + # Master elements, so long as they have the same attributes. + # Additional definitions only need to specify the element ID + # and/or element name. + oldEl = self[ename or eid] + ename = oldEl.name + eid = oldEl.id + + if not issubclass(self.elements[eid], baseClass): + raise TypeError('%s %r (ID 0x%02X) redefined as %s' % + (oldEl.__name__, ename, eid, baseClass.__name__)) + + newatts = self.elementInfo[eid].copy() + newatts.update(attribs) + if self.elementInfo[eid] == newatts: + eclass = self.elements[eid] + else: + raise TypeError('Element %r (ID 0x%02X) redefined with ' + 'different attributes' % (ename, eid)) + else: + # New element class. It requires both a name and an ID. + # Validate both the name and the ID. + if eid is None: + raise ValueError('Element definition missing required ' + '"id" attribute') + elif not isinstance(eid, int): + raise TypeError("Invalid type for element ID: " + + "{} ({})".format(eid, type(eid).__name__)) + + if ename is None: + raise ValueError('Element definition missing required ' + '"name" attribute') + elif not isinstance(ename, (str, bytes, bytearray)): + raise TypeError('Invalid type for element name: ' + + '{} ({})'.format(ename, type(ename).__name__)) + elif not (ename[0].isalpha() or ename[0] == "_"): + raise ValueError("Invalid element name: %r" % ename) + + mandatory = _getBool(attribs, 'mandatory', False) + multiple = _getBool(attribs, 'multiple', False) + precache = _getBool(attribs, 'precache', baseClass.precache) + length = _getInt(attribs, 'length', None) + isGlobal = _getInt(attribs, 'global', None) + + if isGlobal is None: + # Element 'level'. The old schema format used level to define + # the structure (the file itself was flat); the new format's + # schema structure defined the EBML structure. The exception + # are 'global' elements, which may appear anywhere. The old + # format defined these as having a level of -1. The new format + # uses a Boolean attribute, `global`, but fall back to + # reading `level` if `global` isn't defined. + isGlobal = _getInt(attribs, 'level', None) == -1 + + # Create a new Element subclass + eclass = type('%sElement' % ename, (baseClass,), + {'id': eid, 'name': ename, 'schema': self, + 'mandatory': mandatory, 'multiple': multiple, + 'precache': precache, 'length': length, + 'children': dict(), '__doc__': docs, + '__slots__': baseClass.__slots__}) + + self.elements[eid] = eclass + self.elementInfo[eid] = attribs + self.elementsByName[ename] = eclass + + if isGlobal: + self.globals[eid] = eclass + + parent = parent or self + if parent.children is None: + parent.children = {} + parent.children[eid] = eclass + + return eclass + + def __repr__(self): + try: + if isinstance(self.source, (BytesIO, StringIO)): + source = "string" + else: + source = "'%s'" % (self.filename or self.source) + return "<%s %r from %s>" % (self.__class__.__name__, self.name, + source) + except AttributeError: + return object.__repr__(self) + + def __eq__(self, other): + """ Equality check. Schemata are considered equal if the attributes of + their elements match. + """ + try: + return self is other or self.elementInfo == other.elementInfo + except AttributeError: + return False + + def __contains__(self, key): + """ Does the Schema contain a given element name or ID? """ + return (key in self.elementsByName) or (key in self.elements) + + def __getitem__(self, key): + """ Get an Element class from the schema, by name or by ID. """ + try: + return self.elements[key] + except KeyError: + return self.elementsByName[key] + + def get(self, key, default=None): + if key in self: + return self[key] + return default + + def load(self, fp, name=None, headers=False, **kwargs): + """ Load an EBML file using this Schema. + + @param fp: A file-like object containing the EBML to load, or the + name of an EBML file. + @keyword name: The name of the document. Defaults to filename. + @keyword headers: If `False`, the file's ``EBML`` header element + (if present) will not appear as a root element in the + document. The contents of the ``EBML`` element will always be + read. + """ + return self.document(fp, name=name, headers=headers, **kwargs) + + def loads(self, data, name=None): + """ Load EBML from a string using this Schema. + + @param data: A string or bytearray containing raw EBML data. + @keyword name: The name of the document. Defaults to the Schema's + document class name. + """ + # Below updated to add EBML headers to first fragement + #return self.load(BytesIO(data), name=name) + return self.load(BytesIO(data), name=name, headers=True) + + def __call__(self, fp, name=None): + """ Load an EBML file using this Schema. Same as `Schema.load()`. + + @todo: Decide if this is worth keeping. It exists for historical + reasons that may have been refactored out. + + @param fp: A file-like object containing the EBML to load, or the + name of an EBML file. + @keyword name: The name of the document. Defaults to filename. + """ + return self.load(fp, name=name) + + # ========================================================================== + # Schema info stuff. Uses python-ebml schema XML data. Refactor later. + # ========================================================================== + + def _getInfo(self, eid, dtype): + """ Helper method to get the 'default' value of an element. """ + try: + return dtype(self.elementInfo[eid]['default']) + except (KeyError, ValueError): + return None + + @property + def version(self): + """ Schema version, extracted from EBML ``DocTypeVersion`` default. """ + return self._getInfo(0x4287, int) # ID of EBML 'DocTypeVersion' + + @property + def type(self): + """ Schema type name, extracted from EBML ``DocType`` default. """ + return self._getInfo(0x4282, str) # ID of EBML 'DocType' + + # ========================================================================== + # Encoding + # ========================================================================== + + def encode(self, stream, data, headers=False): + """ Write an EBML document using this Schema to a file or file-like + stream. + + @param stream: The file (or ``.write()``-supporting file-like + object) to which to write the encoded EBML. + @param data: The data to encode, provided as a dictionary keyed by + element name, or a list of two-item name/value tuples. Note: + individual items in a list of name/value pairs *must* be tuples! + """ + self.document.encode(stream, data, headers=headers) + return stream + + def encodes(self, data, headers=False): + """ Create an EBML document using this Schema, returned as a string. + + @param data: The data to encode, provided as a dictionary keyed by + element name, or a list of two-item name/value tuples. Note: + individual items in a list of name/value pairs *must* be tuples! + @return: A string containing the encoded EBML binary. + """ + stream = BytesIO() + self.encode(stream, data, headers=headers) + return stream.getvalue() + + def verify(self, data): + """ Perform basic tests on EBML binary data, ensuring it can be parsed + using this `Schema`. Failure will raise an expression. + """ + + def _crawl(el): + if isinstance(el, MasterElement): + for subel in el: + _crawl(subel) + elif isinstance(el, UnknownElement): + raise NameError("Verification failed, unknown element ID %x" % + el.id) + else: + _ = el.value + + return True + + return _crawl(self.loads(data)) + + +# ============================================================================== +# +# ============================================================================== + +def _expandSchemaPath(path, name=''): + """ Helper function to process a schema path or name, converting module + references to Paths. + + @param path: The schema path. May be a directory name, a module + name in braces (e.g., `{idelib.schemata}`), or a module + instance. Directory and module names may contain schema + filenames. + @param name: An optional schema base filename. Will get appended + to the resulting `Path`/`Traversable`. + @return: A `Path`/`Traversable` object. + """ + strpath = str(path) + subdir = '' + + if not strpath: + path = strpath = os.getcwd() + elif '{' in strpath: + if '}' not in strpath: + raise IOError(errno.ENOENT, 'Malformed module path', strpath) + + m = re.match(r'(\{.+\})[/\\](.+)', strpath) + if m: + path, subdir = m.groups() + strpath = path + + if importlib_resources: + if isinstance(path, types.ModuleType): + return importlib_resources.files(path) / subdir / name + elif '{' in strpath: + return importlib_resources.files(strpath.strip('{} ')) / subdir / name + else: + # Pre-3.9: Use naive means of finding the module path. Won't work in + # some cases (module is a zip, etc.); it's just a fallback. To be + # deprecated. + if isinstance(path, types.ModuleType): + path = os.path.dirname(path.__file__) + elif '{' in strpath: + path = os.path.dirname(importlib.import_module(strpath.strip('{}')).__file__) + + return Path(path) / subdir / name + + +def listSchemata(*paths, absolute=True): + """ Gather all EBML schemata. `ebmlite.SCHEMA_PATH` is used by default; + alternatively, one or more paths or modules can be supplied as + arguments. + + @returns: A dictionary of schema files. Keys are the base name of the + schema XML, values are lists of full paths to the XML. The first + filename in the list is what will load if the base name is used + with `loadSchema()`. + """ + schemata = {} + paths = paths or SCHEMA_PATH + + for path in paths: + try: + fullpath = _expandSchemaPath(path) + except ModuleNotFoundError: + continue + + if not fullpath.is_dir(): + continue + + for p in fullpath.iterdir(): + key = p.name + if key.lower().endswith('.xml'): + try: + # Casting to string is py35 fix. Remove in future. + xml = ET.parse(str(p)) + if xml.getroot().tag == 'Schema': + value = p if absolute else Path(path) / p.name + schemata.setdefault(key, []).append(value) + except (ET.ParseError, IOError, TypeError): + continue + + return schemata + + +def loadSchema(filename, reload=False, paths=None, **kwargs): + """ Import a Schema XML file. Loading the same file more than once will + return the initial instantiation, unless `reload` is `True`. + + @param filename: The name of the Schema XML file. If the file cannot + be found and file's path is not absolute, the paths listed in + `SCHEMA_PATH` will be searched (similar to `sys.path` when + importing modules). + @param reload: If `True`, the resulting Schema is guaranteed to be + new. Note: existing references to previous instances of the + Schema and/or its elements will not update. + @param paths: A list of paths to search for schemata, an alternative + to `ebmlite.SCHEMA_PATH` + + Additional keyword arguments are sent verbatim to the `Schema` + constructor. + + @raises: IOError, ModuleNotFoundError + """ + global SCHEMATA + + paths = paths or SCHEMA_PATH + origName = str(filename) + filename = Path(filename) + + if origName in SCHEMATA and not reload: + return SCHEMATA[origName] + + filename = _expandSchemaPath(filename) # raises ModuleNotFoundError + + if not filename.is_file(): + if len(filename.parts) == 1: + # Not a specific path and file not found: search paths in SCHEMA_PATH + for p in paths: + try: + f = _expandSchemaPath(p, filename) + if f.is_file(): + filename = f + break + except ModuleNotFoundError: + continue + + if hasattr(filename, 'expanduser'): + filename = filename.expanduser().absolute() + + if str(filename) in SCHEMATA and not reload: + return SCHEMATA[str(filename)] + + if not filename.is_file(): + raise IOError(errno.ENOENT, 'Could not find schema XML', origName) + + with filename.open() as fs: + schema = Schema(fs, **kwargs) + + SCHEMATA[str(filename)] = SCHEMATA[origName] = schema + return schema + + +def parseSchema(src, name=None, reload=False, **kwargs): + """ Read Schema XML data from a string or stream. Loading one with the + same `name` will return the initial instantiation, unless `reload` + is `True`. Calls to `loadSchema()` using a name previously used with + `parseSchema()` will also return the previously instantiated Schema. + + @param src: The XML string, or a stream containing XML. + @param name: The name of the schema. If none is supplied, + the name defined within the schema will be used. + @param reload: If `True`, the resulting Schema is guaranteed to be + new. Note: existing references to previous instances of the + Schema and/or its elements will not update. + + Additional keyword arguments are sent verbatim to the `Schema` + constructor. + """ + global SCHEMATA + + if name in SCHEMATA and not reload: + return SCHEMATA[name] + + if isinstance(src, IOBase): + stream = src + else: + stream = StringIO(src) + + schema = Schema(stream, **kwargs) + name = name or schema.name + SCHEMATA[name] = schema + return schema diff --git a/lambda/cctv-people-rekognition/ebmlite/decoding.py b/lambda/cctv-people-rekognition/ebmlite/decoding.py new file mode 100644 index 0000000..2322997 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/decoding.py @@ -0,0 +1,233 @@ +""" +Functions for decoding EBML elements and their values. + +Note: this module does not decode `Document`, `BinaryElement`, or +`MasterElement` objects; these are handled entirely in `core.py`. `Document` +and `MasterElement` objects are special cases, and `BinaryElement` objects do +not require special decoding. +""" +__author__ = "David Randall Stokes, Connor Flanigan" +__copyright__ = "Copyright 2021, Mide Technology Corporation" +__credits__ = "David Randall Stokes, Connor Flanigan, Becker Awqatty, Derek Witt" + +__all__ = ['readElementID', 'readElementSize', 'readFloat', 'readInt', + 'readUInt', 'readDate', 'readString', 'readUnicode'] + +from datetime import datetime, timedelta +import struct +import warnings + +# ============================================================================== +# +# ============================================================================== + +# Pre-built structs for packing/unpacking various data types +_struct_uint32 = struct.Struct(">I") +_struct_uint64 = struct.Struct(">Q") +_struct_int64 = struct.Struct(">q") +_struct_float32 = struct.Struct(">f") +_struct_float64 = struct.Struct(">d") + +# Direct references to struct methods. Makes things a marginally faster. +_struct_uint32_unpack = _struct_uint32.unpack +_struct_uint64_unpack = _struct_uint64.unpack +_struct_int64_unpack = _struct_int64.unpack +_struct_uint64_unpack_from = _struct_uint64.unpack_from +_struct_int64_unpack_from = _struct_int64.unpack_from +_struct_float32_unpack = _struct_float32.unpack +_struct_float64_unpack = _struct_float64.unpack + + +# ============================================================================== +# --- Reading and Decoding +# ============================================================================== + +def decodeIntLength(byte): + """ Extract the encoded size from an initial byte. + + @return: The size, and the byte with the size removed (it is the first + byte of the value). + """ + # An inelegant implementation, but it's fast. + if byte >= 128: + return 1, byte & 0b1111111 + elif byte >= 64: + return 2, byte & 0b111111 + elif byte >= 32: + return 3, byte & 0b11111 + elif byte >= 16: + return 4, byte & 0b1111 + elif byte >= 8: + return 5, byte & 0b111 + elif byte >= 4: + return 6, byte & 0b11 + elif byte >= 2: + return 7, byte & 0b1 + + return 8, 0 + + +def decodeIDLength(byte): + """ Extract the encoded ID size from an initial byte. + + @return: The size and the original byte (it is part of the ID). + @raise IOError: raise if the length of an ID is invalid. + """ + if byte >= 128: + return 1, byte + elif byte >= 64: + return 2, byte + elif byte >= 32: + return 3, byte + elif byte >= 16: + return 4, byte + + length, _ = decodeIntLength(byte) + raise IOError('Invalid length for ID: %d' % length) + + +def readElementID(stream): + """ Read an element ID from a file (or file-like stream). + + @param stream: The source file-like object. + @return: The decoded element ID and its length in bytes. + @raise IOError: raised if the length of the ID of an element is greater than 4 bytes. + """ + ch = stream.read(1) + length, eid = decodeIDLength(ord(ch)) + + if length > 4: + raise IOError('Cannot decode element ID with length > 4.') + if length > 1: + eid = _struct_uint32_unpack((ch + stream.read(length-1) + ).rjust(4, b'\x00'))[0] + return eid, length + + +def readElementSize(stream): + """ Read an element size from a file (or file-like stream). + + @param stream: The source file-like object. + @return: The decoded size (or `None`) and the length of the + descriptor in bytes. + """ + ch = stream.read(1) + length, size = decodeIntLength(ord(ch)) + + if length > 1: + size = _struct_uint64_unpack((chr(size).encode('latin-1') + + stream.read(length - 1) + ).rjust(8, b'\x00'))[0] + + if size == (2**(7*length)) - 1: + # EBML 'unknown' size, all bytes 0xFF + size = None + + return size, length + + +def readUInt(stream, size): + """ Read an unsigned integer from a file (or file-like stream). + + @param stream: The source file-like object. + @param size: The number of bytes to read from the stream. + @return: The decoded value. + """ + + if size == 0: + return 0 + + data = stream.read(size) + return _struct_uint64_unpack_from(data.rjust(8, b'\x00'))[0] + + +def readInt(stream, size): + """ Read a signed integer from a file (or file-like stream). + + @param stream: The source file-like object. + @param size: The number of bytes to read from the stream. + @return: The decoded value. + """ + + if size == 0: + return 0 + + data = stream.read(size) + if data[0] & 0b10000000: + pad = b'\xff' + else: + pad = b'\x00' + return _struct_int64_unpack_from(data.rjust(8, pad))[0] + + +def readFloat(stream, size): + """ Read an floating point value from a file (or file-like stream). + + @param stream: The source file-like object. + @param size: The number of bytes to read from the stream. + @return: The decoded value. + @raise IOError: raised if the length of this floating point number is not + valid (0, 4, 8 bytes) + """ + if size == 4: + return _struct_float32_unpack(stream.read(size))[0] + elif size == 8: + return _struct_float64_unpack(stream.read(size))[0] + elif size == 0: + return 0.0 + + raise IOError("Cannot read floating point value of length %s; " + "only lengths of 0, 4, or 8 bytes supported." % size) + + +def readString(stream, size): + """ Read an ASCII string from a file (or file-like stream). + + @param stream: The source file-like object. + @param size: The number of bytes to read from the stream. + @return: The decoded value. + """ + if size == 0: + return u'' + + value = stream.read(size) + value = value.partition(b'\x00')[0] + + try: + return str(value, 'ascii') + except UnicodeDecodeError as ex: + warnings.warn(str(ex), UnicodeWarning) + return str(value, 'ascii', 'replace') + + +def readUnicode(stream, size): + """ Read an UTF-8 encoded string from a file (or file-like stream). + + @param stream: The source file-like object. + @param size: The number of bytes to read from the stream. + @return: The decoded value. + """ + + if size == 0: + return u'' + + data = stream.read(size) + data = data.partition(b'\x00')[0] + return str(data, 'utf_8') + + +def readDate(stream, size=8): + """ Read an EBML encoded date (nanoseconds since UTC 2001-01-01T00:00:00) + from a file (or file-like stream). + + @param stream: The source file-like object. + @param size: The number of bytes to read from the stream. + @return: The decoded value (as `datetime.datetime`). + @raise IOError: raised if the length of the date is not 8 bytes. + """ + if size != 8: + raise IOError("Cannot read date value of length %d, only 8." % size) + data = stream.read(size) + nanoseconds = _struct_int64_unpack(data)[0] + delta = timedelta(microseconds=(nanoseconds // 1000)) + return datetime(2001, 1, 1, tzinfo=None) + delta diff --git a/lambda/cctv-people-rekognition/ebmlite/encoding.py b/lambda/cctv-people-rekognition/ebmlite/encoding.py new file mode 100644 index 0000000..4aefea6 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/encoding.py @@ -0,0 +1,289 @@ +""" +Functions for encoding EBML elements and their values. + +Note: this module does not encode Document or MasterElement objects; they are +special cases, handled in `core.py`. +""" +__author__ = "David Randall Stokes, Connor Flanigan" +__copyright__ = "Copyright 2021, Mide Technology Corporation" +__credits__ = "David Randall Stokes, Connor Flanigan, Becker Awqatty, Derek Witt" + +__all__ = ['encodeBinary', 'encodeDate', 'encodeFloat', 'encodeId', 'encodeInt', + 'encodeSize', 'encodeString', 'encodeUInt', 'encodeUnicode'] + +import datetime +import sys +import warnings + +from .decoding import _struct_uint64, _struct_int64 +from .decoding import _struct_float32, _struct_float64 + +# ============================================================================== +# +# ============================================================================== + +# If no length is given, use the platform's size of a float. +DEFAULT_FLOAT_SIZE = 4 if sys.maxsize <= 2147483647 else 8 + +LENGTH_PREFIXES = [0, + 0x80, + 0x4000, + 0x200000, + 0x10000000, + 0x0800000000, + 0x040000000000, + 0x02000000000000, + 0x0100000000000000 + ] + +# Translation table for removing invalid EBML string characters (32 < x < 127) +STRING_CHARACTERS = (b"?"*32 + bytearray(range(32, 127))).ljust(256, b'?') + +# ============================================================================== +# +# ============================================================================== + + +def getLength(val): + """ Calculate the encoded length of a value. + @param val: A value to be encoded, generally either an ID or a size for + an EBML element + @return The minimum length, in bytes, that can be used to represent val + """ + # Brute force it. Ugly but faster than calculating it. + if val <= 126: + return 1 + elif val <= 16382: + return 2 + elif val <= 2097150: + return 3 + elif val <= 268435454: + return 4 + elif val <= 34359738366: + return 5 + elif val <= 4398046511102: + return 6 + elif val <= 562949953421310: + return 7 + else: + return 8 + + +def encodeSize(val, length=None): + """ Encode an element size. + + @param val: The size to encode. If `None`, the EBML 'unknown' size + will be returned (1 or `length` bytes, all bits 1). + @keyword length: An explicit length for the encoded size. If `None`, + the size will be encoded at the minimum length required. + @return: an encoded size for an EBML element. + @raise ValueError: raised if the length is invalid, or the length cannot + be encoded. + """ + if val is None: + # 'unknown' size: all bits 1. + length = 1 if (length is None or length == -1) else length + return b'\xff' * length + + length = getLength(val) if (length is None or length == -1) else length + try: + prefix = LENGTH_PREFIXES[length] + return encodeUInt(val | prefix, length) + except (IndexError, TypeError): + raise ValueError("Cannot encode element size %s" % length) + + +# ============================================================================== +# --- Encoding +# ============================================================================== + +def encodeId(eid, length=None): + """ Encode an element ID. + + @param eid: The EBML ID to encode. + @keyword length: An explicit length for the encoded data. A `ValueError` + will be raised if the length is too short to encode the value. + @return: The binary representation of ID, left-padded with ``0x00`` if + `length` is not `None`. + @return: The encoded version of the ID. + @raise ValueError: raised if length is less than one or more than 4. + """ + if length is not None: + if length < 1 or length > 4: + raise ValueError("Cannot encode an ID 0x%0x to length %d" % + (eid, length)) + return encodeUInt(eid, length) + + +def encodeUInt(val, length=None): + """ Encode an unsigned integer. + + @param val: The unsigned integer value to encode. + @keyword length: An explicit length for the encoded data. A `ValueError` + will be raised if the length is too short to encode the value. + @return: The binary representation of val as an unsigned integer, + left-padded with ``0x00`` if `length` is not `None`. + @raise ValueError: raised if val is longer than length. + """ + if isinstance(val, float): + fval, val = val, int(val) + if fval != val: + warnings.warn('encodeUInt: float value {} encoded as {}'.format(fval, val)) + + pad = b'\x00' + packed = _struct_uint64.pack(val).lstrip(pad) or pad + + if length is None: + return packed + if len(packed) > length: + raise ValueError("Encoded length (%d) greater than specified length " + "(%d)" % (len(packed), length)) + return packed.rjust(length, pad) + + +def encodeInt(val, length=None): + """ Encode a signed integer. + + @param val: The signed integer value to encode. + @keyword length: An explicit length for the encoded data. A `ValueError` + will be raised if the length is too short to encode the value. + @return: The binary representation of val as a signed integer, + left-padded with either ```0x00`` (for positive values) or ``0xFF`` + (for negative) if `length` is not `None`. + @raise ValueError: raised if val is longer than length. + """ + if isinstance(val, float): + fval, val = val, int(val) + if fval != val: + warnings.warn('encodeInt: float value {} encoded as {}'.format(fval, val)) + + if val >= 0: + pad = b'\x00' + packed = _struct_int64.pack(val).lstrip(pad) or pad + if packed[0] & 0b10000000: + packed = pad + packed + else: + pad = b'\xff' + packed = _struct_int64.pack(val).lstrip(pad) or pad + if not packed[0] & 0b10000000: + packed = pad + packed + + if length is None: + return packed + if len(packed) > length: + raise ValueError("Encoded length (%d) greater than specified length " + "(%d)" % (len(packed), length)) + return packed.rjust(length, pad) + + +def encodeFloat(val, length=None): + """ Encode a floating point value. + + @param val: The floating point value to encode. + @keyword length: An explicit length for the encoded data. Must be + `None`, 0, 4, or 8; otherwise, a `ValueError` will be raised. + @return: The binary representation of val as a float, left-padded with + ``0x00`` if `length` is not `None`. + @raise ValueError: raised if val not length 0, 4, or 8 + """ + if length is None: + if val is None or val == 0.0: + return b'' + else: + length = DEFAULT_FLOAT_SIZE + + if length == 0: + return b'' + if length == 4: + return _struct_float32.pack(val) + elif length == 8: + return _struct_float64.pack(val) + else: + raise ValueError("Cannot encode float of length %d; only 0, 4, or 8" % + length) + + +def encodeBinary(val, length=None): + """ Encode binary data. + + @param val: A string or bytearray containing the data to encode. + @keyword length: An explicit length for the encoded data. A + `ValueError` will be raised if `length` is shorter than the + actual length of the binary data. + @return: The binary representation of value as binary data, left-padded + with ``0x00`` if `length` is not `None`. + @raise ValueError: raised if val is longer than length. + """ + if isinstance(val, str): + val = val.encode('utf_8') + elif val is None: + val = b'' + + if length is None: + return val + elif len(val) <= length: + return val.ljust(length, b'\x00') + else: + raise ValueError("Length of data (%d) exceeds specified length (%d)" % + (len(val), length)) + + +def encodeString(val, length=None): + """ Encode an ASCII string. + + @param val: The string (or bytearray) to encode. + @keyword length: An explicit length for the encoded data. Longer + strings will be truncated. + @keyword length: An explicit length for the encoded data. The result + will be truncated if the length is less than that of the original. + @return: The binary representation of val as a string, truncated or + left-padded with ``0x00`` if `length` is not `None`. + """ + if isinstance(val, str): + val = val.encode('ascii', 'replace') + + if length is not None: + val = val[:length] + + return encodeBinary(val.translate(STRING_CHARACTERS), length) + + +def encodeUnicode(val, length=None): + """ Encode a Unicode string. + + @param val: The Unicode string to encode. + @keyword length: An explicit length for the encoded data. The result + will be truncated if the length is less than that of the original. + @return: The binary representation of val as a string, truncated or + left-padded with ``0x00`` if `length` is not `None`. + """ + val = val.encode('utf_8') + + if length is not None: + val = val[:length] + + return encodeBinary(val, length) + + +def encodeDate(val, length=None): + """ Encode a `datetime` object as an EBML date (i.e. nanoseconds since + 2001-01-01T00:00:00). + + @param val: The `datetime.datetime` object value to encode. + @keyword length: An explicit length for the encoded data. Must be + `None` or 8; otherwise, a `ValueError` will be raised. + @return: The binary representation of val as an 8-byte dateTime. + @raise ValueError: raised if the length of the input is not 8 bytes. + """ + if length is None: + length = 8 + elif length != 8: + raise ValueError("Dates must be of length 8") + + if val is None: + val = datetime.datetime.utcnow() + + delta = val - datetime.datetime(2001, 1, 1, tzinfo=None) + nanoseconds = (delta.microseconds + + ((delta.seconds + (delta.days * 86400)) * 1000000)) * 1000 + return encodeInt(nanoseconds, length) diff --git a/lambda/cctv-people-rekognition/ebmlite/schemata/__init__.py b/lambda/cctv-people-rekognition/ebmlite/schemata/__init__.py new file mode 100644 index 0000000..84a3588 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/schemata/__init__.py @@ -0,0 +1,7 @@ +""" +ebmlite.schemata: Submodule containing the default schema XML files. It doesn't +contain any code; it's just for storage. Making it a module makes it easy to +find. + +For the definition of the `Schema` class, see `ebmlite.core`. +""" diff --git a/lambda/cctv-people-rekognition/ebmlite/schemata/matroska.xml b/lambda/cctv-people-rekognition/ebmlite/schemata/matroska.xml new file mode 100644 index 0000000..a00281c --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/schemata/matroska.xml @@ -0,0 +1,239 @@ + + +Set the EBML characteristics of the data to follow. Each EBML document has to start with this. + The version of EBML parser used to create the file. + The minimum EBML version a parser has to support to read this file. + The maximum length of the IDs you'll find in this file (4 or less in Matroska). + The maximum length of the sizes you'll find in this file (8 or less in Matroska). This does not override the element size indicated at the beginning of an element. Elements that have an indicated size which is larger than what is allowed by EBMLMaxSizeLength shall be considered invalid. + A string that describes the type of document that follows this EBML header. 'matroska' in our case or 'webm' for webm files. + The version of DocType interpreter used to create the file. + The minimum DocType version an interpreter has to support to read this file. + Used to void damaged data, to avoid unexpected behaviors when using damaged data. The content is discarded. Also used to reserve space in a sub-element for later use. + The CRC is computed on all the data of the Master element it's in. The CRC element should be the first in it's parent master for easier reading. All level 1 elements should include a CRC-32. The CRC in use is the IEEE CRC32 Little Endian + Contain signature of some (coming) elements in the stream. + Signature algorithm used (1=RSA, 2=elliptic). + Hash algorithm used (1=SHA1-160, 2=MD5). + The public key to use with the algorithm (in the case of a PKI-based signature). + The signature of the data (until a new. + + Contains elements that will be used to compute the signature. + A list consists of a number of consecutive elements that represent one case where data is used in signature. Ex: Cluster|Block|BlockAdditional means that the BlockAdditional of all Blocks in all Clusters is used for encryption. + An element ID whose data will be used to compute the signature. + + + + +This element contains all other top-level (level 1) elements. Typically a Matroska file is composed of 1 segment. + Contains the position of other level 1 elements. + Contains a single seek entry to an EBML element. + The binary ID corresponding to the element name. + The position of the element in the segment in octets (0 = first level 1 element). + + + Contains miscellaneous general information and statistics on the file. + A randomly generated unique ID to identify the current segment between many others (128 bits). + A filename corresponding to this segment. + A unique ID to identify the previous chained segment (128 bits). + An escaped filename corresponding to the previous segment. + A unique ID to identify the next chained segment (128 bits). + An escaped filename corresponding to the next segment. + A randomly generated unique ID that all segments related to each other must use (128 bits). + A tuple of corresponding ID used by chapter codecs to represent this segment. + Specify an edition UID on which this correspondance applies. When not specified, it means for all editions found in the segment. + The chapter codec using this ID (0: Matroska Script, 1: DVD-menu). + The binary value used to represent this segment in the chapter codec data. The format depends on the ChapProcessCodecID used. + + Timecode scale in nanoseconds (1.000.000 means all timecodes in the segment are expressed in milliseconds). + Duration of the segment (based on TimecodeScale). + Date of the origin of timecode (value 0), i.e. production date. + General name of the segment. + Muxing application or library ("libmatroska-0.4.3"). + Writing application ("mkvmerge-0.3.3"). + + The lower level element containing the (monolithic) Block structure. + Absolute timecode of the cluster (based on TimecodeScale). + The list of tracks that are not used in that part of the stream. It is useful when using overlay tracks on seeking. Then you should decide what track to use. + One of the track number that are not used from now on in the stream. It could change later if not specified as silent in a further Cluster. + + The Position of the Cluster in the segment (0 in live broadcast streams). It might help to resynchronise offset on damaged streams. + Size of the previous Cluster, in octets. Can be useful for backward playing. + Similar to Block but without all the extra information, mostly used to reduced overhead when no extra feature is needed. (see SimpleBlock Structure) + Basic container of information containing a single Block or BlockVirtual, and information specific to that Block/VirtualBlock. + + Similar to SimpleBlock but the data inside the Block are Transformed (encrypt and/or signed). (see EncryptedBlock Structure) + + A top-level block of information with many tracks described. + Describes a track with all elements. + The track number as used in the Block Header (using more than 127 tracks is not encouraged, though the design allows an unlimited number). + A unique ID to identify the Track. This should be kept the same when making a direct stream copy of the Track to another file. + A set of track types coded on 8 bits (1: video, 2: audio, 3: complex, 0x10: logo, 0x11: subtitle, 0x12: buttons, 0x20: control). + Set if the track is used. (1 bit) + Set if that track (audio, video or subs) SHOULD be used if no language found matches the user preference. (1 bit) + Set if that track MUST be used during playback. There can be many forced track for a kind (audio, video or subs), the player should select the one which language matches the user preference or the default + forced track. Overlay MAY happen between a forced and non-forced track of the same kind. (1 bit) + Set if the track may contain blocks using lacing. (1 bit) + The minimum number of frames a player should be able to cache during playback. If set to 0, the reference pseudo-cache system is not used. + The maximum cache size required to store referenced frames in and the current frame. 0 means no cache is needed. + Number of nanoseconds (i.e. not scaled) per frame. + The scale to apply on this track to work at normal speed in relation with other tracks (mostly used to adjust video speed when the audio length differs). + A value to add to the Block's Timecode. This can be used to adjust the playback offset of a track. + The maximum value of BlockAddID. A value 0 means there is no BlockAdditions for this track. + A human-readable track name. + Specifies the language of the track in the Matroska languages form. + An ID corresponding to the codec, see the codec page for more info. + Private data only known to the codec. + A human-readable string specifying the codec. + The UID of an attachment that is used by this codec. + A string describing the encoding setting used. + A URL to find information about the codec used. + A URL to download about the codec used. + The codec can decode potentially damaged data (1 bit). + Specify that this track is an overlay track for the Track specified (in the u-integer). That means when this track has a gap (see SilentTracks) the overlay track should be used instead. The order of multiple TrackOverlay matters, the first one is the one that should be used. If not found it should be the second, etc. + CodecDelay is The codec-built-in delay in nanoseconds. This value MUST be subtracted from each block timestamp in order to get the actual timestamp. The value SHOULD be small so the muxing of tracks with the same actual timestamp are in the same Cluster. + The track identification for the given Chapter Codec. + Specify an edition UID on which this translation applies. When not specified, it means for all editions found in the segment. + The chapter codec using this ID (0: Matroska Script, 1: DVD-menu). + The binary value used to represent this track in the chapter codec data. The format depends on the ChapProcessCodecID used. + + Video settings + Set if the video is interlaced. (1 bit) + Stereo-3D video mode (0: mono, 1: side by side (left eye is first), 2: top-bottom (right eye is first), 3: top-bottom (left eye is first), 4: checkboard (right is first), 5: checkboard (left is first), 6: row interleaved (right is first), 7: row interleaved (left is first), 8: column interleaved (right is first), 9: column interleaved (left is first), 10: anaglyph (cyan/red), 11: side by side (right eye is first), 12: anaglyph (green/magenta)) . There are some more details on 3D support in the Specification Notes. + Bogus StereoMode value used in old versions of libmatroska. DO NOT USE. (0: mono, 1: right eye, 2: left eye, 3: both eyes). + Width of the encoded video frames in pixels. + Height of the encoded video frames in pixels. + The number of video pixels to remove at the bottom of the image (for HDTV content). + The number of video pixels to remove at the top of the image. + The number of video pixels to remove on the left of the image. + The number of video pixels to remove on the right of the image. + Width of the video frames to display. The default value is only valid when DisplayUnit is 0. + Height of the video frames to display. The default value is only valid when DisplayUnit is 0. + How DisplayWidth & DisplayHeight should be interpreted (0: pixels, 1: centimeters, 2: inches, 3: Display Aspect Ratio). + Specify the possible modifications to the aspect ratio (0: free resizing, 1: keep aspect ratio, 2: fixed). + Same value as in AVI (32 bits). + Gamma Value. + Number of frames per second. Informational only. + + Audio settings. + Sampling frequency in Hz. + Real output sampling frequency in Hz (used for SBR techniques). + Numbers of channels in the track. + Table of horizontal angles for each successive channel, see appendix. + Bits per sample, mostly used for PCM. + + Operation that needs to be applied on tracks to create this virtual track. For more details look at the Specification Notes on the subject. + Contains the list of all video plane tracks that need to be combined to create this 3D track + Contains a video plane track that need to be combined to create this 3D track + The trackUID number of the track representing the plane. + The kind of plane this track corresponds to (0: left eye, 1: right eye, 2: background). + + + Contains the list of all tracks whose Blocks need to be combined to create this virtual track + The trackUID number of a track whose blocks are used to create this virtual track. + + + DivX trick track extenstions + DivX trick track extenstions + DivX trick track extenstions + DivX trick track extenstions + DivX trick track extenstions + Settings for several content encoding mechanisms like compression or encryption. + Settings for one content encoding like compression or encryption. + Tells when this modification was used during encoding/muxing starting with 0 and counting upwards. The decoder/demuxer has to start with the highest order number it finds and work its way down. This value has to be unique over all ContentEncodingOrder elements in the segment. + A bit field that describes which elements have been modified in this way. Values (big endian) can be OR'ed. Possible values:
1 - all frame contents,
2 - the track's private data,
4 - the next ContentEncoding (next ContentEncodingOrder. Either the data inside ContentCompression and/or ContentEncryption) + A value describing what kind of transformation has been done. Possible values:
0 - compression,
1 - encryption + Settings describing the compression used. Must be present if the value of ContentEncodingType is 0 and absent otherwise. Each block must be decompressable even if no previous block is available in order not to prevent seeking. + The compression algorithm used. Algorithms that have been specified so far are:
0 - zlib,
1 - bzlib,
2 - lzo1x
3 - Header Stripping + Settings that might be needed by the decompressor. For Header Stripping (ContentCompAlgo=3), the bytes that were removed from the beggining of each frames of the track. + + Settings describing the encryption used. Must be present if the value of ContentEncodingType is 1 and absent otherwise. + The encryption algorithm used. The value '0' means that the contents have not been encrypted but only signed. Predefined values:
1 - DES, 2 - 3DES, 3 - Twofish, 4 - Blowfish, 5 - AES + For public key algorithms this is the ID of the public key the the data was encrypted with. + A cryptographic signature of the contents. + This is the ID of the private key the data was signed with. + The algorithm used for the signature. A value of '0' means that the contents have not been signed but only encrypted. Predefined values:
1 - RSA + The hash algorithm used for the signature. A value of '0' means that the contents have not been signed but only encrypted. Predefined values:
1 - SHA1-160
2 - MD5 + + + + + + A top-level element to speed seeking access. All entries are local to the segment. Should be mandatory for non "live" streams. + Contains all information relative to a seek point in the segment. + Absolute timecode according to the segment time base. + Contain positions for different tracks corresponding to the timecode. + The track for which a position is given. + The position of the Cluster containing the required Block. + Number of the Block in the specified Cluster. + The position of the Codec State corresponding to this Cue element. 0 means that the data is taken from the initial Track Entry. + The Clusters containing the required referenced Blocks. + Timecode of the referenced Block. + The Position of the Cluster containing the referenced Block. + Number of the referenced Block of Track X in the specified Cluster. + The position of the Codec State corresponding to this referenced element. 0 means that the data is taken from the initial Track Entry. + + + + + Contain attached files. + An attached file. + A human-friendly name for the attached file. + Filename of the attached file. + MIME type of the file. + The data of the file. + Unique ID representing the file, as random as possible. + A binary value that a track/codec can refer to when the attachment is needed. + DivX font extension + DivX font extension + + + A system to define basic menus and partition data. For more detailed information, look at the Chapters Explanation. + Contains all information about a segment edition. + A unique ID to identify the edition. It's useful for tagging an edition. + If an edition is hidden (1), it should not be available to the user interface (but still to Control Tracks). (1 bit) + If a flag is set (1) the edition should be used as the default one. (1 bit) + Specify if the chapters can be defined multiple times and the order to play them is enforced. (1 bit) + Contains the atom information to use as the chapter atom (apply to all tracks). + A unique ID to identify the Chapter. + Timecode of the start of Chapter (not scaled). + Timecode of the end of Chapter (timecode excluded, not scaled). + If a chapter is hidden (1), it should not be available to the user interface (but still to Control Tracks). (1 bit) + Specify wether the chapter is enabled. It can be enabled/disabled by a Control Track. When disabled, the movie should skip all the content between the TimeStart and TimeEnd of this chapter. (1 bit) + A segment to play in place of this chapter. Edition ChapterSegmentEditionUID should be used for this segment, otherwise no edition is used. + The edition to play from the segment linked in ChapterSegmentUID. + Specify the physical equivalent of this ChapterAtom like "DVD" (60) or "SIDE" (50), see complete list of values. + List of tracks on which the chapter applies. If this element is not present, all tracks apply + UID of the Track to apply this chapter too. In the absense of a control track, choosing this chapter will select the listed Tracks and deselect unlisted tracks. Absense of this element indicates that the Chapter should be applied to any currently used Tracks. + + Contains all possible strings to use for the chapter display. + Contains all the commands associated to the Atom. + Contains the type of the codec used for the processing. A value of 0 means native Matroska processing (to be defined), a value of 1 means the DVD command set is used. More codec IDs can be added later. + Some optional data attached to the ChapProcessCodecID information. For ChapProcessCodecID = 1, it is the "DVD level" equivalent. + Contains all the commands associated to the Atom. + Defines when the process command should be handled (0: during the whole chapter, 1: before starting playback, 2: after playback of the chapter). + Contains the command information. The data should be interpreted depending on the ChapProcessCodecID value. For ChapProcessCodecID = 1, the data correspond to the binary DVD cell pre/post commands. + + + + + + + Element containing elements specific to Tracks/Chapters. A list of valid tags can be found here. + Element containing elements specific to Tracks/Chapters. + Contain all UIDs where the specified meta data apply. It is empty to describe everything in the segment. + A number to indicate the logical level of the target (see TargetType). + An informational string that can be used to display the logical level of the target like "ALBUM", "TRACK", "MOVIE", "CHAPTER", etc (see TargetType). + A unique ID to identify the Track(s) the tags belong to. If the value is 0 at this level, the tags apply to all tracks in the Segment. + A unique ID to identify the EditionEntry(s) the tags belong to. If the value is 0 at this level, the tags apply to all editions in the Segment. + A unique ID to identify the Chapter(s) the tags belong to. If the value is 0 at this level, the tags apply to all chapters in the Segment. + A unique ID to identify the Attachment(s) the tags belong to. If the value is 0 at this level, the tags apply to all the attachments in the Segment. + + Contains general information about the target. + The name of the Tag that is going to be stored. + Specifies the language of the tag specified, in the Matroska languages form. + Indication to know if this is the default/original language to use for the given tag. (1 bit) + The value of the Tag. + The values of the Tag if it is binary. Note that this cannot be used in the same SimpleTag as TagString. + + + + + diff --git a/lambda/cctv-people-rekognition/ebmlite/schemata/mide_config_ui.xml b/lambda/cctv-people-rekognition/ebmlite/schemata/mide_config_ui.xml new file mode 100644 index 0000000..9836e04 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/schemata/mide_config_ui.xml @@ -0,0 +1,729 @@ + + + + + + + + + Set the EBML characteristics of the data to follow. Each EBML document has to start with this. + The version of EBML parser used to create the file. + The minimum EBML version a parser has to support to read this file. + The maximum length of the IDs you'll find in this file (4 or less in Matroska). + The maximum length of the sizes you'll find in this file (8 or less in Matroska). This does not override the element size indicated at the beginning of an element. Elements that have an indicated size which is larger than what is allowed by EBMLMaxSizeLength shall be considered invalid. + A string that describes the type of document that follows this EBML header. 'mide' for Mide Instrumentation Data Exchange files. + The version of DocType interpreter used to create the file. + The minimum DocType version an interpreter has to support to read this file. + Used to void damaged data, to avoid unexpected behaviors when using damaged data. The content is discarded. Also used to reserve space in a sub-element for later use. + The CRC is computed on all the data of the Master element it's in. The CRC element should be the first in it's parent master for easier reading. All level 1 elements should include a CRC-32. The CRC in use is the IEEE CRC32 Little Endian + Contain signature of some (coming) elements in the stream. + Signature algorithm used (1=RSA, 2=elliptic). + Hash algorithm used (1=SHA1-160, 2=MD5). + The public key to use with the algorithm (in the case of a PKI-based signature). + The signature of the data (until a new. + Contains elements that will be used to compute the signature. + A list consists of a number of consecutive elements that represent one case where data is used in signature. Ex: Cluster|Block|BlockAdditional means that the BlockAdditional of all Blocks in all Clusters is used for encryption. + An element ID whose data will be used to compute the signature. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A 'field' for config items without individual UI widgets (e.g. created from multiple other fields) + + For hidden fields, this determines if the config item will be written. + + + + + + + + + + + + Special-case widget that creates a resizable vertical spacer, placing the fields following it on the bottom of the dialog. + + + + Special-case control that compares device time to system time. + + + + Special-case control that resets all the siblings in its group or tab to their factory default values. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Container For arbitrary name/value attributes, allowing additional data without revising (and bloating) the schema. All of these elements are level -1, allowing an AttributeList to occur at any level, but should always be used at the relative levels implied below. + Attribute name. Should always be child of Atrribute. + Integer Attribute. Should always be child of Atrribute. + Unsigned integer Attribute. Should always be child of Atrribute. + Floating point Attribute. Should always be child of Atrribute. + ASCII String Attribute. Should always be child of Atrribute. + Date Attribute. Should always be child of Atrribute. + Binary Attribute. Should always be child of Atrribute. + ASCII String Attribute. Should always be child of Atrribute. + + + diff --git a/lambda/cctv-people-rekognition/ebmlite/schemata/mide_ide.xml b/lambda/cctv-people-rekognition/ebmlite/schemata/mide_ide.xml new file mode 100644 index 0000000..b97c885 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/schemata/mide_ide.xml @@ -0,0 +1,385 @@ + + + + Set the EBML characteristics of the data to follow. Each EBML document has to start with this. + The version of EBML parser used to create the file. + The minimum EBML version a parser has to support to read this file. + The maximum length of the IDs you'll find in this file (4 or less in Matroska). + The maximum length of the sizes you'll find in this file (8 or less in Matroska). This does not override the element size indicated at the beginning of an element. Elements that have an indicated size which is larger than what is allowed by EBMLMaxSizeLength shall be considered invalid. + A string that describes the type of document that follows this EBML header. 'mide' for Mide Instrumentation Data Exchange files. + The version of DocType interpreter used to create the file. + The minimum DocType version an interpreter has to support to read this file. + Used to void damaged data, to avoid unexpected behaviors when using damaged data. The content is discarded. Also used to reserve space in a sub-element for later use. + The CRC is computed on all the data of the Master element it's in. The CRC element should be the first in it's parent master for easier reading. All level 1 elements should include a CRC-32. The CRC in use is the IEEE CRC32 Little Endian + Contain signature of some (coming) elements in the stream. + Signature algorithm used (1=RSA, 2=elliptic). + Hash algorithm used (1=SHA1-160, 2=MD5). + The public key to use with the algorithm (in the case of a PKI-based signature). + The signature of the data (until a new. + Contains elements that will be used to compute the signature. + A list consists of a number of consecutive elements that represent one case where data is used in signature. Ex: Cluster|Block|BlockAdditional means that the BlockAdditional of all Blocks in all Clusters is used for encryption. + An element ID whose data will be used to compute the signature. + + + + + + + + Device EBML schema (aka 'tagset') hint. Points to a numeric schema ID defined at the receiving side. + Used to provide an arbitrary length sync word (for network / stream framing purposes) at any point in the stream. + + Arbitrary tag. Allow for separate opening and closing tags without knowing the length of the enclosed data in advance. I.e. instead of [tag len value=[subtag len... /]/], [tag len=0][subtags and contents][/tag]. Positive value corresponds to the corresponding ElementID as an opening tag; the corresponding negative value as the closing tag. Value -int_max for any int size is reserved. + + + + + + + Master element for the RecordingProperties branch of the file, if present. + Master element for the RecorderInfo items (ID, etc.). + + Unique Type ID assigned to each recorder variation. May be used to lookup remaining RecordingProperties when streaming. + Recorder unique serial number + ID referencing Schema in use by recorder + Product designation of the recording device + Text name of this recorder, if any. Probably set by the user. + Hardware revision level + Firmware revision level + Device part .number. (text product identifier e.g. VR002-100-XYZ). + Device .birthdate. (manufacture date) in UTC seconds since the Epoch. + Custom hardware identifier. Hardware is a custom version if present. + Custom firmware build. Firmware is a custom build if present. Name should match FW branch/tag name as applicable for identification purposes, but is mainly present so FW updater can generate a warning if a custom build will be replaced by a standard one. + Firmware revision string. + The device CPU's factory-set unique ID. + Code indicating the type of CPU. + Bootloader revision string. + Incrementing bootloader revision level + + + Master element for the SensorList items + Master element for a Sensor entry + ID of a given Sensor entry; can be used to refer back to this sensor. + Text name assigned to a sensor; probably the sensor make/model if present. + Reference to a BwLimitList entry. If present, this discloses the usable bandwidth range of the sensor itself. + Master element for any traceability data tied to the Sensor. + Sensor manufacturer-supplied serial number, if any. + + + + + + Master element for a Channel entry + Master element for a Channel entry + ID of a given Channel entry; can be used to refer to this entry. Referenced by ChannelDataBlock/SimpleChannelDataBlock/etc. + Descriptive overall text name for this channel (e.g. "ADC"). + Reference to a Calibration in CalibrationList. + Declaration of the format and representation of the (sub)channels' data. Modeled after Python Struct format strings, may be expanded at a later date. Currently limited to the expanded format-string format, i.e. [>HHHHH] rather than [>5H]. Whitespace and undefined formatting characters are ignored. + Name of the hardcoded parsing object, if known. Overrides ChannelFormat. + + Channel seconds/tick. String represents a valid numeric expression, such as integer, decimal or ratio, e.g. '16262/32768'. Applies to all subsequent Abs timecodes. If not set, use channel default. + The modulus at which modulo timestamps for this channel roll over. + The samplerate for this channel, if known and fixed. String represents a valid numeric expression, such as integer, decimal or ratio, e.g. '32768/16262'. Element required if both starting and ending timecodes will ever be omitted for blocks in this channel. + Master element for SensorSubChannels. + ID of this SubChannel. Currently, SubChannelIDs must be sequential, starting from 0, for use with Slam Stick Lab. + Display name of the subchannel, typically the axis name for multiaxis measurements (e.g. "X", "Yaw", etc.). Typically omitted if no display name is needed beyond a measurement label and units. + Reference to a Calibration in CalibrationList. + Reference to a BwLimitList entry. If present, this discloses any bandwidth limitations imposed for the acquisition channel, e.g. antialias or other filter settings. Note that the effective bandwidth is the lesser of the Sensor and SubChannel bandwidth! + General measurement type label for this subchannel (e.g. "Acceleration", "Rotation", "Temperature", etc.). + Text name of the engineering unit (e.g. "g") for this subchannel. Unicode characters such as degree or Greek symbols are allowed and encouraged. + Minimum valid sensor value; may be used for data validation or initial axis scaling during display. + Maximum valid sensor value; may be used for data validation or initial axis scaling during display. + Reference to a Sensor ID. Allows the association between SubChannels and a physical sensor to be expressed. + Reference to a warning range, i.e. another sensor measuring something that affects the results of this one. + Allow data channels to specify a visibility level; could have different visibility levels for e.g. 'advanced', 'on request' (channels which are not normally useful on their own, e.g. IRIG stream), 'hidden' (internal diagnostic or other dirty laundry). + The RGB values of the default plotting color (3 bytes). + Text name of the sensor output units. + Reference to a numbered, standardized SI unit. + + + Master element for the PlotList items. It works like a virtual Channel, with Plots being its Subchannels. + Master element for a Plot entry + + SubChannelID + Text name of the subchannel (e.g. axis?). + Reference to a Calibration in CalibrationList. + Minimum valid sensor value. + Maximum valid sensor value. + Reference to a warning range, i.e. another sensor measuring something that affects the results of this one + Allow data channels to specify a visibility level; could have different visibility levels for e.g. 'advanced', 'on request' (channels which are not normally useful on their own, e.g. IRIG stream), 'hidden' (internal diagnostic or other dirty laundry). + The RGB values of the default plotting color (3 bytes). + Text name of the sensor output units. + Reference to a numbered, standardized SI unit. + + + Text name of the axis(?). + Reference to an Axis. + Text name of the sensor output units. + Reference to a numbered, standardized SI unit. + + List of channels/subchannels used by this Plot + SubChannelID + SubChannelID + + + + + + "Idiot Light" warnings for sensors that are inaccurate in certain conditions + + Warning ID, referenced by Subchannels. + ChannelID + SubChannelID + Minimum valid value. Note: this is in real, post-converted units, not raw channel units. + Maximum valid value. Note: this is in real, post-converted units, not raw channel units. + + + + List of bandwidth constraints on sensors and data channels. These will (on a Sensor) provide the intrinsic sensor limits if known, or (on a data channel) provide information about any user-configured limits, e.g. AA filter cutoffs. This data is mandatory for sensor fusion, otherwise optional. + + Limit entry ID, referenced by Sensors or (Sub)channels. + Lower cutoff frequency. + Lower rolloff, in dB/decade. + Upper cutoff frequency. + Upper rolloff, in dB/decade. + Group delay, in channel ticks (channels/subchannels only). + + + + + Container For arbitrary name/value attributes, allowing additional data without revising (and bloating) the schema. All of these elements are level -1, allowing an AttributeList to occur at any level, but should always be used at the relative levels implied below. + Attribute name. Should always be child of Atrribute. + Integer Attribute. Should always be child of Atrribute. + Unsigned integer Attribute. Should always be child of Atrribute. + Floating point Attribute. Should always be child of Atrribute. + ASCII String Attribute. Should always be child of Atrribute. + Date Attribute. Should always be child of Atrribute. + Binary Attribute. Should always be child of Atrribute. + ASCII String Attribute. Should always be child of Atrribute. + + + + + + + + Master element for the CalibrationList items + Master element for a univariate (1-variable) calibration entry + ID of this entry, referenced by SensorList. + Reference value for this channel + Univariate or bivariate polynomial coefficient in canonical order + + Master element for a bivariate (2-variable) calibration entry + ID of this entry, referenced by SensorList. + Reference value for this channel + Reference value for the 2nd channel when using bivariate calibration + Channel ID of the channel to be used as the 2nd parameter for bivariate compensation + SubChannel ID of the subchannel to be used as the 2nd parameter for bivariate compensation + Univariate or bivariate polynomial coefficient in canonical order + + Date of factory calibration, in UTC seconds + Date of factory calibration expiration, in UTC seconds + Mide-assigned serial # for a specific (re)calibration procedure. Can be used to lookup calibration conditions and exact facility / test stand used, etc. + + + + + Master element for the RecorderConfiguration block, if present. This allows a mechanism to attach the actual recorder configuration to the file as metadata, and (hacky) use the same schema document to parse output files and write configuration files. It is expected that by necessity, children of RecorderConfiguration will be recorder-specific. + Master element for the SSX-specific RecorderConfiguration block. This and its subelements are specific to the basic Slam Stick X. + Sampling frequency for the high speed channel(s) in Hz. + Antialias filter corner frequency for the high speed channel(s) in Hz. Omit to let the device choose based on the sampling rate. Enter 0 to disable (bypass) AA filter. (Beware: diagnostic usage only; calibration will be invalid in this state.) + Oversampling ratio. Allowed values are 1 or power-of-2 integer in the range from 16 to 4096. Omit to let the device choose based on the sampling rate. + UTC offset in seconds. May be used for entering local timezone offset. Sign follows the normal convention for specifying timezone offsets, e.g. "UTC-5" (-5*3600) to display the local time as 5 hours earlier than UTC. This value is only used to convert the internal UTC time to local time to generate the FAT16/32 file timestamp, which is expected to be in local time (sans any DST offset, which is added by the host system). If the device clock is directly set in local time for any reason, this field should be 0. + Defines device behavior in response to an unexpected plug-in (or other power/data connection) during recording. Current options are 0 (exit immediately), 1 (exit after current recording finishes - HW default), or 2 (ignore - require button presses to terminate recording mode). + + Master element for SSX trigger configuration. + Absolute, calendar time in UTC at which to arm/start the recorder. WakeTimeUTC and PreRecordDelay are mutually exclusive. + Delay in seconds before the start of recording (or trigger arming). WakeTimeUTC and PreRecordDelay are mutually exclusive. + Automatically rearm at end of triggered recording (do not require button press) + Time in seconds to record for when triggered. Omit to impose no limit. + + Master element for a generic trigger. + The sensor channel used as the trigger. Mandatory for all Triggers. + The sensor subchannel used by the trigger. + Lower value of trigger window in native units. + Upper value of trigger window in native units. + + + Master element for channel-specific configuration elements. + Channel to apply these configuration elements to. + Sampling frequency for this channel in Hz, if supported by device. Unsupported rates will be coerced to nearest supported rate. + Bitmap indicating enabled/disabled SubChannels, if supported by device. If only Channel-level enable/disable supported by this channel, any nonzero value will enable the entire Channel. + + The user-defined properties of the recorder, not used by the recorder itself. + The user-defined name of the recorder + The user-defined description of/notes on the recorder + + + + + New configuration data: pairs of config IDs and values. Duplicated from + the config_ui.xml schema, rather than attempting to parse one element + with a different schema. + + + + + + + + + + + + + + Exported configuration data. Also contains device information and + CONFIG.UI for compatibility. + + + + Actually from the CONFIG.UI schema. Read as binary in this schema, + then parsed with the other one. + + + + + + + This element contains all other top-level (level 1) elements. A session basically comprises one 'recording' and may contain any number of individual channels. A typical file should consist of one segment (mostly because we won't really know its length in advance). + + + + Session time base value in Unix Time. If present, used as the base for all future timecodes in the session. + + This element contains potentially-channelized instrumentation data in a minimalist format. Its mandatory, fixed-length header includes a 2-byte modulo timecode (scaled to the channel's TimecodeScale, default of 1/32768 sec) and a 1-byte integer ChannelID. + + This element contains child elements including instrumentation data associated to a channel. This is used for e.g. binding a timestamp(s) and/or metadata to a specific multi-sample block of sensor data, which may be written asynchronously with respect to other channels' data (i.e. multiplexed). + Child of ChannelDataBlock: the channel this data is associated with + Child of ChannelDataBlock: optional flags to indicate datablock features such as discontinuity + Child of ChannelDataBlock: the actual channel data samples. If there are multiple subchannels, for each sample point, the sample for each subchannel will be written consecutively (i.e. [sc0 sc1 sc2] [sc0 sc1 sc2]). + Absolute timecode as an offset from the session TimeBase. The timecode resolution is given by the channel's TimeCodeScale. + Absolute timecode as an offset from the session TimeBase. The timecode resolution is given by the channel's TimeCodeScale. + Absolute start timecode as an offset from session TimeBase, mod TimeCodeModulus. Allows for a short, rolling-over (but still absolute) code. The timecode resolution is given by the channel's TimeCodeScale. + Absolute end timecode as an offset from session TimeBase, mod TimeCodeModulus. Allows for a short, rolling-over (but still absolute) code. The timecode resolution is given by the channel's TimeCodeScale. + Statistical data for this block's payload consisting of 3 datapoints (min, mean, max) per subchannel. They are organized as [[sc0min] [sc1min] [sc2min] ...] [[sc0mean] [sc1mean] [sc2mean] ...] [[sc0max] [sc1max] [sc2max] ...]. The format and representation of the stat data exactly matches that of the input samples; that is, if the input samples are uint16_t, each stat entry is also a uint16_t. + Super-optional diagnostic element indicating the latency between data acquisition and transfer to the output media. The exact meaning of this value is device-dependent, but may serve as a general indicator of excess activity load, retransmission or congestion (for transmission media) or media wear (for recording media). + + diff --git a/lambda/cctv-people-rekognition/ebmlite/schemata/mide_manifest.xml b/lambda/cctv-people-rekognition/ebmlite/schemata/mide_manifest.xml new file mode 100644 index 0000000..1c5bbd8 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/schemata/mide_manifest.xml @@ -0,0 +1,177 @@ + + + + Set the EBML characteristics of the data to follow. Each EBML document has to start with this. + The version of EBML parser used to create the file. + The minimum EBML version a parser has to support to read this file. + The maximum length of the IDs you'll find in this file (4 or less in Matroska). + The maximum length of the sizes you'll find in this file (8 or less in Matroska). This does not override the element size indicated at the beginning of an element. Elements that have an indicated size which is larger than what is allowed by EBMLMaxSizeLength shall be considered invalid. + A string that describes the type of document that follows this EBML header. 'mide' for Mide Instrumentation Data Exchange files. + The version of DocType interpreter used to create the file. + The minimum DocType version an interpreter has to support to read this file. + Used to void damaged data, to avoid unexpected behaviors when using damaged data. The content is discarded. Also used to reserve space in a sub-element for later use. + The CRC is computed on all the data of the Master element it's in. The CRC element should be the first in it's parent master for easier reading. All level 1 elements should include a CRC-32. The CRC in use is the IEEE CRC32 Little Endian + Contain signature of some (coming) elements in the stream. + Signature algorithm used (1=RSA, 2=elliptic). + Hash algorithm used (1=SHA1-160, 2=MD5). + The public key to use with the algorithm (in the case of a PKI-based signature). + The signature of the data (until a new. + Contains elements that will be used to compute the signature. + A list consists of a number of consecutive elements that represent one case where data is used in signature. Ex: Cluster|Block|BlockAdditional means that the BlockAdditional of all Blocks in all Clusters is used for encryption. + An element ID whose data will be used to compute the signature. + + + + + + + + Device EBML schema (aka 'tagset') hint. Points to a numeric schema ID defined at the receiving side. + Used to provide an arbitrary length sync word (for network / stream framing purposes) at any point in the stream. + + Arbitrary tag. Allow for separate opening and closing tags without knowing the length of the enclosed data in advance. I.e. instead of [tag len value=[subtag len... /]/], [tag len=0][subtags and contents][/tag]. Positive value corresponds to the corresponding ElementID as an opening tag; the corresponding negative value as the closing tag. Value -int_max for any int size is reserved. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Class D (4 byte + 2 length) Device Manifest master. Presence of this master tells the device a valid manifest is present. + Class B (2 byte + 2 length) Master element for the basic system info (serial#, etc). + 4-byte Vendor-defined (MIDE) product serial number. + 4-byte Unique hardware configuration ID used to distinguish recorder types/variants. + 20-byte Displayable text name for this product/variant. + Hardware revision code for this DeviceTypeUID. + + Minimum FW revision code required to support this hardware. + 8-byte date/time of manufacture (initial programming) in UTC seconds since the Epoch. + + + Device part number string (e.g. VR002-100-XYZ). + Custom hardware identifier. Hardware is a custom version if present. + Custom firmware build. Firmware is a custom build if present. Name should match FW branch/tag name as applicable for identification purposes, but is mainly present so FW updater can generate a warning if a custom build will be replaced by a standard one. + Hardware API level of this hardware. This is bumped in response to hardware revs with compatibility implications. + Batch identification string. Text string encoding the date, variant and manufacturer of a given hardware batch. Batch is independent of HwRev as there may be multiple production batches of a given revision, from multiple fab houses. + + + Class B (2 byte + 2 length) Master element for battery info. + 4-byte Nominal battery capacity in mAh. + 1-byte ACMP Vdd scale value corresponding to battery full threshold (0 ~ 63). + 1-byte ACMP Vdd scale value corresponding to battery 'ok' threshold (0 ~ 63). + 1-byte ACMP Vdd scale value corresponding to low-battery alarm threshold (0 ~ 63). + 1-byte ACMP Vdd scale value corresponding to low-battery poweroff threshold (0 ~ 63). + Hardware cutoff voltage value in mV. + + + Class B (2 byte + 2 length) Master element for the antialiasing filter info. + 1-byte filter type code (0=Butterworth, others currently undefined). + 1-byte filter order value. + 2-byte Ratio between input clock frequency and resulting filter corner frequency (clock-tunable filters only). + 4-byte Minimum allowed corner frequency in Hz. + 4-byte Maximum allowed corner frequency in Hz. + 1-byte Boolean specifying that filter bypass is supported (low-fidelity data acquisition allowed in the disabled state). Nonzero = true. + + + Class B (2 byte + 2 length) Master element for one analog sensor of one or more channels. + 1-byte Locally unique ID for referencing this entry. Used as ChannelID. + 1-byte Code identifying the analog sensor. 0 = ADXL001, 1 = 832M1, others=TBD. + 16-byte string for the sensor manufacturer's serial #. This is a string because some vendors (including that of 832M1) love to mix in letters, hyphens and other nonnumeric elements. + + 1-byte Boolean specifies that this sensor goes through the AA filter. Nonzero = true. + 2-byte Sensor start up + settling time in ticks (1/32768s) + 16-byte string for the sensor name / part#. This will probably be overridden by 'Plot'. + 1-byte Reference to a SensorChannel-level CalID. This cal entry will simply determine the fixed sensor scaling and offset. + 4-byte int32 hint expressing the nominal sensor scale and whether it is inverted. For sensors (accelerometer) where one of several nominal sensitivities can be stuffed. This value is optionally used by the device to distinguish stuffed sensitivities without having to interpret a calibration element. + 4-byte float hint expressing the nominal sensor scale and whether it is inverted. For sensors (accelerometer) where one of several nominal sensitivities can be stuffed. This value is optionally used by the device to distinguish stuffed sensitivities without having to interpret a calibration element. + Class B (2 byte + 2 length) Master element for one sensor channel. + 1-byte Entry ID for this channel. + 1-byte ADC channel number corresponding to this sensor channel. + 16-byte Axis name associated with this sensor channel. + 1-byte Reference to a SensorSubChannel-level CalID. This will store actual sensor calibration modifying the basic scale/offset parameters. + 4-byte Lower cutoff frequency in Hz. + 4-byte Upper cutoff frequency in Hz. + CTF filter. + + + + + Indicates the presence of a uSD card on the SPI bus. + + + Indicates the presence of an ADXL362 accelerometer on the SPI bus. + Indicates the presence of an ADXL345 accelerometer on the SPI bus. + Indicates the presence of an ADXL355 accelerometer on the SPI bus. + Configuration data for digital sensors. Value varies by hardware. + + Indicates the presence of an ADXL357 accelerometer on the SPI bus. + Configuration data for digital sensors. Value varies by hardware. + + Indicates the presence of an ADXL375 accelerometer on the SPI bus. + Indicates the presence of a BNO055 IMU on the I2C0 bus. + Indicates the presence of a Bosch BHI160 IMU on the I2C0 bus. + Configuration data for digital sensors. Value varies by hardware. + + Indicates the presence of a BMG250 Gyroscope + Indicates the presence of a BMG250 Gyroscope, with an interrupt line + + + Indicates the presence of an MPL3115A2 P/T sensor on the I2C0 bus. + Indicates the presence of an MS8607 P/T/H sensor on the I2C1 bus. + Indicates the presence of an MS5637 pressure sensor on the I2C1 bus. + Indicates the presence of an HTU21D humidity sensor on the I2C1 bus. + + + Indicates the presence of a digital IR sensor. + + + Indicates the presence of a GPS with full data (location) broken out on UART. + Indicates the presence of a GPS with full data (location) broken out on UART. + Configuration data for digital sensors. Value varies by hardware. + + + + Indicates the presence of a MCP73837 battery charger. + Indicates the presence of a MAX14747 battery charger. + Indicates the presence of no battery charger or just a single cell battery. + + + + Indicates the presence of an ESP32 Wi-Fi module. + + + Indicates the presence of a hardware pushbutton reset controller. + Indicates the presence of a MAX77801 power management IC on the I2C1 bus. + Indicates that the membrane has a 3rd LED driven by the MCU. + Indicates the presence of a SI1133 light sensor on the I2C1 bus. + + + Indicates the presence of an analog heater device. + Configuration data for non-sensor peripherals. For the heater, it is the output power in milliwatts. + + + + + + diff --git a/lambda/cctv-people-rekognition/ebmlite/threaded_file.py b/lambda/cctv-people-rekognition/ebmlite/threaded_file.py new file mode 100644 index 0000000..0fae09d --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/threaded_file.py @@ -0,0 +1,269 @@ +''' +A special-case, drop-in 'replacement' for a standard read-only file stream +that supports simultaneous access by multiple threads without (explicit) +blocking. Each thread actually gets its own stream, so it can perform its +own seeks without affecting other threads that may be reading the file. This +functionality is transparent. + +@author: dstokes +''' +__author__ = "David Randall Stokes, Connor Flanigan" +__copyright__ = "Copyright 2021, Mide Technology Corporation" +__credits__ = "David Randall Stokes, Connor Flanigan, Becker Awqatty, Derek Witt" + +__all__ = ['ThreadAwareFile'] + +import io +import platform +from threading import currentThread, Event + +class ThreadAwareFile(io.FileIO): + """ A 'replacement' for a standard read-only file stream that supports + simultaneous access by multiple threads without (explicit) blocking. + Each thread actually gets its own stream, so it can perform its own + seeks without affecting other threads that may be reading the file. + This functionality is transparent. + + ThreadAwareFile implements the standard `file` methods and has + the standard attributes and properties. Most of these affect only + the current thread. + + @var timeout: A value (in seconds) for blocking operations to wait. + Very few operations block; specifically, only those that do + (or depend upon) internal housekeeping. Timeout should only occur + in certain extreme conditions (e.g. filesystem-related file + access issues). + """ + + def __init__(self, *args, **kwargs): + """ ThreadAwareFile(name[, mode[, buffering]]) -> file object + + Open a read-only file that may already be open in other threads. + Takes the standard `file` arguments, except `mode` can only be + one of the "read" modes (``r``, ``rb``, ``rU``, etc.). + """ + # Ensure the file mode, if specified, is "read." + mode = args[1] if len(args) > 1 else 'r' + if isinstance(mode, (str, bytes, bytearray)): + if 'a' in mode or 'w' in mode or '+' in mode: + raise IOError("%s is read-only" % self.__class__.__name__) + + # Undocumented keyword argument `_new` is used by `makeThreadAware()` + # to prevent a new file for the current thread from being created. + newFile = kwargs.pop('_new', True) + + # Blocking timeout. Not a `file` keyword argument; remove. + self.timeout = kwargs.pop('timeout', 60.0) + + self.initArgs = args + self.initKwargs = kwargs + + self._ready = Event() # NOT a lock; some things block, others wait + self._ready.set() + + self.threads = {} + + if newFile is True: + # Getting the stream for the thread will open the file. + self.getThreadStream() + + # For repr() on files closed by a thread. + self._mode = mode + + + def __repr__(self): + # Format the object's ID appropriately for the architecture (32b/64b) + if '32' in platform.architecture()[0]: + fmt = "<%s %s %r, mode %r at 0x%08X>" + else: + fmt = "<%s %s %r, mode %r at 0x%016X>" + + return fmt % ("closed" if self.closed else "open", + self.__class__.__name__, + self.initArgs[0], + self._mode, + id(self)) + + + @classmethod + def makeThreadAware(cls, fileStream): + """ Create a new `ThreadAwareFile` from an already-open file. If the + object is a `ThreadAwareFile`, it is returned verbatim. + """ + if isinstance(fileStream, cls): + return fileStream + elif not isinstance(fileStream, io.IOBase): + raise TypeError("Not a file: %r" % fileStream) + + f = cls(fileStream.name, fileStream.mode, _new=False) + f.threads[currentThread().ident] = fileStream + return f + + + def getThreadStream(self): + """ Get (or create) the file stream for the current thread. + """ + self._ready.wait(self.timeout) + + ident = currentThread().ident + if ident not in self.threads: + # First access from this thread. Open the file. + fp = io.FileIO(*self.initArgs, **self.initKwargs) + self.threads[ident] = fp + return fp + return self.threads[ident] + + + def closeAll(self): + """ Close all open streams. + + Warning: May not be thread-safe in some situations! + """ + try: + self._ready.wait(self.timeout) + self._ready.clear() + for v in list(self.threads.values()): + v.close() + finally: + self._ready.set() + + + def cleanup(self): + """ Delete all closed streams. + """ + try: + self._ready.wait(self.timeout) + self._ready.clear() + + for i in self.threads.keys(): + if self.threads[i].closed: + del self.threads[i] + finally: + self._ready.set() + + + @property + def closed(self): + """ Is the file not open? Note: A thread that never accessed the file + will get `True`. + """ + ident = currentThread().ident + if ident in self.threads: + return self.threads[ident].closed + return True + + + def close(self, *args, **kwargs): + """ Close the file for the current thread. The file will remain + open for other threads. + """ + result = self.getThreadStream().close(*args, **kwargs) + self.cleanup() + return result + + + # Standard file methods, overridden + + def __format__(self, *args, **kwargs): + return self.getThreadStream().__format__(*args, **kwargs) + + def __hash__(self, *args, **kwargs): + return self.getThreadStream().__hash__(*args, **kwargs) + + def __iter__(self, *args, **kwargs): + return self.getThreadStream().__iter__(*args, **kwargs) + + def __reduce__(self, *args, **kwargs): + return self.getThreadStream().__reduce__(*args, **kwargs) + + def __reduce_ex__(self, *args, **kwargs): + return self.getThreadStream().__reduce_ex__(*args, **kwargs) + + def __sizeof__(self, *args, **kwargs): + return self.getThreadStream().__sizeof__(*args, **kwargs) + + def __str__(self, *args, **kwargs): + return self.getThreadStream().__str__(*args, **kwargs) + + def fileno(self, *args, **kwargs): + return self.getThreadStream().fileno(*args, **kwargs) + + def flush(self, *args, **kwargs): + return self.getThreadStream().flush(*args, **kwargs) + + def isatty(self, *args, **kwargs): + return self.getThreadStream().isatty(*args, **kwargs) + + def next(self, *args, **kwargs): + return self.getThreadStream().next(*args, **kwargs) + + def read(self, *args, **kwargs): + return self.getThreadStream().read(*args, **kwargs) + + def readinto(self, *args, **kwargs): + return self.getThreadStream().readinto(*args, **kwargs) + + def readline(self, *args, **kwargs): + return self.getThreadStream().readline(*args, **kwargs) + + def readlines(self, *args, **kwargs): + return self.getThreadStream().readlines(*args, **kwargs) + + def seek(self, *args, **kwargs): + return self.getThreadStream().seek(*args, **kwargs) + + def tell(self, *args, **kwargs): + return self.getThreadStream().tell(*args, **kwargs) + + def truncate(self, *args, **kwargs): + raise IOError("Can't truncate(); %s is read-only" % + self.__class__.__name__) + + def write(self, *args, **kwargs): + raise IOError("Can't write(); %s is read-only" % + self.__class__.__name__) + + def writelines(self, *args, **kwargs): + raise IOError("Can't writelines(); %s is read-only" % + self.__class__.__name__) + + def xreadlines(self, *args, **kwargs): + return self.getThreadStream().xreadlines(*args, **kwargs) + + def __enter__(self, *args, **kwargs): + return self.getThreadStream().__enter__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + return self.getThreadStream().__exit__(*args, **kwargs) + + + # Standard file attributes, as properties for transparency with 'real' + # file objects. Most are read-only. + + @property + def encoding(self): + return self.getThreadStream().encoding + + @property + def errors(self): + return self.getThreadStream().errors + + @property + def mode(self): + return self.getThreadStream().mode + + @property + def name(self): + return self.getThreadStream().name + + @property + def newlines(self): + return self.getThreadStream().newlines + + @property + def softspace(self): + return self.getThreadStream().softspace + + @softspace.setter + def softspace(self, val): + self.getThreadStream().softspace = val diff --git a/lambda/cctv-people-rekognition/ebmlite/tools/__init__.py b/lambda/cctv-people-rekognition/ebmlite/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lambda/cctv-people-rekognition/ebmlite/tools/ebml2xml.py b/lambda/cctv-people-rekognition/ebmlite/tools/ebml2xml.py new file mode 100644 index 0000000..5e3ca50 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/tools/ebml2xml.py @@ -0,0 +1,76 @@ +import argparse +from xml.dom.minidom import parseString +from xml.etree import ElementTree as ET + +from ebmlite.tools import utils +import ebmlite.util +import ebmlite.xml_codecs + + +def main(): + # Build help text listing the binary codecs, and get the default one. + codecs = list(ebmlite.xml_codecs.BINARY_CODECS) + default_codec = codecs[0] + codec_desc = "" + for name, codec in ebmlite.xml_codecs.BINARY_CODECS.items(): + name = '"{}"'.format(name) + if codec.NAME == default_codec: + name += ' (default)'.format(name) + codec_desc += '{}: {}\n'.format(name, " ".join(codec.__doc__.split())) + + argparser = argparse.ArgumentParser( + description="A tool for converting ebml to xml." + ) + argparser.add_argument( + 'input', metavar="FILE.ebml", help="The source EBML file.", + ) + argparser.add_argument( + 'schema', + metavar="SCHEMA.xml", + help=( + "The name of the schema file. Only the name itself is required if" + " the schema file is in the standard schema directory." + ), + ) + argparser.add_argument( + '-o', '--output', metavar="FILE.xml", help="The output file.", + ) + argparser.add_argument( + '-c', '--clobber', action="store_true", + help="Clobber (overwrite) existing files.", + ) + argparser.add_argument( + '-s', '--single', action="store_true", help="Generate XML as a single line with no newlines or indents", + ) + argparser.add_argument( + '-m', '--max', + action="store_true", + help="Generate XML with maximum description, including offset, size, type, and id info", + ) + argparser.add_argument( + '-e', '--encoding', + choices=codecs, + default=default_codec, + help="The method of encoding binary data as text.\n" + codec_desc + ) + + args = argparser.parse_args() + + codecargs = {'cols': None} if args.single else {} + codec = ebmlite.xml_codecs.BINARY_CODECS[args.encoding.strip().lower()](**codecargs) + + with utils.load_files(args, binary_output=args.single) as (schema, out): + doc = schema.load(args.input, headers=True) + if args.max: + root = ebmlite.util.toXml(doc, offsets=True, sizes=True, types=True, ids=True, binary_codec=codec) + else: + root = ebmlite.util.toXml(doc, offsets=False, sizes=False, types=False, ids=False, binary_codec=codec) + s = ET.tostring(root, encoding="utf-8") + if args.single: + out.write(s) + else: + parseString(s).writexml(out, addindent='\t', newl='\n', encoding='utf-8') + + +if __name__ == "__main__": + main() diff --git a/lambda/cctv-people-rekognition/ebmlite/tools/list_schemata.py b/lambda/cctv-people-rekognition/ebmlite/tools/list_schemata.py new file mode 100644 index 0000000..3aef585 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/tools/list_schemata.py @@ -0,0 +1,37 @@ +""" +A tool for listing all EBML schemata in SCHEMA_PATH, including paths in the +EBMLITE_SCHEMA_PATH (if present), and (optionally) any additional paths +specified by the user. Additional paths may include module names enclosed in +braces (e.g., "{idelib}"). +""" + +import argparse +import sys + +import ebmlite.util +import ebmlite.core + + +def main(): + argparser = argparse.ArgumentParser(description=__doc__.strip()) + + argparser.add_argument( + '-o', '--output', metavar="FILE.txt", help="An optional output file", + default=sys.stdout + ) + argparser.add_argument( + '-r', '--relative', action="store_true", + help="Show schema filenames with package-relative path references", + ) + argparser.add_argument( + 'paths', nargs='*', + help="Additional paths to search for schemata; will be searched before paths in SCHEMA_PATH" + ) + + args = argparser.parse_args() + ebmlite.util.printSchemata(paths=args.paths, out=args.output, absolute=not args.relative) + + +if __name__ == "__main__": + main() + diff --git a/lambda/cctv-people-rekognition/ebmlite/tools/utils.py b/lambda/cctv-people-rekognition/ebmlite/tools/utils.py new file mode 100644 index 0000000..136412a --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/tools/utils.py @@ -0,0 +1,36 @@ +import contextlib +import sys +import os.path + +from ebmlite import core + + +def errPrint(msg): + sys.stderr.write("%s\n" % msg) + sys.stderr.flush() + exit(1) + + +@contextlib.contextmanager +def load_files(args, binary_output=False): + if not os.path.exists(args.input): + sys.stderr.write("Input file does not exist: %s\n" % args.input) + exit(1) + + try: + schema_file = args.schema + if os.path.splitext(schema_file.strip())[1] == '': + schema_file += '.xml' + schema = core.loadSchema(schema_file) + except IOError as err: + errPrint("Error loading schema: %s\n" % err) + + if not args.output: + yield (schema, sys.stdout) + return + + output = os.path.realpath(os.path.expanduser(args.output)) + if os.path.exists(output) and not args.clobber: + errPrint("Error: Output file already exists: %s" % args.output) + with open(output, ('wb' if binary_output else 'w')) as out: + yield (schema, out) diff --git a/lambda/cctv-people-rekognition/ebmlite/tools/view_ebml.py b/lambda/cctv-people-rekognition/ebmlite/tools/view_ebml.py new file mode 100644 index 0000000..4123201 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/tools/view_ebml.py @@ -0,0 +1,57 @@ +import argparse + +from ebmlite.tools import utils +import ebmlite.util +import ebmlite.xml_codecs + + +def main(): + # Build help text listing the binary codecs, and get the default one. + codecs = list(ebmlite.xml_codecs.BINARY_CODECS) + default_codec = "ignore" + codec_desc = "" + for name, codec in ebmlite.xml_codecs.BINARY_CODECS.items(): + name = '"{}"'.format(name) + if codec.NAME == default_codec: + name += ' (default)'.format(name) + codec_desc += '{}: {}\n'.format(name, " ".join(codec.__doc__.split())) + + argparser = argparse.ArgumentParser( + description="A tool for reading ebml file content." + ) + argparser.add_argument( + 'input', metavar="FILE.ebml", help="The source XML file.", + ) + argparser.add_argument( + 'schema', + metavar="SCHEMA.xml", + help=( + "The name of the schema file. Only the name itself is required if" + " the schema file is in the standard schema directory." + ), + ) + argparser.add_argument( + '-o', '--output', metavar="FILE.xml", help="The output file.", + ) + argparser.add_argument( + '-c', '--clobber', action="store_true", + help="Clobber (overwrite) existing files.", + ) + argparser.add_argument( + '-e', '--encoding', + choices=codecs, + default=default_codec, + help="The method of encoding binary data as text.\n" + codec_desc + ) + + args = argparser.parse_args() + + codec = ebmlite.xml_codecs.BINARY_CODECS[args.encoding.strip().lower()]() + + with utils.load_files(args, binary_output=False) as (schema, out): + doc = schema.load(args.input, headers=True) + ebmlite.util.pprint(doc, out=out, binary_codec=codec) + + +if __name__ == "__main__": + main() diff --git a/lambda/cctv-people-rekognition/ebmlite/tools/xml2ebml.py b/lambda/cctv-people-rekognition/ebmlite/tools/xml2ebml.py new file mode 100644 index 0000000..c5f3fed --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/tools/xml2ebml.py @@ -0,0 +1,36 @@ +import argparse + +from ebmlite.tools import utils +import ebmlite.util + + +def main(): + argparser = argparse.ArgumentParser( + description="A tool for converting xml to ebml." + ) + argparser.add_argument( + 'input', metavar="FILE.xml", help="The source XML file.", + ) + argparser.add_argument( + 'schema', + metavar="SCHEMA.xml", + help=( + "The name of the schema file. Only the name itself is required if" + " the schema file is in the standard schema directory." + ), + ) + argparser.add_argument( + '-o', '--output', metavar="FILE.ebml", help="The output file.", + ) + argparser.add_argument( + '-c', '--clobber', action="store_true", + help="Clobber (overwrite) existing files.", + ) + args = argparser.parse_args() + + with utils.load_files(args, binary_output=True) as (schema, out): + ebmlite.util.xml2ebml(args.input, out, schema) # , sizeLength=4, headers=True, unknown=True) + + +if __name__ == "__main__": + main() diff --git a/lambda/cctv-people-rekognition/ebmlite/util.py b/lambda/cctv-people-rekognition/ebmlite/util.py new file mode 100644 index 0000000..6fe77da --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/util.py @@ -0,0 +1,475 @@ +""" +Some utilities for manipulating EBML documents: translate to/from XML, etc. +This module may be imported or used as a command-line utility. + +Created on Aug 11, 2017 + +@todo: Clean up and standardize usage of the term 'size' versus 'length.' +@todo: Modify (or create an alternate version of) `toXml()` that writes + directly to a file, allowing the conversion of huge EBML files. +@todo: Add other options to command-line utility for the other arguments of + `toXml()` and `xml2ebml()`. +""" +__author__ = "David Randall Stokes, Connor Flanigan" +__copyright__ = "Copyright 2021, Mide Technology Corporation" +__credits__ = "David Randall Stokes, Connor Flanigan, Becker Awqatty, Derek Witt" + +__all__ = ['createID', 'validateID', 'toXml', 'xml2ebml', 'loadXml', 'pprint', + 'printSchemata'] + +import ast +from base64 import b64encode, b64decode +from io import StringIO +import pathlib +import struct +import sys +import tempfile +from xml.etree import ElementTree as ET + +from . import core, encoding, decoding +from . import xml_codecs + +# ============================================================================== +# +# ============================================================================== + + +def createID(schema, idClass, exclude=(), minId=0x81, maxId=0x1FFFFFFE, count=1): + """ Generate unique EBML IDs. Primarily intended for use 'offline' by + humans creating EBML schemata. + + @param schema: The `Schema` in which the new IDs must coexist. + @param idClass: The EBML class of ID, one of (case-insensitive): + * `'a'`: Class A (1 octet, base 0x8X) + * `'b'`: Class B (2 octets, base 0x4000) + * `'c'`: Class C (3 octets, base 0x200000) + * `'d'`: Class D (4 octets, base 0x10000000) + @param exclude: A list of additional IDs to avoid. + @param minId: The minimum ID value, within the ID class' range. + @param maxId: The maximum ID value, within the ID class' range. + @param count: The maximum number of IDs to generate. The result may be + fewer than specified if too few meet the given criteria. + @return: A list of EBML IDs that match the given criteria. + """ + ranges = dict(A=(0x81, 0xFE), + B=(0x407F, 0x7FFE), + C=(0x203FFF, 0x3FFFFE), + D=(0x101FFFFF, 0x1FFFFFFE)) + idc = idClass.upper() + if idc not in ranges: + raise KeyError('Invalid ID class %r: must be one of %r' % + (idClass, list(ranges))) + + # Keep range within the one specified and the one imposed by the ID class + idrange = (max(ranges[idc][0], minId), + min(ranges[idc][1], maxId)) + + exclude = set(exclude).union(schema.elements.keys()) + + result = [] + for i in (x for x in range(*idrange) if x not in exclude): + if len(result) == count: + break + result.append(i) + + return result + + +def validateID(elementId): + """ Verify that a number is a valid EBML element ID. A `ValueError` + will be raised if the element ID is invalid. + + Valid ranges for the four classes of EBML ID are: + * A: 0x81 to 0xFE + * B: 0x407F to 0x7FFE + * C: 0x203FFF to 0x3FFFFE + * D: 0x101FFFFF to 0x1FFFFFFE + + @param elementId: The element ID to validate + @raises: `ValueError`, although certain edge cases may raise + another type. + """ + ranges = ((0x81, 0xFE), (0x407F, 0x7FFE), (0x203FFF, 0x3FFFFE), (0x101FFFFF, 0x1FFFFFFE)) + + msg = "Invalid element ID" # Default error message + + # Basic check: is the ID within the bounds of the total ID range? + if not 0x81 <= elementId <= 0x1FFFFFFE: + raise ValueError("Element ID out of range", elementId) + + try: + # See if the first byte properly encodes the length of the ID. + s = struct.pack(">I", elementId).lstrip(b'\x00') + length, _ = decoding.decodeIDLength(s[0]) + valid = len(s) == length # Should always be True if decoding worked + if valid: + minId, maxId = ranges[length-1] + if not minId <= elementId <= maxId: + msg = "ID out of range for class %s %s" % (" ABCD"[length], ranges[length-1]) + valid = False + + # Note: Change this if decoding changes the exceptions it raises + except OSError as err: + valid = False + msg = err.args[0] if err.args else msg + + if not valid: + raise ValueError(msg, elementId) + + return True + +# ============================================================================== +# +# ============================================================================== + + +def toXml(el, parent=None, offsets=True, sizes=True, types=True, ids=True, + binary_codec='base64', void_codec='ignore'): + """ Convert an EBML Document to XML. Binary elements will contain + base64-encoded data in their body. Other non-master elements will + contain their value in a ``value`` attribute. + + @param el: An instance of an EBML Element or Document subclass. + @keyword parent: The resulting XML element's parent element, if any. + @keyword offsets: If `True`, create a ``offset`` attributes for each + generated XML element, containing the corresponding EBML element's + offset. + @keyword sizes: If `True`, create ``size`` attributes containing the + corresponding EBML element's size. + @keyword types: If `True`, create ``type`` attributes containing the + name of the corresponding EBML element type. + @keyword ids: If `True`, create ``id`` attributes containing the + corresponding EBML element's EBML ID. + @keyword binary_codec: The name of an XML codec class from + `ebmlite.xml_codecs`, or an instance of a codec, for rendering + binary elements as text. + @keyword void_codec: The name of an XML codec class from + `ebmlite.xml_codecs`, or an instance of a codec, for rendering + the contents of Void elements as text. + @return The root XML element of the file. + """ + if isinstance(binary_codec, str): + binary_codec = xml_codecs.BINARY_CODECS[binary_codec]() + if isinstance(void_codec, str): + void_codec = xml_codecs.BINARY_CODECS[void_codec]() + + if isinstance(el, core.Document): + elname = el.__class__.__name__ + else: + elname = el.name + + if parent is None: + xmlEl = ET.Element(elname) + else: + xmlEl = ET.SubElement(parent, elname) + if isinstance(el, core.Document): + xmlEl.set('source', el.filename) + xmlEl.set('schemaName', el.schema.name) + xmlEl.set('schemaFile', el.schema.filename) + else: + if ids and isinstance(el.id, int): + xmlEl.set('id', "0x%X" % el.id) + if types: + xmlEl.set('type', el.dtype.__name__) + + if offsets: + xmlEl.set('offset', str(el.offset)) + if sizes: + xmlEl.set('size', str(el.size)) + + if isinstance(el, core.MasterElement): + for chEl in el: + toXml(chEl, xmlEl, offsets, sizes, types, ids, binary_codec, void_codec) + elif isinstance(el, core.VoidElement): + xmlEl.set('size', str(el.size)) + if void_codec.NAME != 'ignore': + xmlEl.set('encoding', void_codec.NAME) + xmlEl.text = void_codec.encode(el.value) + elif isinstance(el, core.BinaryElement): + xmlEl.set('encoding', binary_codec.NAME) + xmlEl.text = binary_codec.encode(el.value, offset=el.offset) + elif not isinstance(el, core.VoidElement): + xmlEl.set('value', str(el.value).encode('ascii', 'xmlcharrefreplace').decode()) + + return xmlEl + + +#=============================================================================== +# +#=============================================================================== + +def xmlElement2ebml(xmlEl, ebmlFile, schema, sizeLength=None, unknown=True): + """ Convert an XML element to EBML, recursing if necessary. For converting + an entire XML document, use `xml2ebml()`. + + @param xmlEl: The XML element. Its tag must match an element defined + in the `schema`. + @param ebmlFile: An open file-like stream, to which the EBML data will + be written. + @param schema: An `ebmlite.core.Schema` instance to use when + writing the EBML document. + @keyword sizeLength: + @param unknown: If `True`, unknown element names will be allowed, + provided their XML elements include an ``id`` attribute with the + EBML ID (in hexadecimal). + @return The length of the encoded element, including header and children. + @raise NameError: raised if an xml element is not present in the schema and unknown is False, OR if the xml + element does not have an ID. + """ + if not isinstance(xmlEl.tag, (str, bytes, bytearray)): + # (Probably) a comment; disregard. + return 0 + + try: + cls = schema[xmlEl.tag] + encId = encoding.encodeId(cls.id) + except (KeyError, AttributeError): + # Element name not in schema. Go ahead if allowed (`unknown` is `True`) + # and the XML element specifies an ID, + if not unknown: + raise NameError("Unrecognized EBML element name: %s" % xmlEl.tag) + + eid = xmlEl.get('id', None) + if eid is None: + raise NameError("Unrecognized EBML element name with no 'id' " + "attribute in XML: %s" % xmlEl.tag) + cls = core.UnknownElement + encId = encoding.encodeId(int(eid, 16)) + cls.id = int(eid, 16) + + codec = xmlEl.get('encoding', 'base64') + + if sizeLength is None: + sl = xmlEl.get('sizeLength', None) + if sl is None: + s = xmlEl.get('size', None) + if s is not None: + sl = encoding.getLength(int(s)) + else: + sl = 4 + else: + sl = int(sl) + else: + sl = xmlEl.get('sizeLength', sizeLength) + + if issubclass(cls, core.MasterElement): + ebmlFile.write(encId) + sizePos = ebmlFile.tell() + ebmlFile.write(encoding.encodeSize(None, sl)) + size = 0 + for chEl in xmlEl: + size += xmlElement2ebml(chEl, ebmlFile, schema, sl) + endPos = ebmlFile.tell() + ebmlFile.seek(sizePos) + ebmlFile.write(encoding.encodeSize(size, sl)) + ebmlFile.seek(endPos) + return len(encId) + (endPos - sizePos) + + elif issubclass(cls, core.BinaryElement): + val = xml_codecs.BINARY_CODECS[codec].decode(xmlEl.text) + elif issubclass(cls, (core.IntegerElement, core.FloatElement)): + val = ast.literal_eval(xmlEl.get('value')) + else: + val = cls.dtype(xmlEl.get('value')) + + size = xmlEl.get('size', None) + if size is not None: + size = int(size) + sl = xmlEl.get('sizeLength') + if sl is not None: + sl = int(sl) + + encoded = cls.encode(val, size, lengthSize=sl) + ebmlFile.write(encoded) + return len(encoded) + + +def xml2ebml(xmlFile, ebmlFile, schema, sizeLength=None, headers=True, + unknown=True): + """ Convert an XML file to EBML. + + @todo: Convert XML on the fly, rather than parsing it first, allowing + for the conversion of arbitrarily huge files. + + @param xmlFile: The XML source. Can be a filename, an open file-like + stream, or a parsed XML document. + @param ebmlFile: The EBML file to write. Can be a filename or an open + file-like stream. + @param schema: The EBML schema to use. Can be a filename or an + instance of a `Schema`. + @keyword sizeLength: The default length of each element's size + descriptor. Must be large enough to store the largest 'master' + element. If an XML element has a ``sizeLength`` attribute, it will + override this. + @keyword headers: If `True`, generate the standard ``EBML`` EBML + element if the XML document does not contain one. + @param unknown: If `True`, unknown element names will be allowed, + provided their XML elements include an ``id`` attribute with the + EBML ID (in hexadecimal). + @return: the size of the ebml file in bytes. + @raise NameError: raises if an xml element is not present in the schema. + """ + if isinstance(ebmlFile, (str, bytes, bytearray)): + ebmlFile = open(ebmlFile, 'wb') + openedEbml = True + else: + openedEbml = False + + if not isinstance(schema, core.Schema): + schema = core.loadSchema(schema) + + if isinstance(xmlFile, ET.Element): + # Already a parsed XML element + xmlRoot = xmlFile + elif isinstance(xmlFile, ET.ElementTree): + # Already a parsed XML document + xmlRoot = xmlFile.getroot() + else: + xmlDoc = ET.parse(xmlFile) + xmlRoot = xmlDoc.getroot() + + if xmlRoot.tag not in schema and xmlRoot.tag != schema.document.__name__: + raise NameError("XML element %s not an element or document in " + "schema %s (wrong schema)" % (xmlRoot.tag, schema.name)) + + headers = headers and 'EBML' in schema + if headers and 'EBML' not in (el.tag for el in xmlRoot): + pos = ebmlFile.tell() + cls = schema.document + ebmlFile.write(cls.encodePayload(cls._createHeaders())) + numBytes = ebmlFile.tell() - pos + else: + numBytes = 0 + + if xmlRoot.tag == schema.document.__name__: + for el in xmlRoot: + numBytes += xmlElement2ebml(el, ebmlFile, schema, sizeLength, + unknown=unknown) + else: + numBytes += xmlElement2ebml(xmlRoot, ebmlFile, schema, sizeLength, + unknown=unknown) + + if openedEbml: + ebmlFile.close() + + return numBytes + +#=============================================================================== +# +#=============================================================================== + + +def loadXml(xmlFile, schema, ebmlFile=None): + """ Helpful utility to load an EBML document from an XML file. + + @param xmlFile: The XML source. Can be a filename, an open file-like + stream, or a parsed XML document. + @param schema: The EBML schema to use. Can be a filename or an + instance of a `Schema`. + @keyword ebmlFile: The name of the temporary EBML file to write, or + ``:memory:`` to use RAM (like `sqlite3`). Defaults to an + automatically-generated temporary file. + @return The root node of the specified EBML file. + """ + if ebmlFile == ":memory:": + ebmlFile = StringIO() + xml2ebml(xmlFile, ebmlFile, schema) + ebmlFile.seek(0) + else: + ebmlFile = tempfile.mktemp() if ebmlFile is None else ebmlFile + xml2ebml(xmlFile, ebmlFile, schema) + + return schema.load(ebmlFile) + + +#=============================================================================== +# +#=============================================================================== + +def pprint(el, values=True, out=sys.stdout, indent=" ", binary_codec="ignore", + void_codec="ignore", _depth=0): + """ Test function to recursively crawl an EBML document or element and + print its structure, with child elements shown indented. + + @param el: An instance of a `Document` or `Element` subclass. + @keyword values: If `True`, show elements' values. + @keyword out: A file-like stream to which to write. + @keyword indent: The string containing the character(s) used for each + indentation. + @keyword binary_codec: The name of a class from `ebmlite.xml_codecs`, + or an instance of a codec, for rendering binary elements as text. + @keyword void_codec: The name of a class from `ebmlite.xml_codecs`, + or an instance of a codec, for rendering the contents of Void + elements as text. + """ + tab = indent * _depth + + if isinstance(binary_codec, str): + binary_codec = xml_codecs.BINARY_CODECS[binary_codec]() + if isinstance(void_codec, str): + void_codec = xml_codecs.BINARY_CODECS[void_codec]() + + if _depth == 0: + if values: + out.write("Offset Size Element (ID): Value\n") + else: + out.write("Offset Size Element (ID)\n") + out.write("====== ====== =================================\n") + + if isinstance(el, core.Document): + out.write("%06s %06s %s %s (Document, type %s)\n" % (el.offset, el.size, tab, el.name, el.type)) + for i in el: + pprint(i, values, out, indent, binary_codec, void_codec, _depth+1) + else: + out.write("%06s %06s %s %s (ID 0x%0X)" % (el.offset, el.size, tab, el.name, el.id)) + if isinstance(el, core.MasterElement): + out.write(": (master) %d subelements\n" % len(el.value)) + for i in el: + pprint(i, values, out, indent, binary_codec, void_codec, _depth+1) + else: + out.write(": (%s)" % el.dtype.__name__) + if values: + if isinstance(el, core.BinaryElement): + indent = tab + " " * 17 + if isinstance(el, core.VoidElement) and void_codec.NAME != 'ignore': + out.write(" <{}>".format(void_codec.NAME)) + void_codec.encode(el.value, offset=el.offset, indent=indent, stream=out) + elif binary_codec.NAME != 'ignore': + out.write(" <{}>".format(binary_codec.NAME)) + binary_codec.encode(el.value, offset=el.offset, indent=indent, stream=out) + else: + out.write(" %r" % (el.value)) + out.write("\n") + + out.flush() + + +#=============================================================================== +# +#=============================================================================== + +def printSchemata(paths=None, out=sys.stdout, absolute=True): + """ Display a list of schemata in `SCHEMA_PATH`. A thin wrapper for the + core `listSchemata()` function. + + @param out: A file-like stream to which to write. + """ + out = out or sys.stdout + newfile = isinstance(out, (str, pathlib.Path)) + if newfile: + out = open(out, 'w') + + try: + if paths: + paths.extend(core.SCHEMA_PATH) + else: + paths = core.SCHEMA_PATH + schemata = core.listSchemata(*paths, absolute=absolute) + for k, v in schemata.items(): + out.write("{}\n".format(k)) + for s in v: + out.write(" {}\n".format(s)) + out.flush() + finally: + if newfile: + out.close() diff --git a/lambda/cctv-people-rekognition/ebmlite/xml_codecs.py b/lambda/cctv-people-rekognition/ebmlite/xml_codecs.py new file mode 100644 index 0000000..225bc28 --- /dev/null +++ b/lambda/cctv-people-rekognition/ebmlite/xml_codecs.py @@ -0,0 +1,305 @@ +""" +Classes for various means of encoding/decoding binary data to/from XML. + +Note: the class docstrings will be shown in the `ebml2xml` help text. +""" + +import base64 +from io import BytesIO, StringIO + + +# ============================================================================== +# +# ============================================================================== + +class BinaryCodec: + """ Base class for binary encoders/decoders, rendering and reading + `BinaryElement` contents as text. + + :cvar NAME: The codec's name, written to the rendered XML as + the `encoding` attribute. Also used as the `--encoding` + argument in the command-line tools. Must be unique, and + should be lowercase. + :type NAME: str + """ + NAME = "" + + def __init__(self, **kwargs): + """ Constructor. All arguments should be optional keyword + arguments. Can be considered optional in subclasses. + """ + pass + + def encode(self, data, stream=None, indent='', offset=0, **kwargs): + """ Convert binary data to text. Typical arguments: + + :param data: The binary data from an EBML `BinaryElement`. + :param stream: An optional stream to which to write the encoded + data. Should be included and used in all implementations. + :param indent: Indentation before each row of text. Used if + the codec was instantiated with `cols` specified. + :param offset: The originating EBML element's offset in the file. + For use with codecs that write line numbers/position info. + :returns: If no `stream`, the encoded data as text. If `stream`, + the number of bytes written. + """ + raise NotImplementedError + + @classmethod + def decode(cls, data, stream=None): + """ Decode binary data in text form (e.g., from an XML file). Note: + this is a `classmethod`, and should work regardless of the + arguments used when the data was encoded (e.g., with or without + indentations and/or line breaks, metadata like offsets, etc.). + + :param data: The text data from an XML file. + :param stream: A stream to which to write the encoded data. + :returns: If no `stream`, the decoded binary data. If `stream`, + the number of bytes written. + """ + raise NotImplementedError + + +# ============================================================================== +# +# ============================================================================== + +class Base64Codec(BinaryCodec): + """ Encoder/decoder for binary data as base64 formatted text to/from text. + """ + NAME = "base64" + + def __init__(self, cols=76, **kwargs): + """ Constructor. + + :param cols: The length of each line of base64 data, excluding + any indentation specified when encoding. If 0 or `None`, + data will be written as a single continuous block with no + newlines. + + Additional keyword arguments will be accepted (to maintain + compatibility with other codecs) but ignored. + """ + self.cols = cols + + + def encode(self, data, stream=None, indent='', **kwargs): + """ Convert binary data to base64 text. + + :param data: The binary data from an EBML `BinaryElement`. + :param stream: An optional stream to which to write the encoded + data. + :param indent: Indentation before each row of text. Used if + the codec was instantiated with `cols` specified. + :returns: If no `stream`, the encoded data as text. If `stream`, + the number of bytes written. + + Additional keyword arguments will be accepted (to maintain + compatibility with other codecs) but ignored. + """ + if isinstance(indent, bytes): + indent = indent.decode() + if isinstance(data, str): + data = data.encode('utf8') + + result = base64.encodebytes(data).decode() + if stream is None: + out = StringIO() + else: + out = stream + + if self.cols == 76: + # Default width of a base64 line; use existing newlines + result = "\n" + result + if indent: + result = result.replace('\n', '\n' + indent) + if stream is not None: + return out.write(result) + return result + + result = result.replace('\n', '') + + if self.cols is None: + if stream is not None: + return out.write(result) + return result + + numbytes = 0 + for chunk in range(0, len(result), self.cols): + numbytes += out.write('\n') + numbytes += out.write(indent) + out.write(result[chunk:chunk+self.cols]) + + if stream is None: + return out.getvalue() + + return numbytes + + + @classmethod + def decode(cls, data, stream=None): + """ Decode binary data in base64 (e.g., from an XML file). Note: this + is a `classmethod`, and works regardles of how the encoded data was + formatted (e.g., with indentations and/or line breaks). + + :param data: The base64 data from an XML file. + :param stream: A stream to which to write the encoded data. + :returns: If no `stream`, the decoded binary data. If `stream`, + the number of bytes written. + """ + if not data: + if stream is None: + return b'' + else: + return 0 + + if isinstance(data, str): + data = data.encode('utf8') + + result = base64.decodebytes(data) + + if stream is not None: + return stream.write(result) + else: + return result + + +# ============================================================================== +# +# ============================================================================== + +class HexCodec(BinaryCodec): + """ Encoder/decoder for binary data as hexadecimal format to/from text. + Encoded text is multiple columns of bytes/words (default is 16 columns, + 2 bytes per column), with an optional file offset at the start of each + row. + """ + # The name shown in the encoded XML element's `encoding` attribute + NAME = "hex" + + def __init__(self, width=2, cols=32, offsets=True, **kwargs): + """ Constructor. + + :param width: The number of bytes displayed per column when + encoding. + :param cols: The number of columns to display when encoding. If 0 + or `None`, data will be written as a single continuous block + with no newlines. + :param offsets: If `True`, each line will start with its offset + (in decimal). Applicable if `cols` is a non-zero number. + """ + self.width = width + self.cols = cols + self.offsets = bool(offsets and cols) + + + def encode(self, data, stream=None, offset=0, indent='', **kwargs): + """ Convert binary data to hexadecimal text. + + :param data: The binary data from an EBML `BinaryElement`. + :param stream: An optional stream to which to write the encoded + data. + :param offset: A starting number for the displayed offsets column. + For showing the data's offset in an EBML file. + :param indent: Indentation before each row of hex text. + :returns: If no `stream`, the encoded data as text. If `stream`, + the number of bytes written. + """ + if not isinstance(indent, str): + indent = indent.decode() + + if stream is None: + out = StringIO() + else: + out = stream + + newline = bool(self.cols) + offsets = self.offsets and newline + + numbytes = 0 + for i, b in enumerate(data): + if newline and not i % self.cols: + numbytes += out.write('\n') + numbytes += out.write(indent) + if offsets: + numbytes += out.write('[{:06d}] '.format(i + offset)) + elif not i % self.width: + numbytes += out.write(' ') + numbytes += out.write('{:02x}'.format(b)) + + if stream is None: + return out.getvalue() + + return numbytes + + + @classmethod + def decode(cls, data, stream=None): + """ Decode binary data in hexadecimal (e.g., from an XML file). Note: + this is a `classmethod`, and works regardles of how the encoded + data was formatted (e.g., number of columns, with or without + offsets, etc.). + + :param data: The base64 data from an XML file. + :param stream: A stream to which to write the encoded data. + :returns: If no `stream`, the decoded binary data. If `stream`, + the number of bytes written. + """ + if stream is None: + out = BytesIO() + else: + out = stream + numbytes = 0 + + if not data: + if stream is None: + return b'' + else: + return 0 + + if isinstance(data, str): + data = data.encode('utf8') + + for word in data.split(): + if b'[' in word or b']' in word: + continue + for i in range(0, len(word), 2): + numbytes += out.write((int(word[i:i+2], 16).to_bytes(1, 'big'))) + + if stream is None: + return out.getvalue() + + return numbytes + + +# ============================================================================== +# +# ============================================================================== + +class IgnoreCodec(BinaryCodec): + """ Suppresses writing binary data as text. + """ + NAME = "ignore" + + @staticmethod + def encode(data, stream=None, **kwargs): + if stream: + return 0 + return '' + + @staticmethod + def decode(data, stream=None, **kwargs): + if stream: + return 0 + return b'' + + +# ============================================================================== +# +# ============================================================================== + +# Collection of codecs. The first one will be the default in the CLI (or at least +# it will be in Python 3.7 and later). User-implemented codecs should be added to +# the dictionary. +BINARY_CODECS = {'base64': Base64Codec, + 'hex': HexCodec, + 'ignore': IgnoreCodec} diff --git a/lambda/cctv-people-rekognition/kinesis_video_fragment_processor.py b/lambda/cctv-people-rekognition/kinesis_video_fragment_processor.py new file mode 100644 index 0000000..100e3c9 --- /dev/null +++ b/lambda/cctv-people-rekognition/kinesis_video_fragment_processor.py @@ -0,0 +1,387 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0. + +''' +Amazon Kinesis Video Stream (KVS) Consumer Library for Python. + +This class provides post-processing fiunctions for a MKV fragement that has been parsed +by the Amazon Kinesis Video Streams Cosumer Library for Python. + + ''' + +__version__ = "0.0.1" +__status__ = "Development" +__copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved." +__author__ = "Dean Colcott " + +import io +import logging +import ebmlite.util as emblite_utils +import wave +import ebmlite.decoding as ebmlite_decoding + +# Init the logger. +log = logging.getLogger(__name__) + +class KvsFragementProcessor(): + + #################################################### + # Fragment processing functions + + def get_fragment_tags(self, fragment_dom): + ''' + Parses a MKV Fragment Doc (of type ebmlite.core.MatroskaDocument) that is returned to the provided callback + from get_streaming_fragments() in this class and returns a dict of the SimpleTag elements found. + + ### Parameters: + + **fragment_dom**: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + ### Returns: + + simple_tags: dict + + Dictionary of all SimpleTag elements with format - TagName : TagValue . + + ''' + + # Get the Segment Element of the Fragment DOM - error if not found + segment_element = None + for element in fragment_dom: + if (element.id == 0x18538067): # MKV Segment Element ID + segment_element = element + break + + if (not segment_element): + raise KeyError('Segment Element required but not found in fragment_doc' ) + + # Save all of the SimpleTag elements in the Segment element + simple_tag_elements = [] + for element in segment_element: + if (element.id == 0x1254C367): # Tags element type ID + for tags in element: + if (tags.id == 0x7373): # Tag element type ID + for tag_type in tags: + if (tag_type.id == 0x67C8 ): # SimpleTag element type ID + simple_tag_elements.append(tag_type) + + # For all SimpleTags types (ID: 0x67C8), save for TagName (ID: 0x7373) and values of TagString (ID:0x4487) or TagBinary (ID: 0x4485 ) + simple_tags_dict = {} + for simple_tag in simple_tag_elements: + + tag_name = None + tag_value = None + for element in simple_tag: + if (element.id == 0x45A3): # Tag Name element type ID + tag_name = element.value + elif (element.id == 0x4487 or element.id == 0x4485): # TagString and TagBinary element type IDs respectively + tag_value = element.value + + # As long as tag name was found add the Tag to the return dict. + if (tag_name): + simple_tags_dict[tag_name] = tag_value + + return simple_tags_dict + + def get_fragement_dom_pretty_string(self, fragment_dom): + ''' + Returns the Pretty Print parsing of the EBMLite fragment DOM as a string + + ### Parameters: + + **fragment_dom**: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + ### Return: + **pretty_print_str**: str + Pretty print string of the Fragment DOM object + ''' + + pretty_print_str = io.StringIO() + + emblite_utils.pprint(fragment_dom, out=pretty_print_str) + return pretty_print_str.getvalue() + + def save_fragment_as_local_mkv(self, fragment_bytes, file_name_path): + ''' + Save the provided fragment_bytes as stand-alone MKV file on local disk. + fragment_bytes as it arrives in is already a well formatted MKV fragment + so can just write the bytes straight to disk and it will be a playable MKV file. + + ### Parameters: + + fragment_bytes: bytearray + A ByteArray with raw bytes from exactly one fragment. + + file_name_path: Str + Local file path / name to save the MKV file to. + + ''' + + f = open(file_name_path, "wb") + f.write(fragment_bytes) + f.close() + + def get_frames_as_ndarray(self, fragment_bytes, one_in_frames_ratio): + ''' + Parses fragment_bytes and returns a ratio of available frames in the MKV fragment as + a list of numpy.ndarray's. + + e.g: Setting one_in_frames_ratio = 5 will return every 5th frame found in the fragment. + (Starting with the first) + + To return all available frames just set one_in_frames_ratio = 1 + + ### Parameters: + + fragment_bytes: bytearray + A ByteArray with raw bytes from exactly one fragment. + + one_in_frames_ratio: Str + Ratio of the available frames in the fragment to process and return. + + ### Return: + + frames: List + A list of frames extracted from the fragment as numpy.ndarray + + ''' + + # Parse all frames in the fragment to frames list + frames = iio.imread(io.BytesIO(fragment_bytes), plugin="pyav", index=...) + + # Store and return frames in frame ratio of total available + ret_frames = [] + for i in range(0, len(frames), one_in_frames_ratio): + ret_frames.append(frames[i]) + + return ret_frames + + def save_frames_as_jpeg(self, fragment_bytes, one_in_frames_ratio, jpg_file_base_path): + ''' + Parses fragment_bytes and saves a ratio of available frames in the MKV fragment as + JPEGs on the local disk. + + e.g: Setting one_in_frames_ratio = 5 will return every 5th frame found in the fragment + (starting with the first). + + To return all available frames just set one_in_frames_ratio = 1 + + ### Parameters: + + fragment_bytes: ByteArray + A ByteArray with raw bytes from exactly one fragment. + + one_in_frames_ratio: Str + Ratio of the available frames in the fragment to process and save. + + ### Return + jpeg_paths : List + A list of file paths to the saved JPEN files. + + ''' + + # Parse all frames in the fragment to frames list + ndarray_frames = self.get_frames_as_ndarray(fragment_bytes, one_in_frames_ratio) + + # Write frames to disk as JPEG images + jpeg_paths = [] + for i in range(len(ndarray_frames)): + frame = ndarray_frames[i] + image_file_path = '{}-{}.jpg'.format(jpg_file_base_path, i) + iio.imwrite(image_file_path, frame, format=None) + jpeg_paths.append(image_file_path) + + return jpeg_paths + + + def get_raw_audio_track_from_simple_block(self, mkv_element): + ''' + This function gets the raw audio track from a SimpleBlock element + in a Matroska file from Amazon Connect. + + It will remove SimpleBlock header as per: + https://github.com/ietf-wg-cellar/matroska-specification/blob/master/notes.md + + Will works only if track number VINT is one octet length. + + ### Parameters: + mkv_element: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + ### Return: + A bytearray containing the raw audio data of the specified track + ''' + + if mkv_element.name == "SimpleBlock": + mkv_element.stream.seek(mkv_element.payloadOffset+4) + return mkv_element.parse(mkv_element.stream, mkv_element.size-4) + return None + + def get_audio_track_number_from_simple_block(self, mkv_element): + ''' + This function gets the number of audio track from a SimpleBlock element + in a Matroska file from Amazon Connect. + + Will works only if track number VINT is one octet length as per: + https://github.com/ietf-wg-cellar/matroska-specification/blob/master/notes.md + + ### Parameters: + mkv_element: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + ### Return: + number of audio track in SimpleBlock + ''' + + if mkv_element.name == "SimpleBlock": + mkv_element.stream.seek(mkv_element.payloadOffset) + ch = mkv_element.stream.read(1) + length, _ = ebmlite_decoding.decodeIntLength(ord(ch)) + if length == 1: + ''' + removing VINT_MARKER as per https://datatracker.ietf.org/doc/rfc8794/ paragraph 4 + ''' + track_nr = ord(ch) & 127 + + return track_nr + return None + + + def get_track_bytearray(self, mkv_dom, track_nr): + ''' + This function extracts the raw audio track from a Matroska + file from Amazon Connect and returns it as a bytearray. It iterates through + the SimpleBlock elements within each Cluster, alternating which + track it appends based on the track number. + + ### Parameters: + mkv_dom: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + track_nr: The track number (1 or 2) to extract + + ### Return: + A bytearray containing the raw audio data of the specified track + + ''' + + track_bytearray = bytearray() + + for element in mkv_dom: + for segment_child in element: + if segment_child.name == "Cluster": + i=0 + for cluster_child in segment_child: + if cluster_child.name == "SimpleBlock": + simple_block_track_nr =self.get_audio_track_number_from_simple_block(cluster_child) + i+=1 + if track_nr == simple_block_track_nr: + track_bytearray.extend(self.get_raw_audio_track_from_simple_block(cluster_child)) + + return track_bytearray + + def get_track_number_by_name(self, fragment_dom, track_name): + ''' + This function gets the track number from a Amazon Connect Matroska fragment + by track name. + + ### Parameters: + fragment_dom: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + track_name (str): The name of the track to lookup. + + ### Returns: + int: The track number (as an integer), or None if not found. + ''' + for element in fragment_dom: + + for segment_child in element: + + if segment_child.name == "Tracks": + for cluster_child in segment_child: + fragment_dom_track_name = '' + fragment_dom_track_number = 0 + if cluster_child.name == "TrackEntry": + for te_child in cluster_child: + if te_child.name == "Name": + fragment_dom_track_name = te_child.value + if te_child.name == "TrackNumber": + fragment_dom_track_number = te_child.value + if fragment_dom_track_name == track_name: + return fragment_dom_track_number + return None + + def convert_track_to_wav(self, track_bytearray): + ''' + This function converts a track bytearray to a wav file. + ''' + + file_wav = io.BytesIO() + with wave.open(file_wav, 'wb') as f: + f.setnchannels(1) + f.setframerate(8000) + f.setsampwidth(2) + f.writeframes(track_bytearray) + return file_wav + + def save_connect_fragment_audio_track_as_wav(self, fragment_dom, track_nr, file_name_path): + ''' + Save the provided fragment_dom as wav file on local disk. + + ### Parameters: + + fragment_dom: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + tranck_nr: int + The track number (1 or 2) to extract + + file_name_path: Str + Local file path / name to save the MKV file to. + + ''' + + fragment_bytes = self.get_track_bytearray(fragment_dom, track_nr) + fragment_wav = self.convert_track_to_wav(fragment_bytes) + with open(file_name_path, 'wb') as f: + f.write(fragment_wav.getvalue()) + + def save_connect_fragment_audio_track_from_customer_as_wav(self, fragment_dom, file_name_path_part): + ''' + Saves the audio track from the customer in a Amazon Connect Matroska fragment + as a WAV file. + + ### Parameters: + + fragment_dom: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + file_name_path_part (str): The file path to save the WAV file to + + ''' + + track_number = self.get_track_number_by_name(fragment_dom, "AUDIO_FROM_CUSTOMER") + if track_number: + file_name_path = file_name_path_part + "-AUDIO_FROM_CUSTOMER.wav" + self.save_connect_fragment_audio_track_as_wav(fragment_dom, track_number, file_name_path) + + def save_connect_fragment_audio_track_to_customer_as_wav(self, fragment_dom, file_name_path_part): + ''' + Saves the audio track to the customer in a Amazon Connect Matroska fragment + as a WAV file. + + ### Parameters: + + fragment_dom: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + file_name_path_part (str): The file path to save the WAV file to + + ''' + track_number = self.get_track_number_by_name(fragment_dom, "AUDIO_TO_CUSTOMER") + if track_number: + file_name_path = file_name_path_part + "-AUDIO_TO_CUSTOMER.wav" + self.save_connect_fragment_audio_track_as_wav(fragment_dom, track_number, file_name_path) \ No newline at end of file diff --git a/lambda/cctv-people-rekognition/kinesis_video_streams_parser.py b/lambda/cctv-people-rekognition/kinesis_video_streams_parser.py new file mode 100644 index 0000000..36a81d7 --- /dev/null +++ b/lambda/cctv-people-rekognition/kinesis_video_streams_parser.py @@ -0,0 +1,227 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0. + +''' +Amazon Kinesis Video Stream (KVS) Consumer Library for Python. + +This library parses streaming bytes (chunks) made available by the StreamingBody returned from calls +to the KVS Media Client GetMedia and KVS Archive Media Client GetMediaForFragmentList +API. + +The Amazon Kinesis Video Stream (KVS) Consumer Library for Python reads in streaming bytes as they become +available and parses to individual MKV fragments. The library is threaded and non-blocking, +once a stream is being read it forwards received MKV fragments to named call-backs in the users application. + +Fragments are returned as raw bytes and a searchable DOM like structure by parsing with EMBLite by MideTechnology. + +The consumer library provides the following functions to further process parsed MKV fragments: +1) get_fragment_tags(): Extract MKV tags from the fragment. +2) save_fragment_as_local_mkv(): Saves the fragment as stand-alone MKV file on local disk. +3) get_frames_as_ndarray(): Returns a ratio of frames in the fragment as a list of NDArray objects. +4) save_frames_as_jpeg(): Returns a ratio of frames in the fragment as a JPEGs to local disk. + +Workflow: +1) Define a on_fragment_arrived and on_read_stream_complete call-backs in user application logic. These to process +fragments as they are received and to handle the parser reaching the end of the stream. (When no more fragments are left), +2) Initialize the KVS Media and / or Archive Media clients, +3) Make a call to KVS Media GetMedia and / or KVS Archive Media GetMediaForFragmentList for the given stream, +4) Initialize this KVS Consumer library and call get_streaming_fragements providing the response from the GetMedia +or GetMediaForFragmentList call, +5) Fragments will then be parsed and delivered to the call-backs for processing as per the example code provided. + +Credits: +# EMBLite by MideTechnology is an external EBML parser found at https://github.com/MideTechnology/ebmlite +# For convenance a slightly modified version of EMBLite is shipped with the KvsConsumerLibrary but adding credit where its due. +# EMBLite MIT License: https://github.com/MideTechnology/ebmlite/blob/development/LICENSE + + ''' + +__version__ = "0.0.1" +__status__ = "Development" +__copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved." +__author__ = "Dean Colcott " + +import timeit +import logging +from threading import Thread +from ebmlite import loadSchema + +# Init the logger. +log = logging.getLogger(__name__) + + +class KvsConsumerLibrary(Thread): + + def __init__(self, + stream_name, + get_media_response_object, + on_fragment_arrived, + on_read_stream_complete, + on_read_stream_exception): + ''' + Initialize the KVS media consumer library + ''' + # Call the Thread class's init function + Thread.__init__(self) + + # Used to trigger graceful exit of this thread + self._stop_get_media = False + + # Init the local vars. + log.info('Initilizing KvsConsumerLibrary...') + self.stream_name = stream_name + self.get_media_response_object = get_media_response_object + self.on_fragment_arrived_callback = on_fragment_arrived + self.on_read_stream_complete_callback = on_read_stream_complete + self.on_read_stream_exception = on_read_stream_exception + + log.info('Loading EBMLlite MKV Schema....') + self.schema = loadSchema('matroska.xml') + + def _get_ebml_header_elements(self, fragement_dom): + ''' + Returns the EBML Header elements in the Fragment DOM. EBML Header elements indicate the start + of a new fragment and so we use them to set the byte boundaries of individual fragments as they + arrive in the raw data stream (chunks). + + ### Parameters: + + **fragment_dom**: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + ''' + ebml_header_elements = [] + # Iterate through the fragment elements and capture any EBML Fragment headers (indicating the start of a new fragment) + for element in fragement_dom: + if (element.id == 0x1A45DFA3): # EBML (Master) element ID = 0x1A45DFA3 (440786851 dec) + ebml_header_elements.append(element) + + return ebml_header_elements + + def _get_simple_block_elements(self, fragement_dom): + ''' + Returns the DOM SimpleBlock elements found in the fragment. + SimpleBlock Elements store the payload of the MKV fragemeny - typically H.264/265 frames but + can be any data playload that was ingested by the KVS producer. + + ### Parameters: + + **fragment_dom**: ebmlite.core.Document + The DOM like structure describing the fragment parsed by EBMLite. + + ''' + simple_block_elements = [] + # Iterate through the fragment elements and capture any Simple Block type elements. + # These carry the fragments payload bytes (typically image frames as raw bytes.) + for element in fragement_dom: + if (element.id == 0x18538067): # Segment element ID = 0x18538067 + + for segement_child in element: + if (segement_child.id == 0x1F43B675): # Cluster element ID = 0x1F43B675 + + for cluster_child in segement_child: + if (cluster_child.id == 0xA3): # SimpleBlock element ID = xA3 + simple_block_elements.append(cluster_child) + + return simple_block_elements + + def stop_thread(self): + self._stop_get_media = True + + #################################################### + # Read and parse streaming media from a Kinesis Video Stream + def run(self): + ''' + Reads in chunks (unframed number of raw bytes) from a KVS GetMedia or GetMediaForFragmentList Streaming Body response + and parses into bounded MKV fragments. Raw data is buffered until a complete fragment is received which is then forwarded to the + on_fragmemt_arrived callback. Fragment is delivered as a raw byte array and also a parsed EBMLite Document that is a DOM like + structure of the elements (including Tags) within the given Fragment. + + Kinesis Video will continually update the streaming buffer with media as soon as its available. For StartSelectorType = NOW, + bytes from the media stream will be available as fast as they arrive into Kinesis Video by the producer. In this case the + consumer bandwidth and fragment rate will be equal to that of the producer. However, if StartSelector is set to sometime + in the past then all fragments from start to end time will be available immediately. The effect is this will + read in bytes as fast as the system resources (KVS limits, CPU and bandwidth) will allow until the stream has + caught up with the leading edge of media being generated. + + ''' + + try: + # Get the steam botocore.response.Streamingody object from the provided GetMedia response + kvs_streaming_buffer=self.get_media_response_object['Payload'] + + ######################################### + # Iterate through reading and parsing streaming body response of KVS GET Media API call to MKV fragments. + ######################################### + chunk_buffer = bytearray() + fragment_read_start_time = timeit.default_timer() + + chunk_read_count = 0 + + # Uses the StreamingBody object iterator to read in (default 1024 byte) chunks from the streaming buffer. + for chunk in kvs_streaming_buffer: + + if self._stop_get_media: + break + + # Append chunk bytes to ByteArray buffer while waiting for the entire MKV fragment to arrive. + chunk_buffer.extend(chunk) + + ############################################# + # Parse current byte buffer to MKV EBML DOM like object using EBMLite + ############################################# + fragement_intrum_dom = self.schema.loads(chunk_buffer) + + ############################################# + # Process a complete fragment if its arrived and send to the on_fragment_arrived callback. + ############################################# + # EBML header elements indicate the start of a new fragment. Here we check if the start of a second fragment + # has arrived and use its start to identify the byte boundary of the first complete fragment to process. + ebml_header_elements = self._get_ebml_header_elements(fragement_intrum_dom) + + # If multiple fragment headers then the first fragment has been received completely and ready to process. + if (len(ebml_header_elements) > 1): + + # Get the offset for the first and second fragments. First fragment offset should be zero or fragment boundary is out of sync! + first_ebml_header_offset = ebml_header_elements[0].offset + second_ebml_header_offset = ebml_header_elements[1].offset + + # Isolate the bytes from the first complete MKV fragments in the received chunk data + fragment_bytes = chunk_buffer[first_ebml_header_offset : second_ebml_header_offset] + + # Parse the complete fragment as EBML to a DOM like object + fragment_dom = self.schema.loads(fragment_bytes) + + # Calculate duration taken receiving this fragment - just for telemetry of the steaming data. + fragment_receive_duration = timeit.default_timer() - fragment_read_start_time + + # Forward fragment to the on_fragment_arrived callback. + self.on_fragment_arrived_callback(self.stream_name, + fragment_bytes, + fragment_dom, + fragment_receive_duration) + + # Remove the processed MKV segment from the raw byte chunk_buffer + chunk_buffer = chunk_buffer[second_ebml_header_offset: ] + + # Reset the chunk read count. + chunk_read_count = 0 + + # Reset the start time for the next segment iteration just to time fragment durations + fragment_read_start_time = timeit.default_timer() + + ############################################# + # Increment to chunk read count for this fragment + chunk_read_count +=1 + + ############################################# + # Exit the thread if the stream has no more chunks. + ############################################# + #call the on_stream_read_complete() callback and exit the thread. + self.on_read_stream_complete_callback(self.stream_name) + + except Exception as err: + # Pass any exceptions to exception callback. + self.on_read_stream_exception(self.stream_name, err) + + diff --git a/lambda/cctv-people-rekognition/lambda_function.py b/lambda/cctv-people-rekognition/lambda_function.py new file mode 100644 index 0000000..006f624 --- /dev/null +++ b/lambda/cctv-people-rekognition/lambda_function.py @@ -0,0 +1,141 @@ +import os +import sys +import time +import boto3 +from botocore.exceptions import ClientError +from datetime import datetime, timedelta +from kinesis_video_streams_parser import KvsConsumerLibrary +from kinesis_video_fragment_processor import KvsFragementProcessor + +rekognition_client = boto3.client("rekognition") +kvs_client = boto3.client('kinesisvideo') +kvs_fragment_processor = KvsFragementProcessor() +last_good_fragment_tags = None + +stream_name = "" ## Provide KVS name here (stream must already exist) +stream_arn = "" ## Provide KVS ARN here (stream must already exist) +stream_processor_name = "" ## Provide a name for the stream processor +s3_bucket_name = "" ## Provide the S3 bucket +role_arn = "" ## Provide the role ARN for Rekognition +sns_topic_arn = "" ## Provide the SNS ARN + + +############################################# +## KVS Consumer Library Callbacks + +def on_fragment_arrived(stream_name, fragment_bytes, fragment_dom, fragment_receive_duration): + try: + last_good_fragment_tags = kvs_fragment_processor.get_fragment_tags(fragment_dom) + fragment_num = last_good_fragment_tags['AWS_KINESISVIDEO_FRAGMENT_NUMBER'] + rekognition_client.start_stream_processor( + Name=stream_processor_name, + StartSelector={ + 'KVSStreamStartSelector': { + 'FragmentNumber': fragment_num, + } + }, + StopSelector={ + 'MaxDurationInSeconds': 2 + } + ) + time.sleep(2) + except Exception as err: + print(err) + +def on_stream_read_complete(stream_name): + print(f"Stream {stream_name} read complete") + +def on_stream_read_exception(stream_name, error): + print(f"Stream {stream_name} read exception: {error}") + +## Main Lambda Function + +def lambda_handler(event, context): + ## Step 1: Ensure stream processor is deleted + try: + rekognition_client.delete_stream_processor( + Name=stream_processor_name, + ) + rekognition_client.create_stream_processor( + Input={ + 'KinesisVideoStream': { + 'Arn': stream_arn + } + }, + Output={ + 'S3Destination': { + 'Bucket': s3_bucket_name, + 'KeyPrefix': 'stream-results' + } + }, + Name=stream_processor_name, + Settings={ + 'ConnectedHome': { + 'Labels': [ + 'PERSON', + ], + 'MinConfidence': 80 + } + }, + RoleArn=role_arn, + NotificationChannel={ + 'SNSTopicArn':sns_topic_arn + } + ) + except: + ## Step 2: Create Rekognition Stream Processor + rekognition_client.create_stream_processor( + Input={ + 'KinesisVideoStream': { + 'Arn': stream_arn + } + }, + Output={ + 'S3Destination': { + 'Bucket': s3_bucket_name, + 'KeyPrefix': 'stream-results' + } + }, + Name=stream_processor_name, + Settings={ + 'ConnectedHome': { + 'Labels': [ + 'PERSON', + ], + 'MinConfidence': 80 + } + }, + RoleArn=role_arn, + NotificationChannel={ + 'SNSTopicArn':sns_topic_arn + } + ) + + ## Step 3: Prepare connection to stream + kvs_client = boto3.client('kinesisvideo') + + response = kvs_client.get_data_endpoint( + StreamARN=stream_arn, + APIName='GET_MEDIA' + ) + + get_endpoint = response['DataEndpoint'] + + kvs_media_client = boto3.client('kinesis-video-media', endpoint_url=get_endpoint) + get_media_response = kvs_media_client.get_media( + StreamName=stream_name, + StartSelector={ + 'StartSelectorType': 'NOW' + } + ) + + ## Step 4: Prepare consumer library + my_stream01_consumer = KvsConsumerLibrary(stream_name, + get_media_response, + on_fragment_arrived, + on_stream_read_complete, + on_stream_read_exception + ) + + ## Step 5: Run consumer + my_stream01_consumer.run() \ No newline at end of file diff --git a/lambda/cctv-people-sns-dynamodb/lambda_function.py b/lambda/cctv-people-sns-dynamodb/lambda_function.py new file mode 100644 index 0000000..87741f7 --- /dev/null +++ b/lambda/cctv-people-sns-dynamodb/lambda_function.py @@ -0,0 +1,160 @@ +import json +import os +import boto3 + +from datetime import datetime, timezone +from decimal import Decimal + +now = datetime.utcnow() +dynamodb = boto3.resource("dynamodb") +table = dynamodb.Table("cctv-people-analytics-table") +region = "eu-west-1" + +from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth +from requests_aws4auth import AWS4Auth + +service = 'aoss' +credentials = boto3.Session().get_credentials().get_frozen_credentials() +print(credentials.access_key, credentials.secret_key, credentials.token) + +auth = AWSV4SignerAuth(credentials, region, service) + +auth = AWS4Auth( + credentials.access_key, + credentials.secret_key, + region, + service, + session_token=credentials.token # ✅ include session token! +) + +host = "5xmv01a4621tqq8agz99.eu-west-1.aoss.amazonaws.com" +index = now.strftime("cctv-people-analytics-%Y-%m-%d-%H") +url = f"{host}/{index}/_doc" + +client = OpenSearch( + hosts=[{'host': host, 'port': 443}], + http_auth=auth, + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection, + pool_maxsize=20, +) + +if not client.indices.exists(index=index): + client.indices.create(index=index) + +def normalize_timestamp(ts: str) -> str: + """ + Normalize timestamp into ISO 8601 with UTC (Z). + Handles: + - %Y-%m-%dT%H-%M-%S (bad format with dashes) + - %Y-%m-%dT%H:%M:%S (already correct) + - Epoch numbers (int/float) + """ + if isinstance(ts, (int, float)): # epoch + return datetime.fromtimestamp(ts/1000, tz=timezone.utc).isoformat().replace("+00:00", "Z") + + if isinstance(ts, str): + # Already ISO with colons + try: + dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S") + return dt.replace(tzinfo=timezone.utc).isoformat().replace("+00:00", "Z") + except ValueError: + pass + + # Bad format with dashes + try: + dt = datetime.strptime(ts, "%Y-%m-%dT%H-%M-%S") + return dt.replace(tzinfo=timezone.utc).isoformat().replace("+00:00", "Z") + except ValueError: + pass + + # Fallback: return as-is + return str(ts) + +def epoch_converter(ts): + """ + Normalize timestamp into epoch milliseconds (integer). + """ + if isinstance(ts, (int, float)): + # already epoch + return int(ts) + if isinstance(ts, str): + try: + # Try ISO format + dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S") + except ValueError: + # Bad format with dashes + dt = datetime.strptime(ts, "%Y-%m-%dT%H-%M-%S") + # Convert to UTC epoch milliseconds + dt = dt.replace(tzinfo=timezone.utc) + return int(dt.timestamp() * 1000) + return int(ts) # fallback + +def lambda_handler(event, context): + print(event) + dict_stream_camid = { + 'CCTV-Stream': 'iMbYu7fVoHRP', + 'TAPO-C246D-Stream': 'piYD7HWDKAub', + 'TAPO-C500-Stream': 'EZ4DnbJjqFvk', + 'HIKVision-2MP-Dome-1-Stream': 'x4j8eVyrF6mO', + 'HIKVision-2MP-Dome-2-Stream': 'NnIXfhJeqX6i', + 'HIKVision-2MP-Bullet-1-Stream': 'vP2wVuCEN5vi', + 'HIKVision-2MP-Bullet-2-Stream': 'SnPNqUVLHinu', + 'Dahua-2MP-Dome-1-Stream': 'YMpY5kku8IXg', + 'Dahua-2MP-Dome-2-Stream': 'yyzi5SqLTftn', + 'Dahua-2MP-Bullet-1-Stream': 'YTOYlpGjviLB', + 'Dahua-2MP-Bullet-2-Stream': 'cLVBa3vAV4yf' + } + + notification = json.loads(event['Records'][0]['Sns']['Message']) + if notification['eventNamespace']['type'] == 'LABEL_DETECTED': + stream_arn = notification['inputInformation']['kinesisVideo']['streamArn'] + camera_id = dict_stream_camid[stream_arn.split('/')[1]] + + for label in notification['labels']: + label_id = label['id'] + detect_timestamp = datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + frame_timestamp = label['videoMapping']['kinesisVideoMapping']['serverTimestamp'] + frame_url = str.replace(label['frameImageUri'], 's3://cctv-stream-results/', 'https://cctv-stream-results.s3.eu-west-1.amazonaws.com/') + label_name = label['name'] + confidence = label['confidence'] + + to_upload_dynamodb = { + 'camera_id': camera_id, + 'label_id': label_id, + 'detect_timestamp': normalize_timestamp(detect_timestamp), + 'frame_timestamp': normalize_timestamp(frame_timestamp), + 'frame_url': frame_url, + 'label_name': label_name, + 'confidence': Decimal(str(confidence)), + 'rekognition_response': json.dumps(label) + } + print() + response= table.put_item(Item=to_upload_dynamodb) + if response['ResponseMetadata']['HTTPStatusCode'] == 200: + print("Item successfully written!") + + # headers = { "Content-Type": "application/json" } + # response = requests.post(url, auth=awsauth, json=to_upload, headers=headers) + # print(response.text, response.status_code) + to_upload_es = { + 'label_id': label_id, + '@timestamp': normalize_timestamp(detect_timestamp), + 'frame_timestamp': normalize_timestamp(frame_timestamp), + 'frame_url': frame_url, + 'label_name': label_name, + 'confidence': Decimal(str(confidence)) + } + + #### + response = client.index( + index = index, + body = to_upload_es + ) + print(response) + + return { + 'statusCode': 200, + 'body': json.dumps('Processing cctv results success'), + } \ No newline at end of file