model.xml

<project heartbeat-interval="1" index="pi_EMPTY" luaroot="@ESP_PROJECT_OUTPUT@/luaroot" name="computer_vision_with_onnx" pubsub="auto" threads="8" use-tagged-token="true">
  <metadata>
    <meta id="studioUploadedBy">anonymousUser</meta>
    <meta id="studioUploaded">1765973789399</meta>
    <meta id="studioModifiedBy">anonymousUser</meta>
    <meta id="studioModified">1765973789399</meta>
    <meta id="layout">{"contquery":{"w_annotate":{"x":50,"y":540},"w_counter":{"x":50,"y":665},"w_data":{"x":290,"y":50},"w_object_tracker":{"x":50,"y":420},"w_postprocess":{"x":50,"y":295},"w_reader":{"x":50,"y":50},"w_score":{"x":50,"y":175}}}</meta>
    <meta id="studioTags">Example</meta>
    <meta id="projectComputationalRequirements">{"minCpu":"3", "minGpu":"1", "minMem":"8"}</meta>
  </metadata>
  <properties>
    <property name="PUBLISH_WIDTH"><![CDATA[1280]]></property>
    <property name="PUBLISH_HEIGHT"><![CDATA[720]]></property>
    <property name="MODEL_WIDTH"><![CDATA[640]]></property>
    <property name="MODEL_HEIGHT"><![CDATA[640]]></property>
  </properties>
  <contqueries>
    <contquery name="contquery">
      <windows>
        <window-source index="pi_EMPTY" insert-only="true" name="w_data">
          <description><![CDATA[w_data is a Source window. This is where video frames enter the project.]]></description>
          <schema>
            <fields>
              <field key="true" name="id" type="int64"/>
              <field name="image" type="blob"/>
            </fields>
          </schema>
          <connectors>
            <connector class="videocap" name="video_publisher">
              <properties>
                <property name="type"><![CDATA[pub]]></property>
                <property name="inputrate"><![CDATA[4]]></property>
                <property name="repeatcount"><![CDATA[999]]></property>
                <property name="resize_x"><![CDATA[@PUBLISH_WIDTH@]]></property>
                <property name="resize_y"><![CDATA[@PUBLISH_HEIGHT@]]></property>
                <property name="blocksize"><![CDATA[1]]></property>
                <property name="filename"><![CDATA[@ESP_PROJECT_HOME@/test_files/video.mp4]]></property>
                <property name="publishformat"><![CDATA[wide]]></property>
              </properties>
            </connector>
          </connectors>
        </window-source>
        <window-model-reader model-type="onnx" name="w_reader">
          <description><![CDATA[w_reader is a Model Reader window. This window reads the ONNX model and passes it to the w_score window. Also, pre-processing steps for the incoming events are specified in this window.]]></description>
          <processing-steps>
            <pre name="pre_processing_default">
              <step name="resize">
                <param name="resizeType"><![CDATA[letterbox]]></param>
                <param name="width"><![CDATA[@MODEL_WIDTH@]]></param>
                <param name="height"><![CDATA[@MODEL_HEIGHT@]]></param>
              </step>
              <step name="color">
                <param name="function"><![CDATA[BGR2RGB]]></param>
              </step>
              <step name="normalize">
                <param name="type"><![CDATA[zero_one]]></param>
              </step>
              <step name="encode">
                <param name="shape"><![CDATA[NCHW]]></param>
              </step>
            </pre>
          </processing-steps>
          <warmup-steps type="random" count="1"/>
          <parameters>
            <properties>
              <property name="reference"><![CDATA[@ESP_PROJECT_HOME@/analytics/yolov7/yolov7-tiny_640x640.onnx]]></property>
              <property name="loggingLevel"><![CDATA[warning]]></property>
              <property name="execProvider"><![CDATA[cuda]]></property>
              <property name="cudaDeviceId"><![CDATA[0]]></property>
            </properties>
          </parameters>
        </window-model-reader>
        <window-score name="w_score">
          <description><![CDATA[w_score is a Score window. This window executes the ONNX model’s code when data passes through the window.]]></description>
          <schema>
            <fields>
              <field key="true" name="id" type="int64"/>
              <field name="image" type="blob"/>
              <field name="output" type="blob"/>
            </fields>
          </schema>
          <models>
            <offline model-type="onnx">
              <input-map>
                <properties>
                  <property name="images"><![CDATA[image]]></property>
                </properties>
              </input-map>
              <output-map>
                <properties>
                  <property name="output"><![CDATA[output]]></property>
                </properties>
              </output-map>
            </offline>
          </models>
        </window-score>
        <window-python events="postprocess_event" name="w_postprocess" output-insert-only="true">
          <description><![CDATA[w_postprocess is a Python window. The Python code in this window converts the model output (tensor format) to more usable formats.]]></description>
          <schema>
            <fields>
              <field key="true" name="id" type="int64"/>
              <field name="image" type="blob"/>
              <field name="_nObjects_" type="int32"/>
              <field name="Object_x" type="array(dbl)"/>
              <field name="Object_y" type="array(dbl)"/>
              <field name="Object_width" type="array(dbl)"/>
              <field name="Object_height" type="array(dbl)"/>
              <field name="Object_score" type="array(dbl)"/>
              <field name="Object_labels" type="string"/>
            </fields>
          </schema>
          <copy exclude="true"><![CDATA[output]]></copy>
          <use expand="true"><![CDATA[output]]></use>
          <code><![CDATA[import esp_utils

with open("@ESP_PROJECT_HOME@/analytics/yolov7/labels.txt") as file:
    labels = [line.rstrip() for line in file]

letterbox_image_shape = (@MODEL_HEIGHT@, @MODEL_WIDTH@)
original_image_shape = (@PUBLISH_HEIGHT@, @PUBLISH_WIDTH@)

def postprocess_event(output):
    event = {}
    output = esp_utils.onnx_tensor.tensor_to_np_array(output)
    event["Object_x"] = []
    event["Object_y"] = []
    event["Object_width"] = []
    event["Object_height"] = []
    event["Object_score"] = []
    event["Object_labels"] = ""

    for object_id, object_attributes in enumerate(output):
        x1, y1, x2, y2 = object_attributes[1:5].astype(int)
        x, y, w, h = esp_utils.postprocessing.bbox_letterbox_to_original(
            (x1, y1, x2 - x1, y2 - y1), letterbox_image_shape, original_image_shape
        )
        label_id = object_attributes[5].astype(int)
        score = object_attributes[6]
        label = labels[label_id]
        
        event["Object_x"].append(float(x))
        event["Object_y"].append(float(y))
        event["Object_width"].append(float(w))
        event["Object_height"].append(float(h))
        event["Object_score"].append(float(score))
        event["Object_labels"] = f"{event['Object_labels']}{label},"
        
    event["Object_labels"] = event["Object_labels"][:-1] # remove last comma
    event["_nObjects_"] = len(output)
    return event
                    ]]></code>
        </window-python>
        <window-object-tracker name="w_object_tracker">
          <description><![CDATA[w_object_tracker is an Object Tracker window. This window enables you to track objects over time.]]></description>
          <tracker method="bytetrack" track-thresh="0.5" high-thresh="0.6" match-thresh="0.8" velocity-vector-frames="15" max-track-lives="10" track-retention="0"/>
          <output mode="array" tracks="20"/>
          <input count="_nObjects_" score="Object_score" label="Object_labels" attributes="_Object%_attributes" label-separator="," coord-type="rect" x="Object_x" y="Object_y" width="Object_width" height="Object_height"/>
        </window-object-tracker>
        <window-counter name="w_counter">
          <description><![CDATA[w_counter is a Counter window. This window provides an indication of the overall performance.]]></description>
        </window-counter>
        <window-custom name="w_annotate" type="Computer Vision Annotation" version="1">
          <description><![CDATA[Annotate the results of an object detection and/or a keypoint detection model.]]></description>
          <schema>
            <fields>
              <field key="true" name="id" type="int64"/>
              <field name="image" type="blob"/>
            </fields>
          </schema>
          <plugin code="@ESP_PROJECT_HOME@/custom_windows/Computer Vision Annotation/Computer Vision Annotation.py">
            <input-map description="Fields related to image and object detection are required. Fields related to keypoints are optional. Object ID (for object tracking) and attributes are also optional.">
              <properties>
                <property description="Input image (blob)" name="image" required="true"><![CDATA[image]]></property>
                <property description="Delimited list containing the class of the detected objects (string)" name="label" required="true"><![CDATA[Object_label]]></property>
                <property description="Top left X-coordinates of the bounding boxes (array(dbl))" name="x" required="true"><![CDATA[Object_x]]></property>
                <property description="Top left Y-coordinates of the bounding boxes  (array(dbl))" name="y" required="true"><![CDATA[Object_y]]></property>
                <property description="Widths of the bounding boxes (array(dbl))" name="w" required="true"><![CDATA[Object_w]]></property>
                <property description="Heights of the bounding boxes (array(dbl))" name="h" required="true"><![CDATA[Object_h]]></property>
                <property description="Confidence scores array (array(dbl))" name="score" required="true"><![CDATA[Object_score]]></property>
                <property description="Tracked object ID, for example from the Object Tracker window (array(i32))" name="object_id"><![CDATA[Object_id]]></property>
                <property description="Delimited list containing the attribute(s) of the detected objects (string)" name="attribute"/>
                <property description="Number of tracks per detected object (array(i32))" name="object_track_count"/>
                <property description="Number of keypoints per detected object for the track (array(i32))" name="object_track_kpts_count"/>
                <property description="X-coordinates for the keypoints track (array(dbl))" name="object_track_kpts_x"/>
                <property description="Y-coordinates for the keypoints track (array(dbl))" name="object_track_kpts_y"/>
                <property description="Confidence scores for the keypoints track (array(dbl))" name="object_track_kpts_score"/>
                <property description="Label IDs for the keypoints track (array(i32))" name="object_track_kpts_label_id"/>
              </properties>
            </input-map>
            <output-map description="Add an output field, of type `blob`, to store the annotated image. If you use the same image as used in the input variables, the original image is overwritten with an annotated image.">
              <properties>
                <property description="Annotated image (blob)" name="annotated_image"><![CDATA[image]]></property>
              </properties>
            </output-map>
            <initialization description="Set the options for the custom window. Note that either `png` or `jpg` is needed as the value for `output_image_encoding` to display images in Grafana.">
              <properties>
                <property description="Input image encoding - must be one of the following: `wide`, `jpg`, `png`" name="input_image_encoding"><![CDATA[wide]]></property>
                <property description="Output image encoding - must be one of the following: `wide`, `jpg`, `png`" name="output_image_encoding"><![CDATA[jpg]]></property>
                <property description="Object label separator" name="object_label_separator"><![CDATA[,]]></property>
                <property description="Keypoint labels" name="kpts_labels"/>
              </properties>
            </initialization>
          </plugin>
        </window-custom>
      </windows>
      <edges>
        <edge role="data" source="w_data" target="w_score"/>
        <edge role="model" source="w_reader" target="w_score"/>
        <edge role="data" source="w_score" target="w_postprocess"/>
        <edge role="data" source="w_postprocess" target="w_object_tracker"/>
        <edge source="w_object_tracker" target="w_annotate"/>
        <edge source="w_annotate" target="w_counter"/>
      </edges>
    </contquery>
  </contqueries>
</project>