Problem in getting bounding boxes for my search plugin

I

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using VideoOS.Platform;
using VideoOS.Platform.Search;
using VideoOS.Platform.Search.Results;
 
namespace Human.SearchAgent
{
 
    public class HumanSearchDefinition : SearchDefinition
    {
        public HumanSearchDefinition(SearchScope searchScope) : base(searchScope)
        { }
 
        protected override void Search(Guid sessionId, DateTime from, DateTime to, IEnumerable<Item> items, CancellationToken cancellationToken)
        {
 
 
 
       
            var results = new List<SearchResultData>();
            var xmlResults = GetResultFromXML();
            foreach (var r in xmlResults)
            {
                var searchResult = new HumanSearchResultData(GenerateSeededGuide(r.Id),r)
                {
                    Title = Convert.ToString("Width:"+r.width+"\nHeight:"+r.height+"\nX:"+r.x_position+"\nY:"+r.y_position),//"Human",    // TODO: Default title is camera name, but you can modify it if needed.
                    Item = items.First(i => i.FQID.ObjectId == r.deviceID),
                    BeginTime = r.objectRecognitionTime,
                    EndTime = r.objectRecognitionTime.AddSeconds(1),
                    TriggerTime = r.objectRecognitionTime//from.AddTicks((to - from).Ticks / 2)
                };
                results.Add(searchResult);
            }
            if(results.Any())
                FireSearchResultsReadyEvent(sessionId, results);
                    
    
 
 
        }
 
        private Guid GenerateSeededGuide(int seed)
        {
            var r = new Random(seed);
            var guide = new byte[16];
            r.NextBytes(guide);
            return new Guid(guide);
        }
 
         
 
        private static List<HumanMetaData> GetResultFromXML()
        {
            var sourceFolder = "C://<Path to XML>";
        
            var files = Directory.GetFiles(sourceFolder, "*.xml", SearchOption.AllDirectories);
            var results = new List<HumanMetaData>();
            int count = 0;
            foreach (var file in files)
            {
                var doc = XDocument.Load(file);
                var h = doc.Root.Element("event").Descendants("entity").ElementAt(0).Attribute("height").Value;
                var time = doc.Root.Element("event").Attribute("frameTimestamp").Value;
                var x = doc.Root.Element("event").Descendants("entity").ElementAt(0).Attribute("position-x").Value;
                var y = doc.Root.Element("event").Descendants("entity").ElementAt(0).Attribute("position-y").Value;
                var w = doc.Root.Element("event").Descendants("entity").ElementAt(0).Attribute("width").Value;
                var t = doc.Root.Element("event").Descendants("entity").ElementAt(0).Attribute("type").Value;
                count = count + 1;
                results.Add(new HumanMetaData
                {   Id = count,
                    deviceID = new Guid("D873A47D-2D9A-4EEF-8D55-F3117D4D8FBA"),
                    objectRecognitionTime = Convert.ToDateTime(time),
                    height = Convert.ToDouble(h),
                    x_position = Convert.ToDouble(x),
                    y_position = Convert.ToDouble(y),
                    width = Convert.ToDouble(w),
                    type = t
 
                });
 
            }
 
            return results;
 
        }
 
 
    }
    internal class HumanSearchResultData : SearchResultData
    {
        public HumanMetaData HumanMetaData { get; }
 
        public HumanSearchResultData(Guid id, HumanMetaData humanMetaData) : base(id)
        {
            HumanMetaData = humanMetaData;
        }
 
 
        protected override Task<ICollection<BoundingShape>> GetBoundingShapesAsync(CancellationToken cancellationToken)
        {
            return Task.Run(delegate
            {
                ICollection<BoundingShape> shapes = new List<BoundingShape>();
                shapes.Add(new RectangleBoundingShape
                {
                    X = HumanMetaData.x_position,
                    Y = HumanMetaData.y_position,
                    Width = HumanMetaData.width,
                    Height = HumanMetaData.height
                });
                return shapes;
            }, cancellationToken);
        }
 
        /*protected override Task<ICollection<ResultProperty>> GetPropertiesAsync(CancellationToken cancellationToken)
        {
            return base.GetPropertiesAsync(cancellationToken);
        }
        protected override Task<ICollection<BoundingShape>> GetBoundingShapesAsync(CancellationToken cancellationToken)
        {
            return Task.Run(delegate
            {
                ICollection<BoundingShape> shapes = new List<BoundingShape>();
                shapes.Add(new RectangleBoundingShape
                {
                    X = HumanMetaData.x_position,
                    Y = HumanMetaData.y_position,
                    Width = HumanMetaData.width,
                    Height = HumanMetaData.height
                });
                return shapes;
            }, cancellationToken);
        }
        */
        protected override Task<ICollection<ResultProperty>> GetPropertiesAsync(CancellationToken cancellationToken)
        {
            return Task.Run(delegate
            {
                ICollection<ResultProperty> properties = new List<ResultProperty>();
                properties.Add(new ResultProperty("Human Type", "Human"));
                return properties;
            }, cancellationToken);
 
 
        }
    }
 
    internal class HumanMetaData
    {
        public int Id { get; set; }
        public Guid deviceID { get; set; }
        public DateTime objectRecognitionTime { get; set; }
        public double height { get; set; }
        public double x_position { get; set; }
        public double y_position { get; set; }
        public double width { get; set; }
        public string type { get; set; }
        //public string colors = "Yellow";
 
    }
 
 
 
 
}
 
 
 
 
 
 

Above is my code for HumanSearchDefinition.cs for detecting human events from xml data and put bounding box around them. This code is able to get the events but the bounding boxes are not visible. Can suggest what is wrong in the code ?

Milestone Development will start an investigation.

I received updates from Milestone Development, please see below.

----------

From the screenshot it looks like there is no recorded video available at the search results trigger time. The images are grayed out. In that case bounding boxes will not be displayed because they are not “guarantied” to match the image shown.

In addition to that please be aware that the overridden method

protected override Task<ICollection> GetBoundingShapesAsync(CancellationToken cancellationToken)

is deprecated and will be removed in XPCO 2021 R3. Use

protected virtual Task<ICollection> GetBoundingShapesAsync(DateTime time, CancellationToken cancellationToken)

instead. See the MIP SDK documentation for more information.

  • TKs Rie for yr answer but I would like to add on extra information after our weeks of tests.

  1. the sample shown might not have recorded video, but we have tested multiple times with a timestamp that has recorded video and have successfully taged our keyword HUMAN to that image. The image ia retrieved and even the coordinates X Y height width are successfully displayed in the image title.

So we can safely conclude that the milestone meta object is getting all the correct information like meta type, timestamp, x y height width.

Further more a search by our meta type Human does retrieve the correct tagged data, time stamp and image.

The question now is why can’t we see the bounding box even though the meta objects x y width and height contains the information? If I tag an a recording that is an hour old with a timetamp one hour ago, there shouldn’t there be no restrictions on displaying such bounding boxes?

I have consulted Milestone Development experts and they said that it is extremely difficult to analyze the code provided and pinpoint exactly what is wrong. So can you please provide us a minimum sample to reproduce the issue for further investigation?

Milestone Development investigated the issue, please see below comment.

-----

I have debugged the search plugin and found the root cause of the missing bounding boxes. It is located in the search plugin.

Two code errors have been found, both in HumanSearchDefinitions.GetResultFromXml(), and the resolution have been identified. This fix is shown in the file comparison below where left is the partners code and right is the fixed code.

Error 1:

The object recognition time extracted from the xml using Convert.ToDateTime() ends up with Kind = Unspecified. That is later used to create the SearchResultData object which is returned to the Smart Client. As specified in the documentation the timestamps on the SearchResultData should be in UTC.

Error 2:

The bounding box location and size is not read correctly in regions not using ‘dot’ as decimal separator. In Denmark we use ‘comma’ and it results in the values being wrong and therefore the bounding box not being displayed.

Note that the fixes in the screenshot above is only for inspiration. The partner should evaluate what the correct fix is in their context.

After changing the code the search results now have bounding boxes

-—