Open video io in media stream + bug fixes (#104)

Author: MathieuFavreau

c_plus_plus/src/model/stream/media_thread.cpp

@@ -85,10 +85,12 @@ void MediaThread::run()
         // Set background color of empty display
         displayImageBuilder.setDisplayImageColor(emptyDisplay);
 
-        // Start audio resources
+        // Start audio and video resources
         audioSource_->open();
         audioSink_->open();
         positionSource_->open();
+        videoInput_->open();
+        videoOutput_->open();
 
         Point<float> fisheyeCenter(videoInputConfig_.resolution.width / 2.f, videoInputConfig_.resolution.height / 2.f);
         std::vector<SphericalAngleRect> detections;
@@ -213,10 +215,12 @@ void MediaThread::run()
         std::cout << "Error in video thread : " << e.what() << std::endl;
     }
 
-    // Clean audio resources
+    // Clean audio and video resources
     audioSource_->close();
     audioSink_->close();
     positionSource_->close();
+    videoInput_->close();
+    videoOutput_->close();
     delete[] audioBuffer;
 
     // Deallocate display images

c_plus_plus/src/model/stream/utils/threads/thread.cpp

@@ -15,6 +15,8 @@ void Thread::start()
 {
     if (!isRunning_)
     {
+        join();    // Make sure the thread is truly finished
+
         thread_ = std::make_unique<std::thread>(&Thread::threadExecution, this);
         isRunning_ = true;
     }
@@ -30,7 +32,7 @@ void Thread::stop()
 
 void Thread::join()
 {
-    if (isRunning_)
+    if (thread_ != nullptr && thread_->joinable())
     {
         thread_->join();
     }
@@ -52,6 +54,10 @@ void Thread::threadExecution()
     {
         run();
     }
+    catch (const std::exception& e)
+    {
+        std::cout << "Exception during thread execution : " << e.what() << std::endl;
+    }
     catch (...)
     {
         std::cout << "Unknown exception during thread execution" << std::endl;
@@ -60,5 +66,4 @@ void Thread::threadExecution()
     isAbortRequested_ = false;
     isRunning_ = false;
 }
-
 }    // namespace Model

c_plus_plus/src/model/stream/video/detection/detection_thread.cpp

@@ -35,54 +35,72 @@ void DetectionThread::run()
     Dim2<int> detectionResolution(detector_->getInputImageDim());
     Point<float> fisheyeCenter(resolution.width / 2.f, resolution.height / 2.f);
 
-    std::vector<DewarpingParameters> dewarpingParams = getDetectionDewarpingParameters(resolution, dewarpCount_);
-    std::vector<ImageFloat> detectionImages = getDetectionImages(detectionResolution, dewarpCount_);
-    std::vector<DewarpingMapping> dewarpingMappings =
-        getDewarpingMappings(dewarpingParams, resolution, detectionResolution, dewarpCount_);
+    std::vector<DewarpingParameters> dewarpingParams;
+    std::vector<ImageFloat> detectionImages;
+    std::vector<DewarpingMapping> dewarpingMappings;
     std::vector<SphericalAngleRect> detections;
 
-    std::cout << "DetectionThread loop started" << std::endl;
-
-    while (!isAbortRequested())
+    try
     {
-        // Make sure a new image is actually in the buffer
-        while (!imageBuffer_->getAndClearSwapCount() && !isAbortRequested())
-        {
-            std::this_thread::sleep_for(std::chrono::milliseconds(1));
-        }
+        // Allocate and prepare objects for dewarping and detection
+        dewarpingParams = getDetectionDewarpingParameters(resolution, dewarpCount_);
+        detectionImages = getDetectionImages(detectionResolution, dewarpCount_);
+        dewarpingMappings = getDewarpingMappings(dewarpingParams, resolution, detectionResolution, dewarpCount_);
 
-        // We lock the image so we can use it without it being overwritten
-        imageBuffer_->lockInUse();
-        const Image& image = imageBuffer_->getLocked();
+        std::cout << "DetectionThread loop started" << std::endl;
 
-        // Dewarp each view, detect on each view and concatenate the results
-        for (int i = 0; i < dewarpCount_; ++i)
+        while (!isAbortRequested())
         {
-            dewarper_->dewarpImage(image, detectionImages[i], dewarpingMappings[i]);
-            synchronizer_->sync();
-            const std::vector<Rectangle> viewDetections = detector_->detectInImage(detectionImages[i]);
+            // Make sure a new image is actually in the buffer
+            while (!imageBuffer_->getAndClearSwapCount() && !isAbortRequested())
+            {
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            }
 
-            for (const Rectangle& detection : viewDetections)
+            // No need to do a detection if thread is aborted
+            if (isAbortRequested())
             {
-                detections.push_back(getAngleRectFromDewarpedImageRectangle(
-                    detection, dewarpingParams[i], detectionImages[i], fisheyeCenter, dewarpingConfig_.fisheyeAngle));
+                break;
             }
-        }
 
-        bool success = false;
+            // We lock the image so we can use it without it being overwritten
+            imageBuffer_->lockInUse();
+            const Image& image = imageBuffer_->getLocked();
 
-        // Output the detections, if queue is full keep trying...
-        while (!success && !isAbortRequested())
-        {
-            success = detectionQueue_->try_enqueue(std::move(detections));
+            // Dewarp each view, detect on each view and concatenate the results
+            for (int i = 0; i < dewarpCount_; ++i)
+            {
+                dewarper_->dewarpImage(image, detectionImages[i], dewarpingMappings[i]);
+                synchronizer_->sync();
+                const std::vector<Rectangle> viewDetections = detector_->detectInImage(detectionImages[i]);
+
+                for (const Rectangle& detection : viewDetections)
+                {
+                    detections.push_back(getAngleRectFromDewarpedImageRectangle(detection, dewarpingParams[i],
+                                                                                detectionImages[i], fisheyeCenter,
+                                                                                dewarpingConfig_.fisheyeAngle));
+                }
+            }
 
-            if (!success)
+            bool success = false;
+
+            // Output the detections, if queue is full keep trying...
+            while (!success && !isAbortRequested())
             {
-                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+                success = detectionQueue_->try_enqueue(std::move(detections));
+
+                if (!success)
+                {
+                    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+                }
             }
-        }
 
-        detections.clear();    // Just making sure...
+            detections.clear();    // Just making sure... Should be empty because of the std::move
+        }
+    }
+    catch (const std::exception& e)
+    {
+        std::cout << "Error in detection thread : " << e.what() << std::endl;
     }
 
     objectFactory_->deallocateObjectVector(detectionImages);

c_plus_plus/src/model/stream/video/impl/implementation_factory.cpp

@@ -182,7 +182,7 @@ std::unique_ptr<IImageConverter> ImplementationFactory::getImageConverter()
     return imageConverter;
 }
 
-std::unique_ptr<IVideoInput> ImplementationFactory::getFileImageReader(const std::string& imageFilePath,
+std::unique_ptr<IVideoInput> ImplementationFactory::getImageFileReader(const std::string& imageFilePath,
                                                                        ImageFormat format)
 {
     std::unique_ptr<IVideoInput> fileImageReader = nullptr;

c_plus_plus/src/model/stream/video/impl/implementation_factory.h

@@ -29,7 +29,7 @@ class ImplementationFactory
     std::unique_ptr<ISynchronizer> getSynchronizer();
     std::unique_ptr<ISynchronizer> getDetectionSynchronizer();
     std::unique_ptr<IImageConverter> getImageConverter();
-    std::unique_ptr<IVideoInput> getFileImageReader(const std::string& imageFilePath, ImageFormat format);
+    std::unique_ptr<IVideoInput> getImageFileReader(const std::string& imageFilePath, ImageFormat format);
     std::unique_ptr<IVideoInput> getCameraReader(const VideoConfig& cameraConfig);
 
    private:

c_plus_plus/src/model/stream/video/input/camera_reader.cpp

@@ -25,36 +25,41 @@ CameraReader::CameraReader(const VideoConfig& videoConfig, std::size_t bufferCou
     : videoConfig_(videoConfig)
     , images_(bufferCount, Image(videoConfig.resolution.width, videoConfig.resolution.height, videoConfig.imageFormat))
     , buffer_({})
+    , fd_(-1)
 {
-    fd_ = open(videoConfig_.deviceName.c_str(), O_RDWR);
+    buffer_.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+    buffer_.memory = V4L2_MEMORY_MMAP;
+}
+
+void CameraReader::open()
+{
+    fd_ = ::open(videoConfig_.deviceName.c_str(), O_RDWR);
 
     if (fd_ == ERROR_CODE)
     {
         throw std::runtime_error("Could not open camera " + videoConfig_.deviceName);
     }
 
-    buffer_.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-    buffer_.memory = V4L2_MEMORY_MMAP;
-
     checkCaps();
     setImageFormat();
-    requestBuffers(bufferCount);
+    requestBuffers(images_.size());
 
     // Queue a first capture for fast read
     v4l2_buffer buffer = buffer_;
     buffer.index = images_.current().index;
     queueCapture(buffer);
 }
 
-CameraReader::~CameraReader()
+void CameraReader::close()
 {
     for (std::size_t i = 0; i < images_.size(); ++i)
     {
         unmapBuffer(images_.current());
         images_.next();
     }
 
-    close(fd_);
+    ::close(fd_);
+    fd_ = -1;
 }
 
 const Image& CameraReader::readImage()
@@ -81,12 +86,12 @@ void CameraReader::queueCapture(v4l2_buffer& buffer)
 {
     if (xioctl(VIDIOC_QBUF, &buffer) == ERROR_CODE)
     {
-        perror("Failed to querry buffer");
+        throw std::runtime_error("Failed to querry camera buffer");
     }
 
     if (xioctl(VIDIOC_STREAMON, &buffer.type) == ERROR_CODE)
     {
-        perror("Failed to capture image");
+        throw std::runtime_error("Failed to capture camera image");
     }
 }
 
@@ -103,12 +108,12 @@ void CameraReader::dequeueCapture(v4l2_buffer& buffer)
 
     if (result == ERROR_CODE)
     {
-        perror("Error waiting for Frame");
+        throw std::runtime_error("Error waiting for camera frame");
     }
 
     if (xioctl(VIDIOC_DQBUF, &buffer) == ERROR_CODE)
     {
-        perror("Failed to retrieve Frame");
+        throw std::runtime_error("Failed to retrieve camera frame");
     }
 }
 

c_plus_plus/src/model/stream/video/input/camera_reader.h

@@ -13,8 +13,9 @@ class CameraReader : public IVideoInput
 {
    public:
     CameraReader(const VideoConfig& cameraConfig, std::size_t bufferCount);
-    virtual ~CameraReader();
 
+    void open() override;
+    void close() override;
     const Image& readImage() override;
 
    protected:

c_plus_plus/src/model/stream/video/input/cuda/cuda_camera_reader.cu

@@ -15,10 +15,6 @@ CudaCameraReader::CudaCameraReader(const VideoConfig& videoConfig)
     }
 
     checkCuda(cudaStreamCreate(&stream_));
-
-    // Read a frame to prepare the next read
-    nextImage_ = &CameraReader::readImage();
-    cudaMemcpyAsync(nextImage_->deviceData, nextImage_->hostData, nextImage_->size, cudaMemcpyHostToDevice, stream_);
 }
 
 CudaCameraReader::~CudaCameraReader()
@@ -32,6 +28,15 @@ CudaCameraReader::~CudaCameraReader()
     cudaStreamDestroy(stream_);
 }
 
+void CudaCameraReader::open()
+{
+    CameraReader::open();
+
+    // Read a frame to prepare the next read
+    nextImage_ = &CameraReader::readImage();
+    cudaMemcpyAsync(nextImage_->deviceData, nextImage_->hostData, nextImage_->size, cudaMemcpyHostToDevice, stream_);
+}
+
 const Image& CudaCameraReader::readImage()
 {
     if (nextImage_ == nullptr)

c_plus_plus/src/model/stream/video/input/cuda/cuda_camera_reader.h

@@ -14,6 +14,7 @@ class CudaCameraReader : public CameraReader
     explicit CudaCameraReader(const VideoConfig& videoConfig);
     virtual ~CudaCameraReader();
 
+    void open() override;
     const Image& readImage() override;
 
    private:

c_plus_plus/src/model/stream/video/input/i_video_input.h

@@ -9,6 +9,8 @@ class IVideoInput
 {
    public:
     virtual ~IVideoInput() = default;
+    virtual void open() = 0;
+    virtual void close() = 0;
     virtual const Image& readImage() = 0;
 };