example_reconstructmesdk_remote_reconstruction.cpp

Content

This is a one minute example showing remote controlled reconstruction.

Boost is only used to generate examples and is not necessary for working with this SDK.

#include <boost/test/unit_test.hpp>
#include <iostream>
#include <conio.h>

#include <reconstructmesdk/reme.h>
BOOST_AUTO_TEST_SUITE(example_reconstructmesdk)

/*
    remote_sensor opens a physical sensor and streams its data via ReMe remote.

    Once the sensor has opened, it announces itself plus the physical sensor config 
    at the topic "sensor/announce". 
    
    It then starts to publish RGBD data at topic "sensor/rgbd". To save bandwidth the 
    RGB image is transmitted every 30th frame only. The application then listens for reconstruction 
    images on "engine/volume". Once it receives a reconstruction image result, it will display the result. 

    Once you close the viewer a signal will be sent to the engine to stop scanning. The engine will
    then respond with a mesh on "engine/mesh". Once received it will be displayed.
*/
BOOST_AUTO_TEST_CASE(remote_sensor)
{
    // Create the context. Since reconstruction is performed remotely, no
    // need to compile OpenCL context.

    reme_context_t c;
    reme_context_create(&c);
    reme_context_set_log_callback(c, reme_default_log_callback, 0);

    // Open the physical sensor

    reme_sensor_t s;
    reme_sensor_create(c, "openni;mskinect", true, &s);
    reme_sensor_open(c, s);

    // Connect to remote service

    reme_remote_t r;
    reme_remote_create(c, &r);
    reme_remote_connect(c, r, "tcp://127.0.0.1:50000", 10000);

    // Announce sensor and config at "sensor/announce"

    reme_bag_builder_t bb;
    reme_bag_builder_create(c, &bb);

    reme_bag_builder_set_string(c, bb, "name", "SensorA");
    reme_bag_builder_set_sensor_config(c, bb, "config", s);

    reme_bag_t bAnnounce;
    reme_bag_create(c, &bAnnounce);
    reme_bag_builder_finalize(c, bb, bAnnounce);
    reme_remote_send(c, r, "sensor/announce", bAnnounce);
    
    // Subscribe to the topics we are interested in
    //
    // Topic "engine/volume" will provide the reconstruction image. We expect this
    // bag to be rather large in terms of byte size, which is why we limit the broker
    // queue size to 1, to avoid the accumulation of images causing delays or possible
    // memory allocation issues.

    reme_remote_subscribe(c, r, "system/shutdown");
    reme_remote_subscribe_advanced(c, r, "engine/volume", "", true, 0, 1); 
    reme_remote_subscribe(c, r, "engine/mesh");
    
    reme_bag_t bSend, bRecv;
    reme_bag_create(c, &bSend);
    reme_bag_create(c, &bRecv);

    reme_image_t depth, aux, volume;    
    reme_image_create(c, &depth);
    reme_image_create(c, &aux);
    reme_image_create(c, &volume);

    reme_viewer_t viewerImages, viewerSurface;
    reme_viewer_create_image(c, "This is ReconstructMe SDK", &viewerImages);
    reme_viewer_add_image(c, viewerImages, volume);    

    reme_surface_t surface;
    reme_surface_create(c, &surface);

    // Enter the main loop

    bool done = false;
    bool viewerClosed = false;
    bool meshRequested = false;
    int frameCounter = 0;
    while (!done) {
        ++frameCounter;

        // Grab from the pyhsical sensor and send data at topic "sensor/rgbd"
        if (REME_SUCCESS(reme_sensor_grab(c, s))) {
            reme_sensor_prepare_image(c, s, REME_IMAGE_RAW_DEPTH);
            reme_sensor_get_image(c, s, REME_IMAGE_RAW_DEPTH, depth);
            reme_bag_builder_set_image(c, bb, "depth", depth, true);

            if (frameCounter % 30 == 0) {
                reme_sensor_prepare_image(c, s, REME_IMAGE_RAW_AUX);
                reme_sensor_get_image(c, s, REME_IMAGE_RAW_AUX, aux);
                reme_bag_builder_set_image(c, bb, "aux", aux, true);
            }

            reme_bag_builder_finalize(c, bb, bSend);
            reme_remote_send(c, r, "sensor/rgbd", bSend);
        }
        else if (!meshRequested) {
            reme_remote_notify(c, r, "engine/requestMesh");
            meshRequested = true;
        }

        // Test if any of our subscribed topics have been triggered
         
        int topicIndex = -1;
        const char *topicName;
        reme_remote_recv(c, r, &topicIndex, &topicName, bRecv, 0);

        if (topicIndex == -1) {
            continue;
        }

        if (strcmp(topicName, "system/shutdown") == 0) 
        {
            done = true;
        }
        else if (strcmp(topicName, "engine/volume") == 0) 
        {
            reme_bag_get_image(c, bRecv, "volume", volume);
            reme_viewer_update(c, viewerImages);

            // In case viewer was closed by user, request engine to stop
            // scanning.

            reme_viewer_is_closed(c, viewerImages, &viewerClosed);
            if (viewerClosed && !meshRequested) {
                reme_remote_notify(c, r, "engine/requestMesh");
                meshRequested = true;
            }   
        }
        else if (strcmp(topicName, "engine/mesh") == 0) 
        {
            reme_bag_get_surface(c, bRecv, "surface", surface);
            reme_viewer_create_surface(c, surface, "This is ReconstructMe SDK", &viewerSurface);    
            reme_viewer_wait(c, viewerSurface);

            reme_remote_notify(c, r, "system/shutdown");
        }
    }

    reme_context_destroy(&c);

}


/*
    remote_reconstruction performs reconstruction on data sent via ReMe remote.

    The application first waits for a announcement of an RGBD sensor on ReMe remote.
    Once a sensor is available, it subscribes to its data stream.

    When new data from the sensor arrives a reconstruction step will be performed.
    The resulting volume rendered reconstruction image will be sent back to the 
    application streaming the sensor data for visualization purposes.
*/
BOOST_AUTO_TEST_CASE(remote_reconstruction)
{
    // Create the context. Context will be prepared for colorized reconstruction

    reme_context_t c;
    reme_context_create(&c);
    reme_context_set_log_callback(c, reme_default_log_callback, 0);

    reme_options_t o;
    reme_options_create(c, &o);
    reme_context_bind_reconstruction_options(c, o);    
    reme_options_set_bool(c, o, "data_integration.use_colors", true);
    reme_context_compile(c);

    // Create a reconstruction volume

    reme_volume_t v;
    reme_volume_create(c, &v);

    // Create a non-physical sensor

    reme_sensor_t s;
    reme_sensor_create(c, "external", false, &s);    

    // Create ReMe remote service

    reme_remote_t r;
    reme_remote_create(c, &r);
    reme_remote_bind(c, r, "tcp://127.0.0.1:50000");

    // Subscribe to the topics we are initially interested in.

    reme_remote_subscribe(c, r, "system/shutdown");
    reme_remote_subscribe(c, r, "sensor/announce");
    reme_remote_subscribe(c, r, "engine/requestMesh");

    reme_bag_builder_t bb;
    reme_bag_builder_create(c, &bb);

    reme_bag_t b;
    reme_bag_create(c, &b);

    reme_image_t depth, aux, volume, rawDepth, rawAux;
    reme_image_create(c, &rawDepth);
    reme_image_create(c, &rawAux);
    reme_image_create(c, &depth);
    reme_image_create(c, &aux);
    reme_image_create(c, &volume);

    reme_surface_t surface;
    reme_surface_create(c, &surface);

    // Enter main loop

    bool done = false;
    while (!done) {

        // Test if any of our subscribed topics have been triggered

        int topicIndex = -1;
        const char *topicName = 0;
        reme_remote_recv(c, r, &topicIndex, &topicName, b, 100);

        if (topicIndex == -1) {
            continue;
        }

        if (strcmp(topicName, "system/shutdown") == 0) 
        {
            done = true;
        }
        else if (strcmp(topicName, "sensor/announce") == 0) 
        {
            const char *sensorName;
            reme_bag_get_string(c, b, "name", &sensorName);
            printf("Found a sensor: %s\n", sensorName);

            // Use the physical sensor config to parametrize the virtual sensor
            
            reme_bag_get_sensor_config(c, b, "config", s);
            reme_sensor_open(c, s);

            // Set reconstruction options

            reme_sensor_set_prescan_position(c, s, REME_SENSOR_POSITION_INFRONT);
            reme_sensor_bind_render_options(c, s, o);
            reme_options_set(c, o, "shade_mode", "SHADE_COLORS");

            // Subscribe to sensor RGBD data topic. We expect this
            // topic to be rather large in terms of byte size, which is why we limit the broker
            // queue size to 1, to avoid the accumulation of images causing delays or possible
            // memory allocation issues.

            reme_remote_subscribe_advanced(c, r, "sensor/rgbd", "", true, 0, 1);

        } 
        else if (strcmp(topicName, "engine/requestMesh") == 0) 
        {
  
            reme_surface_generate(c, surface, v);
            reme_surface_colorize_vertices(c, surface, v);
            reme_bag_builder_set_surface(c, bb, "surface", surface);
            reme_bag_builder_finalize(c, bb, b);
            reme_remote_send(c, r, "engine/mesh", b);
            
        }
        else if (strcmp(topicName, "sensor/rgbd") == 0)
        {
            // Read RGBD data sent and move it into the virtual sensor
            reme_sensor_grab(c, s);

            reme_bag_get_image(c, b, "depth", depth);
            reme_sensor_prepare_image(c, s, REME_IMAGE_RAW_DEPTH);
            reme_sensor_get_image(c, s, REME_IMAGE_RAW_DEPTH, rawDepth);
            reme_image_copy(c, depth, rawDepth);

            bool hasAux = false;
            reme_bag_has_key(c, b, "aux", &hasAux);
            if (hasAux) {
                reme_bag_get_image(c, b, "aux", aux);
                reme_sensor_prepare_image(c, s, REME_IMAGE_RAW_AUX);
                reme_sensor_get_image(c, s, REME_IMAGE_RAW_AUX, rawAux);
                reme_image_copy(c, aux, rawAux);
            }

            // Perform tracking and volume update step.

            reme_sensor_prepare_image(c, s, REME_IMAGE_VOLUME);
            if (REME_SUCCESS(reme_sensor_track_position(c, s))) {
                reme_sensor_update_volume_selectively(c, s, true, hasAux);
            }

            // Retrieve the reconstruction image and send it back to the sensor.
            reme_sensor_get_image(c, s, REME_IMAGE_VOLUME, volume);
            reme_bag_builder_set_image(c, bb, "volume", volume, true);
            reme_bag_builder_finalize(c, bb, b);
            reme_remote_send(c, r, "engine/volume", b);
        }
   }

   reme_context_destroy(&c);

}




BOOST_AUTO_TEST_SUITE_END()