esp::scene::SemanticScene class

Represents a scene with containing semantically annotated levels, regions and objects

Public static functions

static auto loadSemanticSceneDescriptor(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool
Attempt to load SemanticScene descriptor from an unknown file type.
static auto loadGibsonHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool
Attempt to load SemanticScene from a Gibson dataset house format file.
static auto loadHM3DHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool
Attempt to load SemanticScene from a HM3D dataset house format file.
static auto loadMp3dHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool
Attempt to load SemanticScene from a Matterport3D dataset house format file.
static auto loadReplicaHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool
Attempt to load SemanticScene from a Replica dataset house format file.
static auto buildCCBasedSemanticObjs(const std::vector<Mn::Vector3>& verts, const std::unordered_map<uint32_t, std::vector<std::set<uint32_t>>>& clrsToComponents, const std::shared_ptr<SemanticScene>& semanticScene) -> std::unordered_map<uint32_t, std::vector<std::shared_ptr<CCSemanticObject>>>
Builds a mapping of connected component-driven bounding boxes (via CCSemanticObject), keyed by criteria used to decide connectivity (the per-vertex attribute suche as color). If a semanticScene is passed, key for resultant map will be semanticID of object with specified color, otherwise key is hex color value.
static auto buildSemanticOBBs(const std::vector<Mn::Vector3>& verts, const std::vector<uint16_t>& vertSemanticIDs, const std::vector<std::shared_ptr<SemanticObject>>& ssdObjs, const std::string& msgPrefix) -> std::vector<uint32_t>
Build semantic OBBs based on presence of semantic IDs on vertices.
static auto buildSemanticOBBsFromCCs(const std::vector<Mn::Vector3>& verts, const std::unordered_map<uint32_t, std::vector<std::set<uint32_t>>>& clrsToComponents, const std::shared_ptr<SemanticScene>& semanticScene, float maxVolFraction, const std::string& msgPrefix) -> std::vector<uint32_t>
Build semantic object OBBs based on the accumulated per-semantic-color CCs, and some criteria specified in semanticScene .

Constructors, destructors, conversion operators

~SemanticScene()

Public functions

auto aabb() const -> box3f
return axis aligned bounding box of this House
auto count(const std::string& element) const -> int
return total number of given element type
auto categories() const -> const std::vector<std::shared_ptr<SemanticCategory>>&
return all SemanticCategories of objects in this House
auto levels() const -> const std::vector<std::shared_ptr<SemanticLevel>>&
return all Levels in this House
auto regions() const -> const std::vector<std::shared_ptr<SemanticRegion>>&
return all Regions in this House
auto objects() const -> const std::vector<std::shared_ptr<SemanticObject>>&
return all Objects in this House
auto getSemanticIndexMap() const -> const std::unordered_map<int, int>&
auto semanticIndexToObjectIndex(int maskIndex) const -> int
auto hasVertColorsDefined() const -> bool
Whether the source file assigns colors to verts for Semantic Mesh.
auto getSemanticColorMap() const -> const std::vector<Mn::Vector3ub>&
return a read-only reference to the semantic color map, where value is annotation color and index corresponds to id for that color. (HM3D only currently)
auto getSemanticColorToIdAndRegionMap() const -> const std::unordered_map<uint32_t, std::pair<int, int>>&
return a read-only reference to a mapping of semantic color-as-integer to id and region id.(HM3D only currently)
auto buildBBoxFromVertColors() const -> bool
whether or not we should build bounding boxes around vertex annotations on semantic asset load. Currently used for HM3D.
auto CCFractionToUseForBBox() const -> float
What fraction of largest connected component bbox to use for bbox generation. Will always use single largest, along with this fraction of remaining. If set to 0.0, use all CCs (i.e. will bypass CC calc and just use naive vert position mechanism for bbox), if set to 1.0, only use single CC with largest volume. Only used if needBBoxFromVertColors_ is true.

Protected static functions

static auto checkFileExists(const std::string& filename, const std::string& srcFunc) -> bool
Verify a requested file exists.
static auto buildHM3DHouse(std::ifstream& ifs, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool
Build the HM3D semantic data from the passed file stream. File being streamed is expected to be appropriate format.
static auto buildMp3dHouse(std::ifstream& ifs, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool
Build the mp3 semantic data from the passed file stream. File being streamed is expected to be appropriate format.
static auto buildGibsonHouse(const io::JsonDocument& jsonDoc, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool
Build SemanticScene from a Gibson dataset house JSON. JSON is expected to have been verified already.
static auto buildReplicaHouse(const io::JsonDocument& jsonDoc, SemanticScene& scene, bool objectsExist, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool
Build SemanticScene from a Replica dataset house JSON. JSON is expected to have been verified already.

Protected variables

bool hasVertColors_
bool needBBoxFromVertColors_
float ccLargestVolToUseForBBox_
Fraction of vert count for largest CC to use for BBox synth. 0->use all (bypass CC calc), 1->use only largest CC.
std::string name_
std::string label_
box3f bbox_
std::map<std::string, int> elementCounts_
std::vector<std::shared_ptr<SemanticCategory>> categories_
std::vector<std::shared_ptr<SemanticLevel>> levels_
std::vector<std::shared_ptr<SemanticRegion>> regions_
std::vector<std::shared_ptr<SemanticObject>> objects_
std::unordered_map<int, int> segmentToObjectIndex_
map from combined region-segment id to objectIndex for semantic mesh
std::vector<Mn::Vector3ub> semanticColorMapBeingUsed_
List of mapped vertex colors, where index corresponds to object Index/semantic ID (HM3D only currently)
std::unordered_map<uint32_t, std::pair<int, int>> semanticColorToIdAndRegion_
Map of integer color to segment and region ids. Used for transforming provided vertex colors. (HM3D only currently)

Function documentation

static bool esp::scene::SemanticScene::loadSemanticSceneDescriptor(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY))

Attempt to load SemanticScene descriptor from an unknown file type.

Parameters
filename the name of the semantic scene descriptor (house file) to attempt to load
scene reference to sceneNode to assign semantic scene to
rotation rotation to apply to semantic scene upon load.
Returns successfully loaded

static bool esp::scene::SemanticScene::loadGibsonHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY))

Attempt to load SemanticScene from a Gibson dataset house format file.

Parameters
filename the name of the semantic scene descriptor (house file) to attempt to load
scene reference to sceneNode to assign semantic scene to
rotation rotation to apply to semantic scene upon load.
Returns successfully loaded

static bool esp::scene::SemanticScene::loadHM3DHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY))

Attempt to load SemanticScene from a HM3D dataset house format file.

Parameters
filename the name of the semantic scene descriptor (house file) to attempt to load
scene reference to sceneNode to assign semantic scene to
rotation rotation to apply to semantic scene upon load (currently not used for HM3D)
Returns successfully loaded

static bool esp::scene::SemanticScene::loadMp3dHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY))

Attempt to load SemanticScene from a Matterport3D dataset house format file.

Parameters
filename the name of the semantic scene descriptor (house file) to attempt to load
scene reference to sceneNode to assign semantic scene to
rotation rotation to apply to semantic scene upon load.
Returns successfully loaded

static bool esp::scene::SemanticScene::loadReplicaHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY))

Attempt to load SemanticScene from a Replica dataset house format file.

Parameters
filename the name of the semantic scene descriptor (house file) to attempt to load
scene reference to sceneNode to assign semantic scene to
rotation rotation to apply to semantic scene upon load.
Returns successfully loaded

static std::unordered_map<uint32_t, std::vector<std::shared_ptr<CCSemanticObject>>> esp::scene::SemanticScene::buildCCBasedSemanticObjs(const std::vector<Mn::Vector3>& verts, const std::unordered_map<uint32_t, std::vector<std::set<uint32_t>>>& clrsToComponents, const std::shared_ptr<SemanticScene>& semanticScene)

Builds a mapping of connected component-driven bounding boxes (via CCSemanticObject), keyed by criteria used to decide connectivity (the per-vertex attribute suche as color). If a semanticScene is passed, key for resultant map will be semanticID of object with specified color, otherwise key is hex color value.

Parameters
verts Ref to the vertex buffer holding all vertex positions in the mesh.
clrsToComponents an unordered map, keyed by tag/color value encoded as uint, where the value is a vector of all sets of CCs consisting of verts with specified tag/"color". (see findCCsByGivenColor).
semanticScene The SSD for the current semantic mesh. Used to query semantic objs. If nullptr, this function returns hex-color-keyed map, otherwise returns SemanticID-keyed map.
Returns A map keyed by a representattion of the per-vertex "color" where each entry contains a vector of values for all the CCs of verts having the "color" attribute specified by the key. Each element in the vector is a smart pointer to a CCSemanticObject, being used to facilitate collecting pertinent data.

static std::vector<uint32_t> esp::scene::SemanticScene::buildSemanticOBBs(const std::vector<Mn::Vector3>& verts, const std::vector<uint16_t>& vertSemanticIDs, const std::vector<std::shared_ptr<SemanticObject>>& ssdObjs, const std::string& msgPrefix)

Build semantic OBBs based on presence of semantic IDs on vertices.

Parameters
verts Ref to the vertex buffer holding all vertex positions in the mesh.
vertSemanticIDs Ref to per-vertex semantic IDs persent on source mesh, both known in semantic scene descriptor, and unknown. Known IDs are expected to start at 1 and be contiguous, followed by unknown semantic IDs
ssdObjs The known semantic scene descriptor objects for the mesh
msgPrefix Debug message prefix, referencing caller.
Returns vector of semantic object IDXs that have no vertex mapping/presence in the source mesh.

static std::vector<uint32_t> esp::scene::SemanticScene::buildSemanticOBBsFromCCs(const std::vector<Mn::Vector3>& verts, const std::unordered_map<uint32_t, std::vector<std::set<uint32_t>>>& clrsToComponents, const std::shared_ptr<SemanticScene>& semanticScene, float maxVolFraction, const std::string& msgPrefix)

Build semantic object OBBs based on the accumulated per-semantic-color CCs, and some criteria specified in semanticScene .

Parameters
verts Ref to the vertex buffer holding all vertex positions in the mesh.
clrsToComponents an unordered map, keyed by tag/color value encoded as uint, where the value is a vector of all sets of CCs consisting of verts with specified tag/"color". (see findCCsByGivenColor).
semanticScene The SSD for the current semantic mesh. Used to query semantic objs. If nullptr, this function returns hex-color-keyed map, otherwise returns SemanticID-keyed map.
maxVolFraction Fraction of maximum volume bbox CC to include in bbox calc.
msgPrefix Debug message prefix, referencing caller.
Returns vector of semantic object IDXs that have no vertex mapping/presence in the source mesh.

int esp::scene::SemanticScene::semanticIndexToObjectIndex(int maskIndex) const

convert semantic mesh mask index to object index or ID_UNDEFINED if not mapped

static bool esp::scene::SemanticScene::checkFileExists(const std::string& filename, const std::string& srcFunc) protected

Verify a requested file exists.

Parameters
filename the file to attempt to load
srcFunc calling function name to be displayed in failure message
Returns whether found or not

static bool esp::scene::SemanticScene::buildHM3DHouse(std::ifstream& ifs, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) protected

Build the HM3D semantic data from the passed file stream. File being streamed is expected to be appropriate format.

Parameters
ifs The opened file stream describing the HM3D semantic annotations.
scene reference to sceneNode to assign semantic scene to
rotation rotation to apply to semantic scene upon load (currently not used for HM3D)
Returns successfully built. Currently only returns true, but retaining return value for future support.

static bool esp::scene::SemanticScene::buildMp3dHouse(std::ifstream& ifs, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) protected

Build the mp3 semantic data from the passed file stream. File being streamed is expected to be appropriate format.

Parameters
ifs The opened file stream describing the Mp3d semantic annotations.
scene reference to sceneNode to assign semantic scene to
rotation rotation to apply to semantic scene upon load.
Returns successfully built. Currently only returns true, but retaining return value for future support.

static bool esp::scene::SemanticScene::buildGibsonHouse(const io::JsonDocument& jsonDoc, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) protected

Build SemanticScene from a Gibson dataset house JSON. JSON is expected to have been verified already.

Parameters
jsonDoc the JSON document describing the semantic annotations.
scene reference to sceneNode to assign semantic scene to
rotation rotation to apply to semantic scene upon load.
Returns successfully built. Currently only returns true, but retaining return value for future support.

static bool esp::scene::SemanticScene::buildReplicaHouse(const io::JsonDocument& jsonDoc, SemanticScene& scene, bool objectsExist, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) protected

Build SemanticScene from a Replica dataset house JSON. JSON is expected to have been verified already.

Parameters
jsonDoc the JSON document describing the semantic annotations.
scene reference to sceneNode to assign semantic scene to
objectsExist whether objects cell exists in json. This cell will not exist in ReplicaCAD semantic lexicon.
rotation rotation to apply to semantic scene upon load.
Returns successfully built. Currently only returns true, but retaining return value for future support.

Variable documentation

bool esp::scene::SemanticScene::needBBoxFromVertColors_ protected

@Brief whether or not we should build bounding boxes around vertex annotations on semantic asset load. Currently used for HM3D.