esp::scene::SemanticScene class
#include <esp/scene/SemanticScene.h>

Represents a scene with containing semantically annotated levels, regions and objects

Public static functions

static auto loadSemanticSceneDescriptor(const std::shared_ptr<metadata::attributes::SemanticAttributes>& semanticAttr, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool: Attempt to load SemanticScene descriptor from an unknown file type.
static auto loadGibsonHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool: Attempt to load SemanticScene from a Gibson dataset house format file.
static auto loadHM3DHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool: Attempt to load SemanticScene from a HM3D dataset house format file.
static auto loadMp3dHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool: Attempt to load SemanticScene from a Matterport3D dataset house format file.
static auto loadReplicaHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool: Attempt to load SemanticScene from a Replica dataset house format file.
static auto buildCCBasedSemanticObjs(const std::vector<Mn::Vector3>& verts, const std::unordered_map<uint32_t, std::vector<std::set<uint32_t>>>& clrsToComponents, const std::shared_ptr<SemanticScene>& semanticScene) -> std::unordered_map<uint32_t, std::vector<std::shared_ptr<CCSemanticObject>>>: Builds a mapping of connected component-driven bounding boxes (via CCSemanticObject), keyed by criteria used to decide connectivity (the per-vertex attribute suche as color). If a semanticScene is passed, key for resultant map will be semanticID of object with specified color, otherwise key is hex color value.
static auto buildSemanticOBBs(const std::vector<Mn::Vector3>& verts, const std::vector<uint16_t>& vertSemanticIDs, const std::vector<std::shared_ptr<SemanticObject>>& ssdObjs, const std::string& msgPrefix) -> std::vector<uint32_t>: Build semantic OBBs based on presence of semantic IDs on vertices.
static auto buildSemanticOBBsFromCCs(const std::vector<Mn::Vector3>& verts, const std::unordered_map<uint32_t, std::vector<std::set<uint32_t>>>& clrsToComponents, const std::shared_ptr<SemanticScene>& semanticScene, float maxVolFraction, const std::string& msgPrefix) -> std::vector<uint32_t>: Build semantic object OBBs based on the accumulated per-semantic-color CCs, and some criteria specified in semanticScene .

Constructors, destructors, conversion operators

~SemanticScene()

Public functions

auto aabb() const -> box3f: return axis aligned bounding box of this House
auto count(const std::string& element) const -> int: return total number of given element type
auto categories() const -> const std::vector<std::shared_ptr<SemanticCategory>>&: return all SemanticCategories of objects in this House
auto levels() const -> const std::vector<std::shared_ptr<SemanticLevel>>&: return all Levels in this House
auto regions() const -> const std::vector<std::shared_ptr<SemanticRegion>>&: return all Regions in this House
auto objects() const -> const std::vector<std::shared_ptr<SemanticObject>>&: return all Objects in this House
auto getSemanticIndexMap() const -> const std::unordered_map<int, int>&
auto semanticIndexToObjectIndex(int maskIndex) const -> int
auto hasVertColorsDefined() const -> bool: Whether the source file assigns colors to verts for Semantic Mesh.
auto getSemanticColorMap() const -> const std::vector<Mn::Vector3ub>&: return a read-only reference to the semantic color map, where value is annotation color and index corresponds to id for that color. (HM3D only currently)
auto getSemanticColorToIdAndRegionMap() const -> const std::unordered_map<uint32_t, std::pair<int, int>>&: return a read-only reference to a mapping of semantic color-as-integer to id and region id.(HM3D only currently)
auto buildBBoxFromVertColors() const -> bool: whether or not we should build bounding boxes around vertex annotations on semantic asset load. Currently used for HM3D.
auto CCFractionToUseForBBox() const -> float: What fraction of largest connected component bbox to use for bbox generation. Will always use single largest, along with this fraction of remaining. If set to 0.0, use all CCs (i.e. will bypass CC calc and just use naive vert position mechanism for bbox), if set to 1.0, only use single CC with largest volume. Only used if needBBoxFromVertColors_ is true.
auto getRegionsForPoint(const Mn::Vector3& point) const -> std::vector<int>: Compute all SemanticRegions which contain the point and return a list of indices for regions in this SemanticScene.
auto getWeightedRegionsForPoint(const Mn::Vector3& point) const -> std::vector<std::pair<int, double>>: Compute all the SemanticRegions that contain the passed point, and return a vector of indices and weights for each region, where the weights are inverted area of the region (smaller regions weighted higher)
auto getRegionsForPoints(const std::vector<Mn::Vector3>& points) const -> std::vector<std::pair<int, double>>: Compute SemanticRegion containment for a set of points. It is assumed the set of points belong to the same construct (i.e. points from an individual object's mesh)

Protected static functions

static auto checkFileExists(const std::string& filename, const std::string& srcFunc) -> bool: Verify a requested file exists.
static auto buildHM3DHouse(std::ifstream& ifs, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool: Build the HM3D semantic data from the passed file stream. File being streamed is expected to be appropriate format.
static auto buildMp3dHouse(std::ifstream& ifs, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool: Build the mp3 semantic data from the passed file stream. File being streamed is expected to be appropriate format.
static auto buildGibsonHouse(const io::JsonDocument& jsonDoc, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool: Build SemanticScene from a Gibson dataset house JSON. JSON is expected to have been verified already.
static auto buildReplicaHouse(const io::JsonDocument& jsonDoc, SemanticScene& scene, bool objectsExist, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) -> bool: Build SemanticScene from a Replica dataset house JSON. JSON is expected to have been verified already.

Protected variables

bool hasVertColors_
bool needBBoxFromVertColors_
float ccLargestVolToUseForBBox_: Fraction of vert count for largest CC to use for BBox synth. 0->use all (bypass CC calc), 1->use only largest CC.
std::string name_
std::string label_
box3f bbox_
std::map<std::string, int> elementCounts_
std::vector<std::shared_ptr<SemanticCategory>> categories_
std::vector<std::shared_ptr<SemanticLevel>> levels_
std::vector<std::shared_ptr<SemanticRegion>> regions_
std::vector<std::shared_ptr<SemanticObject>> objects_
std::unordered_map<int, int> segmentToObjectIndex_: map from combined region-segment id to objectIndex for semantic mesh
std::vector<Mn::Vector3ub> semanticColorMapBeingUsed_: List of mapped vertex colors, where index corresponds to object Index/semantic ID (HM3D only currently)
std::unordered_map<uint32_t, std::pair<int, int>> semanticColorToIdAndRegion_: Map of integer color to segment and region ids. Used for transforming provided vertex colors. (HM3D only currently)

Function documentation

static bool esp::scene::SemanticScene::loadSemanticSceneDescriptor(const std::shared_ptr<metadata::attributes::SemanticAttributes>& semanticAttr, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY))

Attempt to load SemanticScene descriptor from an unknown file type.

Parameters
semanticAttr	The semantic attributes containing a reference to the name of the semantic scene descriptor (house file) to attempt to load, along with informationd describing semantic constructions (region annotations).
scene	reference to sceneNode to assign semantic scene to
rotation	rotation to apply to semantic scene upon load.
Returns	successfully loaded

static bool esp::scene::SemanticScene::loadGibsonHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY))

Attempt to load SemanticScene from a Gibson dataset house format file.

Parameters
filename	the name of the semantic scene descriptor (house file) to attempt to load
scene	reference to sceneNode to assign semantic scene to
rotation	rotation to apply to semantic scene upon load.
Returns	successfully loaded

static bool esp::scene::SemanticScene::loadHM3DHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY))

Attempt to load SemanticScene from a HM3D dataset house format file.

Parameters
filename	the name of the semantic scene descriptor (house file) to attempt to load
scene	reference to sceneNode to assign semantic scene to
rotation	rotation to apply to semantic scene upon load (currently not used for HM3D)
Returns	successfully loaded

static bool esp::scene::SemanticScene::loadMp3dHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY))

Attempt to load SemanticScene from a Matterport3D dataset house format file.

Parameters
filename	the name of the semantic scene descriptor (house file) to attempt to load
scene	reference to sceneNode to assign semantic scene to
rotation	rotation to apply to semantic scene upon load.
Returns	successfully loaded

static bool esp::scene::SemanticScene::loadReplicaHouse(const std::string& filename, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY))

Attempt to load SemanticScene from a Replica dataset house format file.

Parameters
filename	the name of the semantic scene descriptor (house file) to attempt to load
scene	reference to sceneNode to assign semantic scene to
rotation	rotation to apply to semantic scene upon load.
Returns	successfully loaded

static std::unordered_map<uint32_t, std::vector<std::shared_ptr<CCSemanticObject>>> esp::scene::SemanticScene::buildCCBasedSemanticObjs(const std::vector<Mn::Vector3>& verts, const std::unordered_map<uint32_t, std::vector<std::set<uint32_t>>>& clrsToComponents, const std::shared_ptr<SemanticScene>& semanticScene)

Builds a mapping of connected component-driven bounding boxes (via CCSemanticObject), keyed by criteria used to decide connectivity (the per-vertex attribute suche as color). If a semanticScene is passed, key for resultant map will be semanticID of object with specified color, otherwise key is hex color value.

Parameters
verts	Ref to the vertex buffer holding all vertex positions in the mesh.
clrsToComponents	an unordered map, keyed by tag/color value encoded as uint, where the value is a vector of all sets of CCs consisting of verts with specified tag/"color". (see findCCsByGivenColor).
semanticScene	The SSD for the current semantic mesh. Used to query semantic objs. If nullptr, this function returns hex-color-keyed map, otherwise returns SemanticID-keyed map.
Returns	A map keyed by a representattion of the per-vertex "color" where each entry contains a vector of values for all the CCs of verts having the "color" attribute specified by the key. Each element in the vector is a smart pointer to a CCSemanticObject, being used to facilitate collecting pertinent data.

static std::vector<uint32_t> esp::scene::SemanticScene::buildSemanticOBBs(const std::vector<Mn::Vector3>& verts, const std::vector<uint16_t>& vertSemanticIDs, const std::vector<std::shared_ptr<SemanticObject>>& ssdObjs, const std::string& msgPrefix)

Build semantic OBBs based on presence of semantic IDs on vertices.

Parameters
verts	Ref to the vertex buffer holding all vertex positions in the mesh.
vertSemanticIDs	Ref to per-vertex semantic IDs persent on source mesh, both known in semantic scene descriptor, and unknown. Known IDs are expected to start at 1 and be contiguous, followed by unknown semantic IDs
ssdObjs	The known semantic scene descriptor objects for the mesh
msgPrefix	Debug message prefix, referencing caller.
Returns	vector of semantic object IDXs that have no vertex mapping/presence in the source mesh.

static std::vector<uint32_t> esp::scene::SemanticScene::buildSemanticOBBsFromCCs(const std::vector<Mn::Vector3>& verts, const std::unordered_map<uint32_t, std::vector<std::set<uint32_t>>>& clrsToComponents, const std::shared_ptr<SemanticScene>& semanticScene, float maxVolFraction, const std::string& msgPrefix)

Build semantic object OBBs based on the accumulated per-semantic-color CCs, and some criteria specified in semanticScene .

Parameters
verts	Ref to the vertex buffer holding all vertex positions in the mesh.
clrsToComponents	an unordered map, keyed by tag/color value encoded as uint, where the value is a vector of all sets of CCs consisting of verts with specified tag/"color". (see findCCsByGivenColor).
semanticScene	The SSD for the current semantic mesh. Used to query semantic objs. If nullptr, this function returns hex-color-keyed map, otherwise returns SemanticID-keyed map.
maxVolFraction	Fraction of maximum volume bbox CC to include in bbox calc.
msgPrefix	Debug message prefix, referencing caller.
Returns	vector of semantic object IDXs that have no vertex mapping/presence in the source mesh.

int esp::scene::SemanticScene::semanticIndexToObjectIndex(int maskIndex) const

convert semantic mesh mask index to object index or ID_UNDEFINED if not mapped

std::vector<int> esp::scene::SemanticScene::getRegionsForPoint(const Mn::Vector3& point) const

Compute all SemanticRegions which contain the point and return a list of indices for regions in this SemanticScene.

Parameters
point	The query point.
Returns	A list of indices for regions which contain the point.

std::vector<std::pair<int, double>> esp::scene::SemanticScene::getWeightedRegionsForPoint(const Mn::Vector3& point) const

Compute all the SemanticRegions that contain the passed point, and return a vector of indices and weights for each region, where the weights are inverted area of the region (smaller regions weighted higher)

Parameters
point	The query point.
Returns	std::vector<std::pair<int, float>> A sorted list of tuples containing region index and inverse area of that region

std::vector<std::pair<int, double>> esp::scene::SemanticScene::getRegionsForPoints(const std::vector<Mn::Vector3>& points) const

Compute SemanticRegion containment for a set of points. It is assumed the set of points belong to the same construct (i.e. points from an individual object's mesh)

Parameters
points	A set of points to test for semantic containment.
Returns	std::vector<std::pair<int, float>> A sorted list of tuples containing region index and percentage of input points contained in that region.

static bool esp::scene::SemanticScene::checkFileExists(const std::string& filename, const std::string& srcFunc) protected

Verify a requested file exists.

Parameters
filename	the file to attempt to load
srcFunc	calling function name to be displayed in failure message
Returns	whether found or not

static bool esp::scene::SemanticScene::buildHM3DHouse(std::ifstream& ifs, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) protected

Build the HM3D semantic data from the passed file stream. File being streamed is expected to be appropriate format.

Parameters
ifs	The opened file stream describing the HM3D semantic annotations.
scene	reference to sceneNode to assign semantic scene to
rotation	rotation to apply to semantic scene upon load (currently not used for HM3D)
Returns	successfully built. Currently only returns true, but retaining return value for future support.

static bool esp::scene::SemanticScene::buildMp3dHouse(std::ifstream& ifs, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) protected

Build the mp3 semantic data from the passed file stream. File being streamed is expected to be appropriate format.

Parameters
ifs	The opened file stream describing the Mp3d semantic annotations.
scene	reference to sceneNode to assign semantic scene to
rotation	rotation to apply to semantic scene upon load.
Returns	successfully built. Currently only returns true, but retaining return value for future support.

static bool esp::scene::SemanticScene::buildGibsonHouse(const io::JsonDocument& jsonDoc, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) protected

Build SemanticScene from a Gibson dataset house JSON. JSON is expected to have been verified already.

Parameters
jsonDoc	the JSON document describing the semantic annotations.
scene	reference to sceneNode to assign semantic scene to
rotation	rotation to apply to semantic scene upon load.
Returns	successfully built. Currently only returns true, but retaining return value for future support.

static bool esp::scene::SemanticScene::buildReplicaHouse(const io::JsonDocument& jsonDoc, SemanticScene& scene, bool objectsExist, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)) protected

Build SemanticScene from a Replica dataset house JSON. JSON is expected to have been verified already.

Parameters
jsonDoc	the JSON document describing the semantic annotations.
scene	reference to sceneNode to assign semantic scene to
objectsExist	whether objects cell exists in json. This cell will not exist in ReplicaCAD semantic lexicon.
rotation	rotation to apply to semantic scene upon load.
Returns	successfully built. Currently only returns true, but retaining return value for future support.

Variable documentation

bool esp::scene::SemanticScene::needBBoxFromVertColors_ protected

@Brief whether or not we should build bounding boxes around vertex annotations on semantic asset load. Currently used for HM3D.

esp::scene::SemanticScene class #include <esp/scene/SemanticScene.h>