Namespaces
namespace	common
namespace	detail
namespace	libtorch
namespace	onnx
namespace	osd
namespace	sam3
namespace	tensor_rt
namespace	torchvision

Classes
struct	AnnotationConfig
	Configuration for ML annotation inference and data loading. More...
class	BackendFactory
class	Dataloader
struct	DescribeConfig
	Options for the VLM attribute-extraction stage. More...
struct	IData
class	IDataset
class	IMLBackend
class	IModel
class	IVideoModel
struct	IVlmClient
	A model that, given an image crop and a text prompt, returns structured attributes. More...
class	OpenAiCompatibleVlmClient
struct	ProjectedBox
	A projected 2D bounding box of an instance in one frame, with the count of points that landed inside the image. More...
class	Sam3GeometryFailures
	When a request's geometry path fails at run time (encode_boxes / decode with a geometry slot), the request is re-run text-only. More...
struct	Sam3Prompt
	One SAM3 prompt: text class name plus optional positive/negative bounding boxes and positive click points. More...
struct	SegmentImageInfo
	What segment_image() can report beyond the label map. More...
struct	SegmentPanoramaOptions
	Options controlling how a panorama is tiled and segmented. More...
struct	VlmResult
	The description + arbitrary attributes a vision-language model returned for one crop. More...

Typedefs
using	SegmentBox = std::pair<std::string, std::array<float, 4>>
	A bounding-box prompt: label ("pos" or "neg") and pixel coords [x1,y1,x2,y2].
using	SegmentPoint = std::array<float, 2>
	A click-point prompt: pixel [x, y].

Enumerations
enum class	Backend { opencv , tensor_rt , libtorch , dnn , onnx_runtime , openvino , unknown }
enum class	Model { yolo , sam3 , sam3p1 }

Functions
auto	annotate (const std::filesystem::path &dbPath, const std::filesystem::path &modelPath, const AnnotationConfig &config=AnnotationConfig{}) -> int
	Run semantic annotation on sensor frames using ML models.
bool	is_sequence_boundary (std::size_t index, int node_id, int prev_id)
	Whether the video tracker must reset before the frame at `index`.
std::string	default_describe_prompt ()
	The default instruction: asks for a JSON object with a free-text description and an arbitrary attributes map.
int	describe (const std::filesystem::path &dbPath, const DescribeConfig &config)
	Run the attribute stage against the project at `dbPath`, constructing the concrete OpenAI-compatible client from `config`.
int	describe_with_client (const std::filesystem::path &dbPath, const DescribeConfig &config, IVlmClient &client)
	Testable core: run the stage with an injected client (no network).
const std::vector< std::string > &	glass_prompt_list ()
	Returns the canonical list of glass/mirror/transparent-surface prompt strings used by the glass depth filter.
bool	is_glass_class (std::string_view class_name)
	Returns true when class_name matches one of the glass prompt strings.
cv::Mat	build_glass_confidence_map (const cv::Mat &label_image, const std::vector< int > &glass_ids)
	Build a per-frame binary glass confidence map from a semantic label image.
std::string	find_duplicate_prompt_concept (const std::vector< std::string > &prompts)
	Returns the first duplicate concept text found in `prompts` after stripping optional :threshold suffixes, or an empty string if the list is duplicate-free.
std::optional< ProjectedBox >	project_points_to_box (const std::vector< Eigen::Vector3f > &world_points, const std::array< double, 16 > &pose, const core::SensorIntrinsics &intrinsics, int image_width, int image_height)
	Project `world_points` into the camera described by `pose` (row-major 4x4 base-to-world) and `intrinsics`, returning the pixel bounding box of the points that fall in front of the camera AND inside the image.
std::unique_ptr< IModel >	create_model_from_path (const std::filesystem::path &modelPath, bool use_cuda)
	Detect the backend + model kind from `modelPath` and build the model.
torch::Tensor	xyxy_to_xywh (const torch::Tensor &x)
	Convert bounding boxes from (x1,y1,x2,y2) to (cx,cy,w,h) format.
torch::Tensor	xywh_to_xyxy (const torch::Tensor &x)
	Convert bounding boxes from (cx,cy,w,h) to (x1,y1,x2,y2) format.
torch::Tensor	nms (const torch::Tensor &bboxes, const torch::Tensor &scores, float iou_threshold=0.45)
	Non-maximum suppression on bounding boxes.
torch::Tensor	non_max_suppression (torch::Tensor predictions, float confThreshold=0.25, float iouThreshold=0.45, int maxDetections=300)
	YOLO-style non-maximum suppression with class-aware filtering.
auto	project (const ProjectDB &db, CloudConstPtr cloud) -> CloudLPtr
	Project the labels stored in the database on tho the assebled point cloud.
std::array< float, 4 >	sam3_box_cxcywh (const std::array< float, 4 > &xyxy, int image_w, int image_h)
	Pixel box [x1, y1, x2, y2] (corners in any order) -> the normalised [cx, cy, w, h] the SAM3 geometry encoder takes, clamped to the `image_w` x `image_h` image.
std::int64_t	sam3_box_label (std::string_view polarity) noexcept
	SAM3 box label: 1 for a positive ("pos") box, 0 for a negative one.
int	sam3_geometry_slot_len (int max_boxes) noexcept
	Geometry tokens one prompt slot holds when the largest prompt in a decoder batch has `max_boxes` boxes: the geometry encoder emits one token per box plus a CLS token, so max_boxes + 1, or 0 when no prompt has a box.
std::vector< std::uint8_t >	sam3_geometry_slot_mask (int n_boxes, int slot_len)
	Validity of one prompt's geometry slot of `slot_len` tokens holding `n_boxes` encoded boxes: 1 == a token the decoder attends to, 0 == padding.
std::optional< bool >	sam3_mask_true_is_valid (const std::vector< std::uint8_t > &mask, int real_tokens)
	Which way an export's bool token masks point, read off the text encoder's own output for an input whose first `real_tokens` tokens are real and the rest padding.
constexpr bool	sam3_mask_value (bool valid, bool true_is_valid) noexcept
	The bool an engine mask holds for a token that is `valid`, under the polarity sam3_mask_true_is_valid() found.
int	sam3_geometry_box_count (std::size_t requested, int capacity) noexcept
	Boxes actually fed for a prompt with `requested` boxes when the model accepts at most `capacity` (0 = geometry disabled).
int	sam3_geometry_capacity (int model_cap, int encoder_min_boxes, int encoder_max_boxes, int text_len, int decoder_max_len) noexcept
	Most boxes per prompt the engines can take, or 0 when geometry prompting is unusable.
bool	sam3_detection_selected (const std::array< float, 4 > &det_xyxy, const std::vector< SegmentBox > &boxes, const std::vector< std::array< float, 2 > > &points={}, const std::function< bool(float, float)> &covers={})
	Does a detection with pixel box `det_xyxy` belong to a box-prompted selection?
const std::vector< float > &	sam3_point_exemplar_fracs ()
	Exemplar box half-sizes, as fractions of the image's shorter side, a click point is tried at.
std::array< float, 4 >	sam3_point_exemplar_box (const std::array< float, 2 > &point, int image_w, int image_h, float frac)
	The exemplar box [x1, y1, x2, y2] for a click point: half-size `frac` of the image's shorter side, clamped to the image.
std::vector< std::size_t >	sam3_best_detection_per_point (const std::vector< float > &scores, const std::vector< std::vector< bool > > &covers)
	Which detections a click selects: for each point, the single highest- scoring detection that covers it (`covers`[d][p]: detection d covers point p).
std::vector< std::size_t >	sam3_point_prompt_selection (const std::vector< float > &scores, const std::vector< std::vector< bool > > &covers, const std::vector< bool > &box_selected)
	The detections a prompt with click points keeps: the UNION of.
int	sam3_label_value (bool by_prompt_index, int prompt_index, int cache_id) noexcept
	The value a backend writes into the label map for one prompt's detections.
std::size_t	clip_labels_to_prompt_boxes (cv::Mat &labels, const std::vector< Sam3Prompt > &prompts)
	Segment a single BGR image with a SAM3 model.
cv::Mat	segment_image (IModel &model, const cv::Mat &image_bgr, const std::vector< Sam3Prompt > &prompts={}, float confidence=0.5f, SegmentImageInfo *info=nullptr)
	Segment a single BGR image with a SAM3 model (see the parameters above).
cv::Mat	segment_panorama (IModel &model, const cv::Mat &equirect_bgr, const SegmentPanoramaOptions &opts)
	Segment a 360 equirectangular panorama with a SAM3 model.
float	generate_scale (const cv::Mat &image, const cv::Size &target_size, bool scale_up=false)
float	letterbox (cv::Mat &input_image, cv::Mat &output_image, const cv::Size &target_size)
float	cropbox (cv::Mat &input_image, cv::Mat &output_image, const cv::Size &target_size)
nlohmann::json	build_chat_request_json (const std::string &model, const std::string &b64_jpeg, const std::string &prompt)
	Build an OpenAI-compatible chat/completions request body for one image.
VlmResult	parse_vlm_response (const std::string &body)
	Parse an OpenAI-compatible chat/completions response into a VlmResult.

Variables
constexpr std::string_view	kGeometryOnlyPromptText = "visual"
	Text sent to SAM3 for a prompt that carries boxes but no class name.
std::vector< std::string >	Yolov8_className
	YOLO v8 class names for object detection.

Typedef Documentation

◆ SegmentBox

using reusex::vision::SegmentBox = std::pair<std::string, std::array<float, 4>>

A bounding-box prompt: label ("pos" or "neg") and pixel coords [x1,y1,x2,y2].

Definition at line 20 of file sam3_prompt.hpp.

◆ SegmentPoint

using reusex::vision::SegmentPoint = std::array<float, 2>

A click-point prompt: pixel [x, y].

Definition at line 43 of file sam3_prompt.hpp.

Enumeration Type Documentation

◆ Backend

enum class reusex::vision::Backend

strong

Enumerator
opencv
tensor_rt
libtorch
dnn
onnx_runtime
openvino
unknown

Definition at line 32 of file BackendFactory.hpp.

◆ Model

enum class reusex::vision::Model

strong

Enumerator
yolo
sam3
sam3p1

Definition at line 15 of file IMLBackend.hpp.

Function Documentation

◆ annotate()

auto reusex::vision::annotate	(	const std::filesystem::path &	dbPath,
		const std::filesystem::path &	modelPath,
		const AnnotationConfig &	config = AnnotationConfig{} ) -> int

Run semantic annotation on sensor frames using ML models.

Parameters

dbPath	Path to project database containing sensor frames
modelPath	Path to ML model (.pt, .engine, .onnx)
config	Configuration for inference and data loading

Returns: 0 on success, error code on failure

References annotate().

Referenced by annotate().

◆ build_chat_request_json()

nlohmann::json reusex::vision::build_chat_request_json	(	const std::string &	model,
		const std::string &	b64_jpeg,
		const std::string &	prompt )

inline

Build an OpenAI-compatible chat/completions request body for one image.

Parameters

model	Model name the endpoint should route to (e.g. "qwen2.5vl").
b64_jpeg	Base64-encoded JPEG bytes of the object crop (no data URI prefix — this function adds the data:image/jpeg;base64,).
prompt	The instruction text sent alongside the image.

Determinism: temperature is pinned to 0. JSON output is requested via BOTH the Ollama format: "json" key and the OpenAI response_format: {type:"json_object"} key — a server ignores whichever it does not know, so including both keeps the same body usable across providers (STANDARDS §6).

Definition at line 51 of file vlm_protocol.hpp.

References build_chat_request_json().

Referenced by build_chat_request_json().

◆ build_glass_confidence_map()

cv::Mat reusex::vision::build_glass_confidence_map	(	const cv::Mat &	label_image,
		const std::vector< int > &	glass_ids )

Build a per-frame binary glass confidence map from a semantic label image.

glass_ids: the 0-based class IDs that correspond to glass/mirror classes in the label image (these are the positions of glass prompts in the merged prompt list). label_image must be CV_32S (-1 = background, 0..N = class).

Returns CV_8U mat the same size as label_image: 255 = trust depth (non-glass or unlabeled) 0 = suppress depth (glass/mirror detected)

An empty glass_ids or empty label_image returns an all-255 map.

References build_glass_confidence_map().

Referenced by build_glass_confidence_map().

◆ clip_labels_to_prompt_boxes()

std::size_t reusex::vision::clip_labels_to_prompt_boxes	(	cv::Mat &	labels,
		const std::vector< Sam3Prompt > &	prompts )

Segment a single BGR image with a SAM3 model.

Parameters

model	A SAM3 model created via create_model_from_path(). Must be TensorRT or ONNX; other types return an all-(-1) map.
image_bgr	Input BGR image (CV_8UC3, non-empty).
prompts	SAM3 prompts (text + optional boxes). Empty ⟹ model default.
confidence	Global detection confidence threshold [0,1]. Per-prompt Sam3Prompt::confidence overrides when >= 0.

Returns: CV_32S label map the same size as image_bgr. -1 = background; 0..N = class index matching prompt order. Returns an empty Mat on empty input; all-(-1) on no detections. When the backend cannot use boxes (see clip_labels_to_prompt_boxes), boxed prompts are clipped to their boxes and a warning is logged once per process. Restrict each boxed prompt's pixels to its boxes — the fallback for a backend that cannot feed boxes to SAM3 (ONNX, or TensorRT engines that cannot take geometry, see TensorRTSam3), where a boxed prompt would otherwise segment its text concept across the whole image.

For prompt k (label value k in labels, the prompt-index numbering) with at least one box: when it has "pos" boxes, its pixels outside the union of those boxes become background (-1); pixels inside any of its "neg" boxes become background too. Prompts without boxes and other labels are left alone. Box coordinates are pixels of labels, clamped to the image. Pixel clipping (not an IoU gate on whole detections) is the chosen rule: the label map no longer carries per-detection boxes, and clipping keeps the part of a detection the user actually boxed.

Parameters

labels CV_32S label map, modified in place.

Returns: Number of pixels set to background.

References clip_labels_to_prompt_boxes().

Referenced by clip_labels_to_prompt_boxes().

◆ create_model_from_path()

std::unique_ptr< IModel > reusex::vision::create_model_from_path	(	const std::filesystem::path &	modelPath,
		bool	use_cuda )

Detect the backend + model kind from modelPath and build the model.

Exceptions

std::runtime_error if the required backend is not compiled in.

References create_model_from_path().

Referenced by create_model_from_path().

◆ cropbox()

float reusex::vision::cropbox	(	cv::Mat &	input_image,
		cv::Mat &	output_image,
		const cv::Size &	target_size )

References cropbox().

Referenced by cropbox().

◆ default_describe_prompt()

std::string reusex::vision::default_describe_prompt ( )

The default instruction: asks for a JSON object with a free-text description and an arbitrary attributes map.

Exposed so the CLI help can show it and tests can pin it. The caller may override it to request any keys.

References default_describe_prompt().

Referenced by default_describe_prompt().

◆ describe()

int reusex::vision::describe	(	const std::filesystem::path &	dbPath,
		const DescribeConfig &	config )

Run the attribute stage against the project at dbPath, constructing the concrete OpenAI-compatible client from config.

Returns: 0 on success, non-zero on a fatal error. A per-material VLM failure is logged and skipped, not fatal (STANDARDS §5).

References describe().

Referenced by describe().

◆ describe_with_client()

int reusex::vision::describe_with_client	(	const std::filesystem::path &	dbPath,
		const DescribeConfig &	config,
		IVlmClient &	client )

Testable core: run the stage with an injected client (no network).

Used by unit tests and by the network entry point above.

Returns: the number of materials for which an annotation was stored.

References describe_with_client().

Referenced by describe_with_client().

◆ find_duplicate_prompt_concept()

std::string reusex::vision::find_duplicate_prompt_concept ( const std::vector< std::string > & prompts )

Returns the first duplicate concept text found in prompts after stripping optional :threshold suffixes, or an empty string if the list is duplicate-free.

Mirrors the first-seen dedup order that the model applies internally, so callers can validate their prompt list before computing glass class IDs.

References find_duplicate_prompt_concept().

Referenced by find_duplicate_prompt_concept().

◆ generate_scale()

float reusex::vision::generate_scale	(	const cv::Mat &	image,
		const cv::Size &	target_size,
		bool	scale_up = false )

References generate_scale().

Referenced by generate_scale().

◆ glass_prompt_list()

const std::vector< std::string > & reusex::vision::glass_prompt_list ( )

Returns the canonical list of glass/mirror/transparent-surface prompt strings used by the glass depth filter.

These are appended to the structural prompt list when –glass-filter is requested.

References glass_prompt_list().

Referenced by glass_prompt_list().

◆ is_glass_class()

bool reusex::vision::is_glass_class ( std::string_view class_name )

Returns true when class_name matches one of the glass prompt strings.

References is_glass_class().

Referenced by is_glass_class().

◆ is_sequence_boundary()

bool reusex::vision::is_sequence_boundary	(	std::size_t	index,
		int	node_id,
		int	prev_id )

inline

Whether the video tracker must reset before the frame at index.

A stateful video model must start a fresh temporal memory at the beginning of every ordered sequence. Within one scan the sensor-frame node ids increase monotonically (small forward gaps from dropped frames are normal and must NOT trigger a reset), so a boundary is only the very first frame (index 0) or a point where the node id fails to increase — which happens when a dataset concatenates multiple sequences that restart their numbering.

Kept as a free function (independent of any tracker/dataset object) so the boundary logic can be unit-tested directly.

Parameters

index	Position in the (possibly prefix-trimmed) ordered list.
node_id	Node id of the frame at `index`.
prev_id	Node id of the frame at `index-1` (unused when index==0).

Returns: true if the tracker should reset before processing this frame.

Definition at line 72 of file annotate.hpp.

References is_sequence_boundary().

Referenced by is_sequence_boundary().

◆ letterbox()

float reusex::vision::letterbox	(	cv::Mat &	input_image,
		cv::Mat &	output_image,
		const cv::Size &	target_size )

References letterbox().

Referenced by letterbox().

◆ nms()

torch::Tensor reusex::vision::nms	(	const torch::Tensor &	bboxes,
		const torch::Tensor &	scores,
		float	iou_threshold = 0.45 )

Non-maximum suppression on bounding boxes.

The algorithm is torchvision's reference CPU kernel, vendored under BSD-3-Clause in vision/third_party/torchvision_nms_kernel.hpp (#141). This wrapper owns the ReUseX-facing contract: CPU, float32, empty input tolerated.

Boxes are suppressed when IoU is strictly greater than iou_threshold, so a threshold of 1.0 suppresses nothing and ties at the threshold survive. Equal scores are broken towards the lower index (the sort is stable).

Parameters

bboxes	Bounding boxes tensor [N, 4] in xyxy format. CPU, float32.
scores	Confidence scores tensor [N]. CPU, float32.
iou_threshold	IoU threshold for suppression.

Returns: Indices of kept boxes, in descending score order (int64).

Exceptions

c10::Error if the tensors are not CPU float32, or are malformed.

References nms().

Referenced by nms().

◆ non_max_suppression()

torch::Tensor reusex::vision::non_max_suppression	(	torch::Tensor	predictions,
		float	confThreshold = 0.25,
		float	iouThreshold = 0.45,
		int	maxDetections = 300 )

YOLO-style non-maximum suppression with class-aware filtering.

Parameters

predictions	Raw model output [batch_size, 116, 8400].
confThreshold	Confidence threshold for filtering.
iouThreshold	IoU threshold for NMS.
maxDetections	Maximum detections to keep per image.

Returns: Filtered detections [batch_size, maxDetections, 6+32].

References non_max_suppression().

Referenced by non_max_suppression().

◆ parse_vlm_response()

VlmResult reusex::vision::parse_vlm_response ( const std::string & body )

inline

Parse an OpenAI-compatible chat/completions response into a VlmResult.

Defensive by design (STANDARDS §5): the assistant's content is often JSON wrapped in a markdown ``json fence, so this function unwraps the fence and parses whatever JSON object it finds. It accepts BOTH shapes the prompt can elicit:

nested:

{ "description": "...", "attributes": { k: v, ... } } — the description is the free text and every scalar under attributes is a key/value pair;

flat: any object where a

description/desc/summary` key is the free text and every OTHER scalar field is treated as an attribute. Numbers and bools become stringified attribute values; nested containers are ignored (they are not scalar values). Nothing is fabricated.

ok is true ONLY when a non-empty description OR at least one attribute was recovered. In particular:

a chat envelope whose assistant content is empty does NOT fall back to re-parsing the whole envelope object (that would "recover" the envelope itself and falsely report success on a model that returned nothing);
an object carrying neither a description nor any attribute leaves ok false so the stage never stores an all-empty annotation. The full response body is always preserved in raw_json. Attributes are returned in the model's key order de-duplicated (last value wins), which the storage layer then keys deterministically.

Definition at line 194 of file vlm_protocol.hpp.

References reusex::vision::VlmResult::attributes, reusex::vision::VlmResult::description, reusex::vision::detail::extract_message_content(), reusex::vision::detail::is_chat_envelope(), reusex::vision::detail::is_description_key(), reusex::vision::VlmResult::ok, parse_vlm_response(), reusex::vision::VlmResult::raw_json, reusex::vision::detail::scalar_to_string(), and reusex::vision::detail::strip_json_fence().

Referenced by parse_vlm_response().

◆ project()

auto reusex::vision::project	(	const ProjectDB &	db,
		CloudConstPtr	cloud ) -> CloudLPtr

Project the labels stored in the database on tho the assebled point cloud.

For each view in the database this constructs a croped point cloud, computes a z-buffer and the assigns the precomputed labels to the closses point in the point cloud.

References project().

Referenced by project().

◆ project_points_to_box()

std::optional< ProjectedBox > reusex::vision::project_points_to_box	(	const std::vector< Eigen::Vector3f > &	world_points,
		const std::array< double, 16 > &	pose,
		const core::SensorIntrinsics &	intrinsics,
		int	image_width,
		int	image_height )

Project world_points into the camera described by pose (row-major 4x4 base-to-world) and intrinsics, returning the pixel bounding box of the points that fall in front of the camera AND inside the image.

Returns nullopt when no point projects in-bounds. Deterministic: no randomness, results depend only on the inputs.

Parameters

image_width/

param image_height Target image size the box is clamped to; intrinsics are scaled from their native size to this.

References project_points_to_box().

Referenced by project_points_to_box().

◆ sam3_best_detection_per_point()

std::vector< std::size_t > reusex::vision::sam3_best_detection_per_point	(	const std::vector< float > &	scores,
		const std::vector< std::vector< bool > > &	covers )

Which detections a click selects: for each point, the single highest- scoring detection that covers it (covers[d][p]: detection d covers point p).

Returns the chosen detection indices, ascending, without duplicates; a point no detection covers selects nothing.

References sam3_best_detection_per_point().

Referenced by sam3_best_detection_per_point().

◆ sam3_box_cxcywh()

std::array< float, 4 > reusex::vision::sam3_box_cxcywh	(	const std::array< float, 4 > &	xyxy,
		int	image_w,
		int	image_h )

Pixel box [x1, y1, x2, y2] (corners in any order) -> the normalised [cx, cy, w, h] the SAM3 geometry encoder takes, clamped to the image_w x image_h image.

A degenerate image size yields all zeros.

References sam3_box_cxcywh().

Referenced by sam3_box_cxcywh().

◆ sam3_box_label()

std::int64_t reusex::vision::sam3_box_label ( std::string_view polarity )

noexcept

SAM3 box label: 1 for a positive ("pos") box, 0 for a negative one.

References sam3_box_label().

Referenced by sam3_box_label().

◆ sam3_detection_selected()

bool reusex::vision::sam3_detection_selected	(	const std::array< float, 4 > &	det_xyxy,
		const std::vector< SegmentBox > &	boxes,
		const std::vector< std::array< float, 2 > > &	points = {},
		const std::function< bool(float, float)> &	covers = {} )

Does a detection with pixel box det_xyxy belong to a box-prompted selection?

SAM3 treats a prompt box as an exemplar and returns every similar object; a selection keeps only what the boxes point at:

with positive boxes, a detection is kept when one of them covers at least half of the detection (the box drawn around the object) or at least half of that box lies inside the detection (a small box drawn on a larger object);
a detection at least half inside a negative box is dropped;
with no positive box and no point, everything not dropped is kept;
a detection that covers one of points (click points) is kept; "covers" is covers(x, y) — the detection's mask — or, without it, its box. A zero-area detection is never kept.

References sam3_detection_selected().

Referenced by sam3_detection_selected().

◆ sam3_geometry_box_count()

int reusex::vision::sam3_geometry_box_count	(	std::size_t	requested,
		int	capacity )

noexcept

Boxes actually fed for a prompt with requested boxes when the model accepts at most capacity (0 = geometry disabled).

References sam3_geometry_box_count().

Referenced by sam3_geometry_box_count().

◆ sam3_geometry_capacity()

int reusex::vision::sam3_geometry_capacity	(	int	model_cap,
		int	encoder_min_boxes,
		int	encoder_max_boxes,
		int	text_len,
		int	decoder_max_len )

noexcept

Most boxes per prompt the engines can take, or 0 when geometry prompting is unusable.

model_cap is the backend's own cap; encoder_min_boxes / encoder_max_boxes the geometry encoder's num_boxes profile range; text_len the text tokens per prompt and decoder_max_len the decoder's prompt_len profile max. Needs an encoder that accepts one box and a decoder with room for at least one box token plus the CLS token.

References sam3_geometry_capacity().

Referenced by sam3_geometry_capacity().

◆ sam3_geometry_slot_len()

int reusex::vision::sam3_geometry_slot_len ( int max_boxes )

noexcept

Geometry tokens one prompt slot holds when the largest prompt in a decoder batch has max_boxes boxes: the geometry encoder emits one token per box plus a CLS token, so max_boxes + 1, or 0 when no prompt has a box.

References sam3_geometry_slot_len().

Referenced by sam3_geometry_slot_len().

◆ sam3_geometry_slot_mask()

std::vector< std::uint8_t > reusex::vision::sam3_geometry_slot_mask	(	int	n_boxes,
		int	slot_len )

Validity of one prompt's geometry slot of slot_len tokens holding n_boxes encoded boxes: 1 == a token the decoder attends to, 0 == padding.

The first n_boxes + 1 tokens (boxes + CLS) are valid, the rest padding. A prompt with no boxes masks its whole slot — not even the CLS token — so it decodes exactly as on the text-only path. Convert to the engines' bool polarity with sam3_mask_value().

References sam3_geometry_slot_mask().

Referenced by sam3_geometry_slot_mask().

◆ sam3_label_value()

int reusex::vision::sam3_label_value	(	bool	by_prompt_index,
		int	prompt_index,
		int	cache_id )

inlinenoexcept

The value a backend writes into the label map for one prompt's detections.

With by_prompt_index (single-image callers such as segment_image / segment_panorama) it is the prompt's position in the request, so label k always means prompt k and two prompts with the same text stay distinct. Without it (the rux create annotate dataset path) it is cache_id, the backend's per-text id that stays stable across every frame of a run. A negative prompt_index (no prompt list) always falls back to cache_id.

Definition at line 37 of file sam3_prompt.hpp.

References sam3_label_value().

Referenced by sam3_label_value().

◆ sam3_mask_true_is_valid()

std::optional< bool > reusex::vision::sam3_mask_true_is_valid	(	const std::vector< std::uint8_t > &	mask,
		int	real_tokens )

Which way an export's bool token masks point, read off the text encoder's own output for an input whose first real_tokens tokens are real and the rest padding.

The released sam3.1-onnx-v1 graphs use True == valid token (their text_mask is attention_mask > 0); the in-repo exporter's wrappers write True == padding (attention_mask == 0). The decoder reads its prompt_mask in the text encoder's polarity, since the text path copies text_mask straight in. Returns true for True == valid, false for True == padding, and std::nullopt when the mask does not split cleanly into the two runs (a broken probe: do not guess).

References sam3_mask_true_is_valid().

Referenced by sam3_mask_true_is_valid().

◆ sam3_mask_value()

bool reusex::vision::sam3_mask_value	(	bool	valid,
		bool	true_is_valid )

constexprnoexcept

The bool an engine mask holds for a token that is valid, under the polarity sam3_mask_true_is_valid() found.

Definition at line 62 of file sam3_geometry.hpp.

References sam3_mask_value().

Referenced by sam3_mask_value().

◆ sam3_point_exemplar_box()

std::array< float, 4 > reusex::vision::sam3_point_exemplar_box	(	const std::array< float, 2 > &	point,
		int	image_w,
		int	image_h,
		float	frac )

The exemplar box [x1, y1, x2, y2] for a click point: half-size frac of the image's shorter side, clamped to the image.

References sam3_point_exemplar_box().

Referenced by sam3_point_exemplar_box().

◆ sam3_point_exemplar_fracs()

const std::vector< float > & reusex::vision::sam3_point_exemplar_fracs ( )

Exemplar box half-sizes, as fractions of the image's shorter side, a click point is tried at.

SAM3's exported detector takes boxes only, and a box is an exemplar: the detection it yields at the point tracks the box's size (a small box finds the patch under the click, a large one the surrounding furniture). Trying several sizes and keeping the most confident detection under the point picks the object's own scale (measured on NewOffice frame 1500: monitor, chair, bag and pillar each win at a different size).

References sam3_point_exemplar_fracs().

Referenced by sam3_point_exemplar_fracs().

◆ sam3_point_prompt_selection()

std::vector< std::size_t > reusex::vision::sam3_point_prompt_selection	(	const std::vector< float > &	scores,
		const std::vector< std::vector< bool > > &	covers,
		const std::vector< bool > &	box_selected )

The detections a prompt with click points keeps: the UNION of.

per point, the single most confident detection covering it (sam3_best_detection_per_point over scores / covers), and
every detection its boxes select (box_selected[d]; the caller counts each box hit once, not once per exemplar-size pass). A detection that is both is listed once. Returns ascending indices.

References sam3_point_prompt_selection().

Referenced by sam3_point_prompt_selection().

◆ segment_image()

cv::Mat reusex::vision::segment_image	(	IModel &	model,
		const cv::Mat &	image_bgr,
		const std::vector< Sam3Prompt > &	prompts = {},
		float	confidence = 0.5f,
		SegmentImageInfo *	info = nullptr )

Segment a single BGR image with a SAM3 model (see the parameters above).

Parameters

info	Optional out-parameter; see SegmentImageInfo.

References segment_image().

Referenced by segment_image().

◆ segment_panorama()

cv::Mat reusex::vision::segment_panorama	(	IModel &	model,
		const cv::Mat &	equirect_bgr,
		const SegmentPanoramaOptions &	opts )

Segment a 360 equirectangular panorama with a SAM3 model.

Parameters

model	SAM3 model handle (created via BackendFactory).
equirect_bgr	Equirectangular panorama (BGR, CV_8UC3).
opts	Tiling / prompt / confidence options.

Returns: A CV_32S equirect label map the same size as equirect_bgr, with -1 for background. Class ids are stable across tiles because a single model instance assigns each prompt text a fixed class id.

References segment_panorama().

Referenced by segment_panorama().

◆ xywh_to_xyxy()

torch::Tensor reusex::vision::xywh_to_xyxy ( const torch::Tensor & x )

Convert bounding boxes from (cx,cy,w,h) to (x1,y1,x2,y2) format.

References xywh_to_xyxy().

Referenced by xywh_to_xyxy().

◆ xyxy_to_xywh()

torch::Tensor reusex::vision::xyxy_to_xywh ( const torch::Tensor & x )

Convert bounding boxes from (x1,y1,x2,y2) to (cx,cy,w,h) format.

References xyxy_to_xywh().

Referenced by xyxy_to_xywh().

Variable Documentation

◆ kGeometryOnlyPromptText

std::string_view reusex::vision::kGeometryOnlyPromptText = "visual"

inlineconstexpr

Text sent to SAM3 for a prompt that carries boxes but no class name.

This is the upstream SAM3 convention for a geometry-only prompt: Meta's Sam3Processor.add_geometric_prompt encodes the text "visual" when only geometry is given, so the model relies on the boxes. Shared by the GUI endpoint and rux create segment-frame so both send the same text.

Definition at line 27 of file sam3_prompt.hpp.

◆ Yolov8_className

std::vector<std::string> reusex::vision::Yolov8_className

extern

YOLO v8 class names for object detection.

Contains the list of class names that can be detected by YOLO v8 model.

Namespaces

Classes

Typedefs

Enumerations

Functions

Variables

Typedef Documentation

◆ SegmentBox

◆ SegmentPoint

Enumeration Type Documentation

◆ Backend

◆ Model

Function Documentation

◆ annotate()

◆ build_chat_request_json()

◆ build_glass_confidence_map()

◆ clip_labels_to_prompt_boxes()

◆ create_model_from_path()

◆ cropbox()

◆ default_describe_prompt()

◆ describe()

◆ describe_with_client()

◆ find_duplicate_prompt_concept()

◆ generate_scale()

◆ glass_prompt_list()

◆ is_glass_class()

◆ is_sequence_boundary()

◆ letterbox()

◆ nms()

◆ non_max_suppression()

◆ parse_vlm_response()

◆ project()

◆ project_points_to_box()

◆ sam3_best_detection_per_point()

◆ sam3_box_cxcywh()

◆ sam3_box_label()

◆ sam3_detection_selected()

◆ sam3_geometry_box_count()

◆ sam3_geometry_capacity()

◆ sam3_geometry_slot_len()

◆ sam3_geometry_slot_mask()

◆ sam3_label_value()

◆ sam3_mask_true_is_valid()

◆ sam3_mask_value()

◆ sam3_point_exemplar_box()

◆ sam3_point_exemplar_fracs()

◆ sam3_point_prompt_selection()

◆ segment_image()

◆ segment_panorama()

◆ xywh_to_xyxy()

◆ xyxy_to_xywh()

Variable Documentation

◆ kGeometryOnlyPromptText

◆ Yolov8_className